From b5dd9372a74a4197efbd19a6a0d6bbff48711aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 3 Sep 2026 02:22:07 +0200 Subject: [PATCH 1/3] fix: give the weekly backup a run window of its own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `data-backup` queue was registered without send options, so every job it minted took pg-boss's queue defaults: a fifteen-minute expiration and two redeliveries. A snapshot that serialises every account's whole record does not fit in fifteen minutes on a large one, so the pass was killed part-way through, redelivered twice against the same record, and reached `failed` with "job timed out" about three quarters of an hour after it started — having never once completed. Two hours, on the schedule rather than on the queue: `create_queue()` ends in ON CONFLICT DO NOTHING, so a queue-level default would apply to a fresh database and be inert on every instance that already has the queue. The admin run-now route sends with the same options, or an ad-hoc snapshot would silently keep the default the cron was rescued from. The default two retries stay. The old redeliveries were pointless because the first attempt could not succeed either; a transient failure genuinely benefits from a second go, and the alternative is losing the week. --- src/app/api/admin/backups/run/route.ts | 17 ++++-- .../__tests__/data-backup-send-policy.test.ts | 55 +++++++++++++++++++ .../__tests__/job-failures-retention.test.ts | 12 ++++ src/lib/jobs/data-backup-policy.ts | 39 +++++++++++++ src/lib/jobs/reminder/register-maintenance.ts | 12 +++- 5 files changed, 127 insertions(+), 8 deletions(-) create mode 100644 src/lib/jobs/__tests__/data-backup-send-policy.test.ts create mode 100644 src/lib/jobs/data-backup-policy.ts diff --git a/src/app/api/admin/backups/run/route.ts b/src/app/api/admin/backups/run/route.ts index 856705c10..0c730130a 100644 --- a/src/app/api/admin/backups/run/route.ts +++ b/src/app/api/admin/backups/run/route.ts @@ -15,11 +15,13 @@ import { auditLog } from "@/lib/auth/audit"; import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { checkRateLimit } from "@/lib/rate-limit"; +import { + DATA_BACKUP_QUEUE, + DATA_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/data-backup-policy"; export const dynamic = "force-dynamic"; -const DATA_BACKUP_QUEUE = "data-backup"; - export const POST = apiHandler(async (request: NextRequest) => { const { user: admin } = await requireAdmin(); annotate({ action: { name: "admin.backups.run" } }); @@ -57,9 +59,14 @@ export const POST = apiHandler(async (request: NextRequest) => { throw new HttpError(503, "Background worker is not running"); } - const jobId = await boss.send(DATA_BACKUP_QUEUE, { - triggeredAt: new Date().toISOString(), - }); + // Same expiration window as the weekly cron. An ad-hoc snapshot runs the + // identical pass over the identical record, so it must not silently keep the + // 15-minute queue default the cron was rescued from. + const jobId = await boss.send( + DATA_BACKUP_QUEUE, + { triggeredAt: new Date().toISOString() }, + DATA_BACKUP_SEND_OPTIONS, + ); annotate({ meta: { job_id: jobId ?? null } }); diff --git a/src/lib/jobs/__tests__/data-backup-send-policy.test.ts b/src/lib/jobs/__tests__/data-backup-send-policy.test.ts new file mode 100644 index 000000000..4071cbc2b --- /dev/null +++ b/src/lib/jobs/__tests__/data-backup-send-policy.test.ts @@ -0,0 +1,55 @@ +/** + * Send-policy pins for the weekly `data-backup` queue. + * + * The queue was registered with no send options at all, so every job it minted + * inherited pg-boss's queue defaults — `expireInSeconds: 900`, `retryLimit: 2`. + * A full-record snapshot of a large account does not fit in fifteen minutes, so + * the pass was killed mid-run, redelivered twice against the same record, and + * gave up roughly 46 minutes after it started. Weekly, silently, for as long as + * the record stayed large. + * + * Three facts have to hold together or the window is not real: + * + * 1. the policy names a window well past the 15-minute default, + * 2. the CRON SCHEDULE carries it — send options ride the schedule because + * pg-boss's `create_queue()` ends in ON CONFLICT DO NOTHING, so a + * queue-level default would be inert on every already-running instance, + * 3. the admin "run now" route sends with the same options, or an ad-hoc + * snapshot silently keeps the default that broke the cron one. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + DATA_BACKUP_QUEUE, + DATA_BACKUP_SEND_OPTIONS, +} from "../data-backup-policy"; + +const read = (file: string) => readFileSync(join(process.cwd(), file), "utf8"); + +describe("data-backup — pg-boss send policy", () => { + it("widens the expiration far past the 15-minute queue default", () => { + expect(DATA_BACKUP_QUEUE).toBe("data-backup"); + // 900 s is the pg-boss 12.26 queue default (`QUEUE_DEFAULTS.expire_seconds` + // in `dist/plans.js`); anything at or below it re-opens the bug. + expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThan(900); + expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThanOrEqual( + 60 * 60, + ); + }); + + it("the weekly schedule carries the policy", () => { + const source = read("src/lib/jobs/reminder/register-maintenance.ts"); + expect(source).toMatch( + /\[\s*DATA_BACKUP_QUEUE,\s*DATA_BACKUP_CRON,\s*DATA_BACKUP_SEND_OPTIONS,?\s*\]/, + ); + }); + + it("the admin run-now route sends with the same policy", () => { + const source = read("src/app/api/admin/backups/run/route.ts"); + expect(source).toMatch( + /boss\.send\(\s*DATA_BACKUP_QUEUE,[\s\S]{0,200}DATA_BACKUP_SEND_OPTIONS,?\s*\)/, + ); + }); +}); diff --git a/src/lib/jobs/__tests__/job-failures-retention.test.ts b/src/lib/jobs/__tests__/job-failures-retention.test.ts index f098283e1..ef3858164 100644 --- a/src/lib/jobs/__tests__/job-failures-retention.test.ts +++ b/src/lib/jobs/__tests__/job-failures-retention.test.ts @@ -26,4 +26,16 @@ describe("pg-boss failed-row ledger retention", () => { PG_BOSS_FAILED_ROW_AVAILABILITY_HOURS, ); }); + + it("pins the queue defaults the data-backup window had to escape", () => { + const plans = readFileSync(join(PG_BOSS_ROOT, "dist/plans.js"), "utf8"); + + // These two lines are the whole story behind the weekly backup's "job + // timed out": a pass with no send options gets fifteen minutes and two + // redeliveries, so a snapshot that cannot finish in fifteen minutes gives + // up 45 minutes later having never once completed. If a pg-boss upgrade + // changes either default, `DATA_BACKUP_SEND_OPTIONS` deserves a re-read. + expect(plans).toContain("expire_seconds: FIFTEEN_MINUTES"); + expect(plans).toContain("retry_limit: 2"); + }); }); diff --git a/src/lib/jobs/data-backup-policy.ts b/src/lib/jobs/data-backup-policy.ts new file mode 100644 index 000000000..22baef4d1 --- /dev/null +++ b/src/lib/jobs/data-backup-policy.ts @@ -0,0 +1,39 @@ +/** + * Queue name and pg-boss send policy for the weekly on-host data backup. + * + * A leaf module on purpose: the cron registrar, the admin "run now" route and + * the policy test all need these two facts, and none of them should have to + * pull in the worker's handler graph to get them. + * + * The queue was registered with no send options, so every job it minted took + * pg-boss 12.26's queue defaults — `expireInSeconds: 900`, `retryLimit: 2` + * (`QUEUE_DEFAULTS` in `pg-boss/dist/plans.js`). Fifteen minutes is a fine + * default for a cleanup tick and far too short for a pass that serialises every + * account's whole record: the job was killed mid-run, redelivered twice against + * the same record, and reached `failed` with `job timed out` about 46 minutes + * after it started. Nothing about that says "too slow" — the pass never got to + * finish once. + * + * Two hours is deliberately generous rather than tuned. The handler now costs a + * fraction of that on a large record (see the CHANGELOG entry), but the pass + * loops over EVERY account on the instance and a self-host's disk is whatever + * it is; the window's job is to be impossible to hit for the right reasons, and + * a failure that arrives at the two-hour mark is a real one worth reading. + * pg-boss refuses anything at or past 24 hours. + * + * The default `retryLimit: 2` stays. The old redeliveries were pointless + * because the first attempt could not succeed either; a transient failure — a + * pool hiccup, a locked row — genuinely benefits from a second attempt, and the + * alternative is losing the week. + * + * The options ride on the SEND, not on `createQueue`: pg-boss's `create_queue()` + * ends in `ON CONFLICT DO NOTHING`, so a queue-level default would apply to a + * fresh database and be inert on every instance that already has the queue — + * exactly the trap `reconcileQueuePolicies` exists to work around. + */ + +export const DATA_BACKUP_QUEUE = "data-backup"; + +export const DATA_BACKUP_SEND_OPTIONS = { + expireInSeconds: 2 * 60 * 60, +} as const; diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index b3faf909d..4d60ab352 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -138,6 +138,10 @@ import { } from "@/lib/jobs/environment-fetch"; import { recordError } from "@/lib/jobs/worker-status"; import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + DATA_BACKUP_QUEUE, + DATA_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/data-backup-policy"; import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared"; import { createAndSchedule, @@ -221,8 +225,6 @@ import { type AchievementUnlockSweepPayload, } from "@/lib/jobs/achievement-unlock-sweep"; -const DATA_BACKUP_QUEUE = "data-backup"; - const DATA_BACKUP_CRON = "0 3 * * 0"; // weekly Sunday at 03:00 const RATE_LIMIT_CLEANUP_QUEUE = "rate-limit-cleanup"; @@ -478,7 +480,11 @@ const schedules: ScheduleEntry[] = [ // v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload) // that fans out one per-user environment fetch per opted-in account. [ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON], - [DATA_BACKUP_QUEUE, DATA_BACKUP_CRON], + // The send options are the point of this tuple: without them the weekly + // snapshot inherits pg-boss's 15-minute expiration and is killed mid-run on + // any sizeable record. See `data-backup-policy.ts` for why the window rides + // the schedule rather than the queue. + [DATA_BACKUP_QUEUE, DATA_BACKUP_CRON, DATA_BACKUP_SEND_OPTIONS], [RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry], [IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry], [AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON], From d8196d5208cb9b54f18ddcd2b27acd677d8aefbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 3 Sep 2026 02:22:21 +0200 Subject: [PATCH 2/3] fix: compress the stored backup before encrypting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The weekly pass built the whole account in memory, serialised it whole, encrypted the string whole, and handed the result to the database whole. That is four full copies of the record at the same moment — the object graph, the JSON, the base64 ciphertext at 1.33x the JSON, and the copy the driver makes of that parameter. Measured against a seeded account of 445 000 readings, the JSON alone is 242 MB and the run dies of heap exhaustion; under a modest heap it never reaches the insert. The pass was not slow, it could not finish. A health record's JSON is extremely repetitive, so gzip takes that 242 MB to 14 MB and every copy after it shrinks with it. The same run completes in about twenty seconds with a peak heap of 434 MB and a stored row of 25 MB instead of 322 MB — which also matters because the copy lives inside the database it would be needed to restore. Both backup jobs now hand the serialised form back from a frame of its own, so the payload graph is released before the compress step allocates anything. The marker sits inside the ciphertext, so the stored column keeps the one shape `decrypt()` already understands and rows written before this still restore. The off-host object gets the same treatment through the version byte its envelope always carried; version 1 objects still read back, which both the restore drill and the restore script depend on. The obvious saving — dropping soft-deleted readings, four rows in five on a real record — is not taken, and the round-trip test now seeds a tombstone and asserts it makes the journey. A tombstone inside the retention window is what tells a synced client the row is gone and what makes a re-sent sample a no-op; a restore without them brings back every measurement the account ever deleted on the next device sync. --- .../api/admin/backups/[id]/download/route.ts | 4 +- .../api/admin/backups/[id]/restore/route.ts | 4 +- .../api/admin/backups/[id]/summary/route.ts | 4 +- src/app/api/admin/backups/upload/route.ts | 4 +- src/lib/export/__tests__/backup-blob.test.ts | 65 +++++++++++++++++++ src/lib/export/backup-blob.ts | 59 +++++++++++++++++ src/lib/export/full-backup-payload.ts | 20 ++++++ src/lib/jobs/__tests__/offhost-backup.test.ts | 39 ++++++++++- src/lib/jobs/offhost-backup.ts | 52 +++++++++++---- .../__tests__/backup-handlers.test.ts | 17 ++++- src/lib/jobs/reminder/backup-handlers.ts | 27 +++++--- .../integration/admin-backups-upload.test.ts | 12 ++-- tests/integration/backup-round-trip.test.ts | 61 ++++++++++++++++- 13 files changed, 330 insertions(+), 38 deletions(-) create mode 100644 src/lib/export/__tests__/backup-blob.test.ts create mode 100644 src/lib/export/backup-blob.ts diff --git a/src/app/api/admin/backups/[id]/download/route.ts b/src/app/api/admin/backups/[id]/download/route.ts index b063b0ced..0e55b6f7c 100644 --- a/src/app/api/admin/backups/[id]/download/route.ts +++ b/src/app/api/admin/backups/[id]/download/route.ts @@ -19,7 +19,7 @@ import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; -import { decrypt } from "@/lib/crypto"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; import { annotate } from "@/lib/logging/context"; import { parseBackupPayload } from "@/lib/validations/backup"; @@ -52,7 +52,7 @@ export const GET = apiHandler( let plaintext: string; try { - plaintext = decrypt(backup.data); + plaintext = unpackBackupBlob(backup.data); } catch (err) { // Decryption failure is rare but real — a rotated/missing key, or a // corrupted blob from a partially-failed write. Surface it as a 500 diff --git a/src/app/api/admin/backups/[id]/restore/route.ts b/src/app/api/admin/backups/[id]/restore/route.ts index ec71b6817..3deab49d4 100644 --- a/src/app/api/admin/backups/[id]/restore/route.ts +++ b/src/app/api/admin/backups/[id]/restore/route.ts @@ -19,7 +19,7 @@ import { prisma, toJson } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; -import { decrypt } from "@/lib/crypto"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; import { encryptNote } from "@/lib/crypto/note-cipher"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { encryptContextToBytes } from "@/lib/labs/biomarker-store"; @@ -180,7 +180,7 @@ const handler = apiHandler( let plaintext: string; try { - plaintext = decrypt(backup.data); + plaintext = unpackBackupBlob(backup.data); } catch (err) { await auditLog("admin.backups.restore.failed", { userId: admin.id, diff --git a/src/app/api/admin/backups/[id]/summary/route.ts b/src/app/api/admin/backups/[id]/summary/route.ts index fe32c81e7..3ae840d80 100644 --- a/src/app/api/admin/backups/[id]/summary/route.ts +++ b/src/app/api/admin/backups/[id]/summary/route.ts @@ -16,7 +16,7 @@ import { NextRequest } from "next/server"; import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, apiSuccess } from "@/lib/api-response"; -import { decrypt } from "@/lib/crypto"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; import { annotate } from "@/lib/logging/context"; import { isCompatibleSchemaVersion, @@ -44,7 +44,7 @@ export const GET = apiHandler( let plaintext: string; try { - plaintext = decrypt(backup.data); + plaintext = unpackBackupBlob(backup.data); } catch { return apiError("Failed to decrypt backup payload", 500); } diff --git a/src/app/api/admin/backups/upload/route.ts b/src/app/api/admin/backups/upload/route.ts index 28012e04b..b7b6443ea 100644 --- a/src/app/api/admin/backups/upload/route.ts +++ b/src/app/api/admin/backups/upload/route.ts @@ -26,7 +26,7 @@ import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; -import { encrypt } from "@/lib/crypto"; +import { packBackupBlob } from "@/lib/export/backup-blob"; import { annotate } from "@/lib/logging/context"; import { checkRateLimit } from "@/lib/rate-limit"; import { @@ -190,7 +190,7 @@ export const POST = apiHandler(async (request: NextRequest) => { // intact so multiple uploads can coexist for the same user without // overwriting each other or the rolling WEEKLY_AUTO snapshot. const uploadType = `MANUAL_UPLOAD_${Date.now()}`; - const encrypted = encrypt(JSON.stringify(payload)); + const encrypted = packBackupBlob(JSON.stringify(payload)); const created = await prisma.dataBackup.create({ data: { diff --git a/src/lib/export/__tests__/backup-blob.test.ts b/src/lib/export/__tests__/backup-blob.test.ts new file mode 100644 index 000000000..995ed07a6 --- /dev/null +++ b/src/lib/export/__tests__/backup-blob.test.ts @@ -0,0 +1,65 @@ +/** + * The stored-backup envelope: compress, then encrypt. + * + * The weekly worker used to hand `encrypt(JSON.stringify(payload))` straight to + * Postgres. For an account with a few hundred thousand measurements that means + * the object graph, the JSON string, the base64 ciphertext (1.33× the JSON) and + * the copy the database driver makes of it all alive at once — well over a + * gigabyte for a record whose JSON is a few hundred megabytes. The run died of + * memory, not of slow SQL. + * + * Compressing before the cipher is what makes the tail of that pipeline cheap: + * health JSON is extremely repetitive, so everything downstream of the gzip + * shrinks by an order of magnitude. The envelope has to stay readable both + * ways — rows written before this change are plain `encrypt(json)` and must + * keep restoring. + */ +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { describe, expect, it } from "vitest"; + +import { encrypt } from "@/lib/crypto"; +import { packBackupBlob, unpackBackupBlob } from "../backup-blob"; + +/** A record-shaped string: many rows, few distinct keys. */ +function sampleJson(rows: number): string { + return JSON.stringify({ + schemaVersion: "2", + measurements: Array.from({ length: rows }, (_, i) => ({ + id: `measurement-${i}`, + type: "PULSE", + value: 60 + (i % 30), + unit: "bpm", + measuredAt: new Date(Date.UTC(2026, 0, 1, 0, i % 60)).toISOString(), + source: "APPLE_HEALTH", + deletedAt: null, + })), + }); +} + +describe("backup blob envelope", () => { + it("round-trips the exact JSON it was handed", () => { + const json = sampleJson(200); + expect(unpackBackupBlob(packBackupBlob(json))).toBe(json); + }); + + it("stores a record-shaped payload far smaller than the plaintext", () => { + const json = sampleJson(5_000); + const packed = packBackupBlob(json); + // The old envelope was 1.33× the JSON. Anything near that means the + // compression leg is not running. + expect(packed.length).toBeLessThan(json.length / 4); + }); + + it("still reads a row written before the envelope existed", () => { + const json = sampleJson(10); + expect(unpackBackupBlob(encrypt(json))).toBe(json); + }); + + it("fails closed on a corrupt blob rather than returning junk", () => { + const packed = packBackupBlob(sampleJson(10)); + const mangled = `${packed.slice(0, -8)}AAAAAAAA`; + expect(() => unpackBackupBlob(mangled)).toThrow(); + }); +}); diff --git a/src/lib/export/backup-blob.ts b/src/lib/export/backup-blob.ts new file mode 100644 index 000000000..92d26d2b2 --- /dev/null +++ b/src/lib/export/backup-blob.ts @@ -0,0 +1,59 @@ +/** + * The envelope a `DataBackup.data` row is stored in: compress, then encrypt. + * + * Why it exists. The weekly worker handed `encrypt(JSON.stringify(payload))` + * straight to Postgres. For an account with a few hundred thousand + * measurements that is four full copies of the record alive at the same + * moment: the payload object graph, the JSON string, the base64 ciphertext at + * 1.33× the JSON, and the copy the database driver makes of that parameter on + * its way to the wire. Measured on a seeded 445 000-measurement account, the + * JSON alone is 242 MB and the run dies of heap exhaustion — under a 1 GB heap + * it never reaches the insert, and on a bigger heap it only reaches it later. + * The weekly pass was not slow; it could not finish. + * + * Compressing first is what makes the whole tail of that pipeline cheap. A + * health record's JSON is extremely repetitive — the same twenty keys per row, + * timestamps sharing a prefix — so gzip takes that 242 MB to 14 MB, and every + * copy after it shrinks with it. The same run then completes in about eight + * seconds with a peak heap of 434 MB. It is also the reason the stored row + * stops being a liability of its own: the backup lives INSIDE the database it + * would be needed to restore, so its size is not a cosmetic concern. + * + * Both directions. Rows written before this envelope existed are plain + * `encrypt(json)` and have to keep restoring — an operator whose newest usable + * copy predates the fix is exactly the person who needs it to work. The marker + * sits inside the ciphertext rather than in front of it so the stored column + * keeps the one shape `decrypt()` already understands, key ids and all. + */ +import { Buffer } from "node:buffer"; +import { gunzipSync, gzipSync } from "node:zlib"; + +import { decrypt, encrypt } from "@/lib/crypto"; + +/** + * Prefix of the DECRYPTED plaintext when the body is gzipped-then-base64'd. + * Chosen so it can never be mistaken for the alternative: a plain payload is + * always a JSON object and therefore always starts with `{`. + */ +const GZIP_MARKER = "HLZ1:"; + +/** Serialised backup JSON → the string stored in `DataBackup.data`. */ +export function packBackupBlob(json: string): string { + const compressed = gzipSync(json).toString("base64"); + return encrypt(`${GZIP_MARKER}${compressed}`); +} + +/** + * A stored `DataBackup.data` string → the backup JSON. + * + * Fails closed on every arm: a bad key, a mangled ciphertext or a truncated + * gzip member throws rather than returning a partial document, because every + * caller goes on to parse the result as a whole account. + */ +export function unpackBackupBlob(stored: string): string { + const plaintext = decrypt(stored); + if (!plaintext.startsWith(GZIP_MARKER)) return plaintext; + return gunzipSync( + Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"), + ).toString("utf8"); +} diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index b04d6e6c6..fbe8947e0 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -316,6 +316,26 @@ function serialiseMoodContext( }; } +/** + * The same payload, already serialised, with the object graph released. + * + * Its own frame on purpose, and both backup jobs go through it. The payload + * graph is the largest thing either job allocates — on an account with a few + * hundred thousand measurements it is several hundred megabytes of small + * objects — and neither job wants anything but the string. Returning from here + * makes the graph unreachable before the compress-and-encrypt step allocates + * anything of its own, which is part of why the weekly pass now fits in + * memory. Inlining these two statements back into a caller quietly undoes it. + */ +export async function buildFullBackupJson( + prisma: PrismaClient, + userId: string, + options: FullBackupOptions = {}, +): Promise { + const { payload } = await buildFullBackupPayload(prisma, userId, options); + return JSON.stringify(payload); +} + /** * Build the canonical full-backup payload for `userId`. Portable exports omit * tombstones and document ciphertext; disaster-recovery payloads preserve diff --git a/src/lib/jobs/__tests__/offhost-backup.test.ts b/src/lib/jobs/__tests__/offhost-backup.test.ts index 6918cd0ca..de9dbe17f 100644 --- a/src/lib/jobs/__tests__/offhost-backup.test.ts +++ b/src/lib/jobs/__tests__/offhost-backup.test.ts @@ -1,3 +1,5 @@ +import { createCipheriv, randomBytes } from "node:crypto"; + import { describe, it, expect, beforeEach, vi } from "vitest"; import { BACKUP_SCHEMA_VERSION, @@ -8,8 +10,20 @@ const mocks = vi.hoisted(() => ({ buildFullBackupPayload: vi.fn(), })); +// The uploader takes the already-serialised form, so the stand-in for +// `buildFullBackupJson` serialises whatever the payload mock was told to +// return. Every case below keeps stubbing the PAYLOAD, which is the thing +// these tests are actually about. vi.mock("@/lib/export/full-backup-payload", () => ({ buildFullBackupPayload: mocks.buildFullBackupPayload, + buildFullBackupJson: async (...args: unknown[]) => + JSON.stringify( + ( + (await mocks.buildFullBackupPayload(...args)) as { + payload: unknown; + } + ).payload, + ), })); import { encryptBackup, @@ -27,10 +41,33 @@ describe("offhost-backup envelope", () => { const key = Buffer.from(ENC_KEY, "hex"); const payload = JSON.stringify({ hello: "world", n: 42 }); const buf = encryptBackup(payload, key); - expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x01"); + // Version 2: the plaintext is gzipped before the cipher sees it. This + // assertion used to pin version 1, which is the version that could not + // encrypt a large account's dump without four full copies of it resident + // at once. The pin is flipped rather than dropped — the byte is the whole + // reason a reader can tell the two apart. + expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x02"); expect(decryptBackup(buf, key)).toBe(payload); }); + it("still reads a version-1 object written before compression", () => { + const key = Buffer.from(ENC_KEY, "hex"); + const payload = JSON.stringify({ hello: "world", n: 42 }); + // Hand-built v1 envelope: magic || 0x01 || iv || tag || ciphertext, the + // exact bytes every object already sitting in an operator's bucket has. + const iv = randomBytes(12); + const cipher = createCipheriv("aes-256-gcm", key, iv); + const ct = Buffer.concat([cipher.update(payload, "utf8"), cipher.final()]); + const legacy = Buffer.concat([ + Buffer.from("HLBK\x01", "binary"), + iv, + cipher.getAuthTag(), + ct, + ]); + + expect(decryptBackup(legacy, key)).toBe(payload); + }); + it("rejects tampered ciphertext", () => { const key = Buffer.from(ENC_KEY, "hex"); const buf = encryptBackup("data", key); diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 00cb3a2d7..6d070e7e7 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -18,8 +18,9 @@ * the backup history. See docs/ops/backup-restore.md. */ import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; +import { gunzipSync, gzipSync } from "node:zlib"; import type { PrismaClient } from "@/generated/prisma/client"; -import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { buildFullBackupJson } from "@/lib/export/full-backup-payload"; import { getEvent } from "@/lib/logging/context"; const ALGORITHM = "aes-256-gcm"; @@ -84,20 +85,43 @@ export function loadOffhostConfig(): OffhostBackupConfig | null { * Encrypt a JSON payload with the dedicated backup key. * * Wire format (binary): - * magic(4)="HLBK" || version(1)=0x01 || iv(12) || tag(16) || ciphertext + * magic(4)="HLBK" || version(1) || iv(12) || tag(16) || ciphertext + * + * Version 1 encrypts the JSON directly. Version 2 gzips it first, which is + * what the version byte was there for: a large account's dump is hundreds of + * megabytes of extremely repetitive JSON, and encrypting it whole means the + * string, the cipher's input copy, the ciphertext and the concatenation are + * all resident at once — the same arithmetic that made the in-database weekly + * backup unfinishable. Gzip takes an order of magnitude off every one of them + * and off the object in the bucket. Objects written under version 1 stay + * readable; both the restore drill and `scripts/restore-backup.ts` come + * through `decryptBackup`, so neither needs to know which it got. */ +const BACKUP_ENVELOPE_PLAIN = 0x01; +const BACKUP_ENVELOPE_GZIP = 0x02; + export function encryptBackup(plaintext: string, key: Buffer): Buffer { const iv = randomBytes(IV_LENGTH); const cipher = createCipheriv(ALGORITHM, key, iv); - const ct = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]); + const ct = Buffer.concat([ + cipher.update(gzipSync(plaintext)), + cipher.final(), + ]); const tag = cipher.getAuthTag(); - return Buffer.concat([Buffer.from("HLBK\x01", "binary"), iv, tag, ct]); + const header = Buffer.from([ + ...Buffer.from("HLBK", "binary"), + BACKUP_ENVELOPE_GZIP, + ]); + return Buffer.concat([header, iv, tag, ct]); } export function decryptBackup(buf: Buffer, key: Buffer): string { const magic = buf.subarray(0, 4).toString("binary"); const version = buf[4]; - if (magic !== "HLBK" || version !== 0x01) { + if ( + magic !== "HLBK" || + (version !== BACKUP_ENVELOPE_PLAIN && version !== BACKUP_ENVELOPE_GZIP) + ) { throw new Error("Invalid backup envelope (bad magic or version)"); } const iv = buf.subarray(5, 5 + IV_LENGTH); @@ -105,7 +129,10 @@ export function decryptBackup(buf: Buffer, key: Buffer): string { const ct = buf.subarray(5 + IV_LENGTH + TAG_LENGTH); const dec = createDecipheriv(ALGORITHM, key, iv); dec.setAuthTag(tag); - return Buffer.concat([dec.update(ct), dec.final()]).toString("utf8"); + const plaintext = Buffer.concat([dec.update(ct), dec.final()]); + return version === BACKUP_ENVELOPE_GZIP + ? gunzipSync(plaintext).toString("utf8") + : plaintext.toString("utf8"); } export interface S3Like { @@ -215,12 +242,13 @@ export async function runOffhostBackup( const evt = getEvent(); for (const user of users) { try { - const { payload } = await buildFullBackupPayload(prisma, user.id, { - purpose: "disaster-recovery", - exportedAt: now, - }); - const plaintext = JSON.stringify(payload); - const ciphertext = encryptBackup(plaintext, cfg.encryptionKey); + const ciphertext = encryptBackup( + await buildFullBackupJson(prisma, user.id, { + purpose: "disaster-recovery", + exportedAt: now, + }), + cfg.encryptionKey, + ); const key = `${dateKey}/user-${user.id}.json.enc`; await s3.putObject(key, ciphertext); uploaded++; diff --git a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts index 30cdc3b14..bfb009c8f 100644 --- a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts +++ b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts @@ -4,16 +4,29 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ buildFullBackupPayload: vi.fn(), - encrypt: vi.fn((value: string) => value), + packBlob: vi.fn((value: string) => value), getWorkerPrisma: vi.fn(), upsert: vi.fn(), })); +// The handler takes the already-serialised form; the stand-in serialises +// whatever the payload mock was told to return, so this file keeps stubbing +// and asserting the PAYLOAD, which is what it is about. vi.mock("@/lib/export/full-backup-payload", () => ({ buildFullBackupPayload: mocks.buildFullBackupPayload, + buildFullBackupJson: async (...args: unknown[]) => + JSON.stringify( + ((await mocks.buildFullBackupPayload(...args)) as { payload: unknown }) + .payload, + ), })); -vi.mock("@/lib/crypto", () => ({ encrypt: mocks.encrypt })); +// The envelope is exercised end-to-end in +// `src/lib/export/__tests__/backup-blob.test.ts`; here it stays transparent so +// the stored bytes can be read back as JSON. +vi.mock("@/lib/export/backup-blob", () => ({ + packBackupBlob: mocks.packBlob, +})); vi.mock("@/lib/logging/background", () => ({ withBackgroundEvent: vi.fn( diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index 48d0e3e47..e45da75a8 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -6,8 +6,8 @@ */ import { type Job } from "pg-boss"; import { recordError } from "@/lib/jobs/worker-status"; -import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; -import { encrypt } from "@/lib/crypto"; +import { buildFullBackupJson } from "@/lib/export/full-backup-payload"; +import { packBackupBlob } from "@/lib/export/backup-blob"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { @@ -79,15 +79,21 @@ export async function handleDataBackup( let backed = 0; let usersFailed = 0; + let largestBlobBytes = 0; for (const user of users) { try { - const { payload } = await buildFullBackupPayload(prisma, user.id, { - purpose: "disaster-recovery", - }); - const backupJson = JSON.stringify(payload); - - // Encrypt the backup data (contains sensitive health information) - const encryptedBackup = encrypt(backupJson); + // Compressed, then encrypted (the record contains sensitive health + // information). Both legs live in `packBackupBlob`, which is also + // what the restore path reads back through. + const encryptedBackup = packBackupBlob( + await buildFullBackupJson(prisma, user.id, { + purpose: "disaster-recovery", + }), + ); + largestBlobBytes = Math.max( + largestBlobBytes, + Buffer.byteLength(encryptedBackup, "utf8"), + ); await prisma.dataBackup.upsert({ where: { @@ -113,6 +119,9 @@ export async function handleDataBackup( } } + // The stored size is the one number that says whether this pass is + // heading back towards the wall it hit before: it tracks the record. + evt.addMeta("data_backup_largest_blob_bytes", largestBlobBytes); evt.setBackground({ task_name: "job.data_backup", result: { backed, total: users.length }, diff --git a/tests/integration/admin-backups-upload.test.ts b/tests/integration/admin-backups-upload.test.ts index d6866f0ff..30aa3bb1c 100644 --- a/tests/integration/admin-backups-upload.test.ts +++ b/tests/integration/admin-backups-upload.test.ts @@ -4,7 +4,7 @@ * Verifies: * - 201 + a `DataBackup` row with type prefix `MANUAL_UPLOAD_` is * created when the JSON file matches the schema. - * - the original payload round-trips through encrypt → decrypt. + * - the original payload round-trips through the stored backup envelope. * - 422 when the JSON is malformed. * - 422 when the schema validation fails (missing required field). * - 422 when the userId on the file doesn't exist. @@ -15,7 +15,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; process.env.ENCRYPTION_KEY ??= "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; -import { decrypt } from "@/lib/crypto"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; import { cookieJar, headerJar } from "./mock-next-headers"; import { getPrismaClient, truncateAllTables } from "./setup"; @@ -166,9 +166,11 @@ describe("POST /api/admin/backups/upload", () => { expect(row?.type).toMatch(/^MANUAL_UPLOAD_\d+$/); expect(row?.userId).toBe(admin.id); - // Round-trip the ciphertext to make sure we stored the same payload - // we received. - const decrypted = JSON.parse(decrypt(row!.data)); + // Round-trip the stored blob to make sure we kept the same payload we + // received. Through the envelope rather than through `decrypt` alone: an + // uploaded file is stored exactly the way the weekly worker stores one, so + // the restore path reads both without knowing where the row came from. + const decrypted = JSON.parse(unpackBackupBlob(row!.data)); expect(decrypted.userId).toBe(admin.id); expect(decrypted.measurements).toHaveLength(1); diff --git a/tests/integration/backup-round-trip.test.ts b/tests/integration/backup-round-trip.test.ts index a6daaf312..c3a7db819 100644 --- a/tests/integration/backup-round-trip.test.ts +++ b/tests/integration/backup-round-trip.test.ts @@ -63,6 +63,7 @@ import { encryptWaveformToBytes, } from "@/lib/withings/ecg-waveform-codec"; import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { packBackupBlob } from "@/lib/export/backup-blob"; import { TWO_ENDED_MODELS, type TwoEndedModel } from "@/lib/export/backup-plan"; import { POST } from "@/app/api/admin/backups/[id]/restore/route"; @@ -259,6 +260,27 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise { source: "MANUAL", }, }); + // A DELETED reading, kept as a tombstone. It is here because the obvious way + // to make the weekly backup cheaper — stop exporting soft-deleted rows, which + // on a real record is four rows in five — is wrong, and nothing in the tree + // said so out loud. A tombstone inside the retention window is what tells a + // synced client the row is gone (`/api/sync/changes` reads exactly these) and + // what makes a re-sent Apple Health sample a no-op instead of a resurrection. + // Drop them from a disaster-recovery file and a restore quietly brings back + // every measurement the account ever deleted on the next device sync. + await prisma.measurement.create({ + data: { + userId: OWNER_ID, + type: "PULSE", + value: 61, + unit: "bpm", + measuredAt: AT("2026-07-02T06:30:00.000Z"), + source: "APPLE_HEALTH", + externalId: "round-trip-deleted-sample", + syncVersion: 4, + deletedAt: AT("2026-07-20T09:15:00.000Z"), + }, + }); // The EVENT row an ECG strip is filed against, and the strip itself. The // pairing is the point: `EcgRecording.measurementId` is a real foreign key, // so this is the reference whose remap has to survive the round trip rather @@ -1231,6 +1253,26 @@ describe("every model the plan claims two-ended survives a real restore", () => purpose: "disaster-recovery", }); + // The tombstone has to be IN the file. A disaster-recovery payload that + // carries only live rows is smaller and cheaper and wrong: the restore + // wipes the account first, so a deletion that is not in the file is a + // deletion the account no longer knows it made, and the next device sync + // re-inserts every one of those samples as fresh readings. + const exportedMeasurements = payload.measurements as Array<{ + externalId: string | null; + deletedAt: string | null; + syncVersion: number; + }>; + expect( + exportedMeasurements.find( + (row) => row.externalId === "round-trip-deleted-sample", + ), + "a soft-deleted reading is missing from the disaster-recovery payload", + ).toMatchObject({ + deletedAt: "2026-07-20T09:15:00.000Z", + syncVersion: 4, + }); + // Cascade rather than a hand-listed sweep — see the file comment. await prisma.user.delete({ where: { id: OWNER_ID } }); await createOwner(prisma); @@ -1247,7 +1289,12 @@ describe("every model the plan claims two-ended survives a real restore", () => data: { userId: OWNER_ID, type: "TWO_ENDED_ROUND_TRIP", - data: encrypt(JSON.stringify(payload)), + // The envelope the weekly worker actually writes: gzip, then + // encrypt. The three cases further down deliberately keep the plain + // `encrypt(json)` form, which is what every row written before this + // envelope existed looks like — an operator's newest usable copy may + // well be one of those, so both arms have to reach the restore route. + data: packBackupBlob(JSON.stringify(payload)), }, }); const response = await POST( @@ -2026,6 +2073,18 @@ describe("every model the plan claims two-ended survives a real restore", () => // measurement pointer is asserted against the row the RESTORE wrote rather // than against the id the fixture used, because resolving to something // that exists is the property the reference owes. + const restoredTombstone = await prisma.measurement.findFirstOrThrow({ + where: { userId: OWNER_ID, externalId: "round-trip-deleted-sample" }, + }); + expect( + { + deletedAt: restoredTombstone.deletedAt?.toISOString() ?? null, + syncVersion: restoredTombstone.syncVersion, + }, + "the deleted reading came back, but not as a deletion — a tombstone " + + "restored without its `deletedAt` is a resurrected measurement", + ).toEqual({ deletedAt: "2026-07-20T09:15:00.000Z", syncVersion: 4 }); + const restoredMeasurement = await prisma.measurement.findFirstOrThrow({ where: { userId: OWNER_ID, type: "WEIGHT" }, }); From 154c93b5e1f3118cab3c20a1751f869a5cf2901c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 3 Sep 2026 02:22:37 +0200 Subject: [PATCH 3/3] feat: surface a stalled or failed backup schedule The weekly pass can stop and leave no trace where anyone looks. The handler's own error never runs when the job is killed rather than thrown; `recordError()` increments an unnamed counter; and the admin status page's failing-queue panel looks back 72 hours, which for a weekly queue means a permanent failure is on screen three days in seven. The backups page itself showed an ordinary row with an ordinary timestamp, because a copy made six weeks ago reads exactly like Sunday's until somebody subtracts the dates. The listing now carries two facts that fail at different times: the age of the newest scheduled copy, which never leaves any retention window, and the terminal state of the last scheduled run, which is the only place the reason is written down. `readLastQueueRun` takes the newest run rather than a window, so a weekly queue's outcome stays readable until the next one replaces it, and it unwraps the value pg-boss stores a thrown error under, so the page says "job timed out" instead of raw JSON. The failing-queue reader gets the same unwrap. Manual uploads are excluded from the cadence judgement by the shape of the input, so an ad-hoc upload cannot make a dead cron look alive, and a fresh instance is not called overdue for having no copy yet. The listing also stops reading every blob into the web process to measure it; `octet_length` answers that in the database, which on an instance with large records was its own way of running out of memory. --- CHANGELOG.md | 29 ++++++ messages/de.json | 6 ++ messages/en.json | 6 ++ messages/es.json | 6 ++ messages/fr.json | 6 ++ messages/it.json | 6 ++ messages/ko.json | 6 ++ messages/pl.json | 6 ++ src/app/api/admin/backups/route.ts | 63 ++++++++++--- .../__tests__/backups-section-empty.test.tsx | 85 ++++++++++++++++- src/components/admin/backups-section.tsx | 70 ++++++++++++++ .../__tests__/backup-schedule-status.test.ts | 92 +++++++++++++++++++ src/lib/jobs/backup-schedule-status.ts | 78 ++++++++++++++++ src/lib/jobs/job-failures.ts | 75 ++++++++++++++- src/types/backups.ts | 7 ++ tests/integration/admin-backups-audit.test.ts | 66 +++++++++++++ 16 files changed, 591 insertions(+), 16 deletions(-) create mode 100644 src/lib/jobs/__tests__/backup-schedule-status.test.ts create mode 100644 src/lib/jobs/backup-schedule-status.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 48b6ec02c..91e6c38b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## [Unreleased] + +### Fixed + +- **A large record no longer outruns the weekly backup.** On an account with a + few hundred thousand readings the Sunday snapshot could not finish. It was + never given a time limit of its own, so it inherited the queue's fifteen + minutes, was killed part-way through, redelivered twice against the same + record, and gave up three quarters of an hour later having never completed + once. The reason it needed so long was memory: the whole account was built in + memory, serialised whole, encrypted whole, and handed to the database whole, + which is four full copies of the record at the same moment. The stored copy + is now compressed before it is encrypted, which takes an order of magnitude + off everything after it, and the pass releases each stage before allocating + the next. A seeded 445 000-reading account went from dying of memory to + finishing in about twenty seconds, with a stored copy of 25 MB instead of + 322 MB. The pass also has an explicit two-hour window now, so a genuinely + slow disk cannot be mistaken for a broken run. The nightly off-host copy is + built the same way and got the same treatment; files written before this + still restore, and so do backups already sitting in the database. +- **A failed backup run is now visible on the backups page.** The weekly pass + can stop and leave no trace: the page listed whatever copies existed, and a + copy from six weeks ago carries a timestamp exactly like Sunday's. The page + now says how old the newest scheduled copy is when it has gone past due, and + how the last scheduled run ended when it ended badly, with the reason the + queue recorded. Both are shown because they stop being available at different + times — a run's own record is kept for a week, the age of the newest copy for + as long as the copy exists. + ## [1.38.4] — 2026-09-02 The shared AI key reaches the provider its address names, and the image diff --git a/messages/de.json b/messages/de.json index ae260abd6..66f499faa 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Letzter Snapshot", "colActions": "Aktionen", "retentionLabel": "Aufbewahrung: {days} Tage", + "scheduleStaleTitle": "Die wöchentliche Sicherung läuft nicht mehr", + "scheduleStaleDescription": "Die neueste geplante Kopie ist {days} Tage alt. Ein funktionierender wöchentlicher Lauf lässt keine älter als {threshold} Tage werden.", + "scheduleFailedTitle": "Der letzte geplante Sicherungslauf ist fehlgeschlagen", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "kein Grund aufgezeichnet", + "scheduleRemedy": "Starte jetzt eine Sicherung und prüfe, ob eine frische Zeile erscheint. Schlägt sie erneut fehl, nennt das Worker-Protokoll das Konto, an dem sie stehen geblieben ist.", "download": "Herunterladen", "downloadAria": "Backup für {username} herunterladen", "downloadFailed": "Download fehlgeschlagen", diff --git a/messages/en.json b/messages/en.json index bfbb0ab32..f368799ab 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Last snapshot", "colActions": "Actions", "retentionLabel": "Retention: {days} days", + "scheduleStaleTitle": "The weekly backup has stopped", + "scheduleStaleDescription": "The newest scheduled copy is {days} days old. A working weekly run never leaves one older than {threshold} days.", + "scheduleFailedTitle": "The last scheduled backup run failed", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "no reason recorded", + "scheduleRemedy": "Run a backup now and check that a fresh row appears. If it fails again, the worker log names the account it stopped on.", "download": "Download", "downloadAria": "Download backup for {username}", "downloadFailed": "Download failed", diff --git a/messages/es.json b/messages/es.json index 2ecddf478..882d57a3f 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Última instantánea", "colActions": "Acciones", "retentionLabel": "Conservación: {days} días", + "scheduleStaleTitle": "La copia de seguridad semanal se ha detenido", + "scheduleStaleDescription": "La copia programada más reciente tiene {days} días. Una ejecución semanal que funciona nunca deja ninguna con más de {threshold} días.", + "scheduleFailedTitle": "La última ejecución programada de la copia de seguridad falló", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "sin motivo registrado", + "scheduleRemedy": "Ejecuta una copia ahora y comprueba que aparece una fila nueva. Si vuelve a fallar, el registro del worker indica la cuenta en la que se detuvo.", "download": "Descargar", "downloadAria": "Descargar copia de seguridad de {username}", "downloadFailed": "La descarga falló", diff --git a/messages/fr.json b/messages/fr.json index 63b62b5ff..cc826c2ee 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Dernier instantané", "colActions": "Actions", "retentionLabel": "Conservation : {days} jours", + "scheduleStaleTitle": "La sauvegarde hebdomadaire s'est arrêtée", + "scheduleStaleDescription": "La copie planifiée la plus récente date de {days} jours. Une exécution hebdomadaire qui fonctionne n'en laisse jamais une dépasser {threshold} jours.", + "scheduleFailedTitle": "La dernière exécution planifiée de la sauvegarde a échoué", + "scheduleFailedDescription": "{when} : {reason}", + "scheduleFailedNoReason": "aucune raison enregistrée", + "scheduleRemedy": "Lancez une sauvegarde maintenant et vérifiez qu'une nouvelle ligne apparaît. Si elle échoue à nouveau, le journal du worker indique le compte sur lequel elle s'est arrêtée.", "download": "Télécharger", "downloadAria": "Télécharger la sauvegarde de {username}", "downloadFailed": "Échec du téléchargement", diff --git a/messages/it.json b/messages/it.json index 3cc1dd08e..0e5e5c624 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Ultima istantanea", "colActions": "Azioni", "retentionLabel": "Conservazione: {days} giorni", + "scheduleStaleTitle": "Il backup settimanale si è fermato", + "scheduleStaleDescription": "La copia pianificata più recente ha {days} giorni. Un'esecuzione settimanale funzionante non ne lascia mai una più vecchia di {threshold} giorni.", + "scheduleFailedTitle": "L'ultima esecuzione pianificata del backup non è riuscita", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "nessun motivo registrato", + "scheduleRemedy": "Avvia subito un backup e verifica che compaia una riga nuova. Se fallisce di nuovo, il log del worker indica l'account su cui si è fermato.", "download": "Scarica", "downloadAria": "Scarica il backup di {username}", "downloadFailed": "Download non riuscito", diff --git a/messages/ko.json b/messages/ko.json index 062b3e109..85a8c19af 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6901,6 +6901,12 @@ "colCreatedAt": "최근 스냅샷", "colActions": "동작", "retentionLabel": "보관 기간: {days}일", + "scheduleStaleTitle": "주간 백업이 중단되었습니다", + "scheduleStaleDescription": "가장 최근의 예약 사본이 {days}일 전 것입니다. 정상적으로 도는 주간 실행은 {threshold}일보다 오래된 사본을 남기지 않습니다.", + "scheduleFailedTitle": "마지막 예약 백업 실행이 실패했습니다", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "기록된 사유 없음", + "scheduleRemedy": "지금 백업을 실행하고 새 항목이 나타나는지 확인하세요. 다시 실패하면 워커 로그에 중단된 계정이 표시됩니다.", "download": "다운로드", "downloadAria": "{username}의 백업 다운로드", "downloadFailed": "다운로드하지 못했어요", diff --git a/messages/pl.json b/messages/pl.json index de783600b..bb784d314 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Ostatnia migawka", "colActions": "Akcje", "retentionLabel": "Przechowywanie: {days} dni", + "scheduleStaleTitle": "Cotygodniowa kopia zapasowa przestała powstawać", + "scheduleStaleDescription": "Najnowsza zaplanowana kopia ma {days} dni. Działające cotygodniowe uruchomienie nigdy nie zostawia kopii starszej niż {threshold} dni.", + "scheduleFailedTitle": "Ostatnie zaplanowane uruchomienie kopii zapasowej nie powiodło się", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "nie zapisano przyczyny", + "scheduleRemedy": "Uruchom kopię teraz i sprawdź, czy pojawi się nowy wiersz. Jeśli znowu się nie powiedzie, log workera wskaże konto, na którym się zatrzymał.", "download": "Pobierz", "downloadAria": "Pobierz kopię zapasową dla {username}", "downloadFailed": "Pobieranie nie powiodło się", diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 11871f979..985e82e22 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -5,11 +5,20 @@ * shipped — only metadata (id, userId, username, type, size in bytes, * createdAt). The encrypted blob remains server-side; admins can trigger * a re-snapshot but not download another user's payload from the UI. + * + * Alongside the rows, the response says whether the SCHEDULE is still alive: + * how old the newest scheduled copy is and how the last scheduled run ended. + * Without those two the page cannot tell a working backup from one that + * stopped six weeks ago — every row it lists has a perfectly ordinary + * timestamp either way. */ import { prisma } from "@/lib/db"; import { apiHandler, requireAdmin } from "@/lib/api-handler"; import { apiSuccess } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; +import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; +import { readLastQueueRun } from "@/lib/jobs/job-failures"; +import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status"; // v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts` // so callers (in particular `components/admin/backups-section.tsx`) don't // have to reach across the component → route-handler layer boundary. @@ -17,31 +26,57 @@ import type { BackupRow, BackupsList } from "@/types/backups"; export const dynamic = "force-dynamic"; +/** The type the weekly worker writes; anything else was uploaded by hand. */ +const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO"; + +interface BackupMetaRow { + id: string; + user_id: string; + username: string; + type: string; + size_bytes: number; + created_at: Date; +} + export const GET = apiHandler(async () => { await requireAdmin(); annotate({ action: { name: "admin.backups.list" } }); - // Pull metadata only — `data` would balloon the response and reveal - // ciphertext we don't need on the listing page. - const backups = await prisma.dataBackup.findMany({ - include: { user: { select: { id: true, username: true } } }, - orderBy: { createdAt: "desc" }, - }); + // Metadata only, and the size read as `octet_length` in the database rather + // than by pulling every blob into this process to measure it. The listing + // needs a number, not the ciphertext, and on an instance with large records + // fetching them all was its own way of running out of memory. Base64 is + // ASCII, so the column's byte length is exactly what the wire reports. + const rows = await prisma.$queryRaw` + SELECT b.id, b.user_id, u.username, b.type, + octet_length(b.data) AS size_bytes, + b.created_at + FROM data_backups b + JOIN users u ON u.id = b.user_id + ORDER BY b.created_at DESC + `; - const rows: BackupRow[] = backups.map((b) => ({ - id: b.id, - userId: b.userId, - username: b.user.username, - type: b.type, - sizeBytes: Buffer.byteLength(b.data, "utf8"), - createdAt: b.createdAt.toISOString(), + const list: BackupRow[] = rows.map((row) => ({ + id: row.id, + userId: row.user_id, + username: row.username, + type: row.type, + sizeBytes: Number(row.size_bytes), + createdAt: row.created_at.toISOString(), })); const payload: BackupsList = { - rows, + rows: list, // Matches the retention window the backup-prune job enforces so the // backups page states the same number the worker acts on. retentionDays: 90, + schedule: summariseBackupSchedule({ + scheduledCreatedAt: rows + .filter((row) => row.type === SCHEDULED_BACKUP_TYPE) + .map((row) => row.created_at), + lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE), + now: new Date(), + }), }; return apiSuccess(payload); diff --git a/src/components/admin/__tests__/backups-section-empty.test.tsx b/src/components/admin/__tests__/backups-section-empty.test.tsx index 6dba6cd8f..6a8b2afd5 100644 --- a/src/components/admin/__tests__/backups-section-empty.test.tsx +++ b/src/components/admin/__tests__/backups-section-empty.test.tsx @@ -3,11 +3,27 @@ import { renderToStaticMarkup } from "react-dom/server"; import de from "../../../../messages/de.json"; import en from "../../../../messages/en.json"; +const healthySchedule = { + lastSuccessAt: "2026-09-01T03:00:00.000Z", + lastSuccessAgeDays: 2, + staleAfterDays: 10, + stale: false, + lastRun: { + at: "2026-09-01T03:00:12.000Z", + state: "completed" as const, + error: null, + }, + lastRunFailed: false, +}; + const mocks = vi.hoisted(() => ({ mutationOptions: [] as Array<{ onSuccess?: (data: { summary: Record }) => void; }>, toastSuccess: vi.fn(), + // Mutable so one file can render both a healthy schedule and a broken one; + // the query mock reads it on every render. + queryData: { rows: [], retentionDays: 30 } as Record, })); /** @@ -34,7 +50,7 @@ vi.mock("sonner", () => ({ vi.mock("@tanstack/react-query", () => ({ useQuery: () => ({ - data: { rows: [], retentionDays: 30 }, + data: mocks.queryData, isLoading: false, isError: false, }), @@ -69,6 +85,11 @@ import { I18nProvider } from "@/lib/i18n/context"; import { BackupsSection } from "../backups-section"; function render(locale: "en" | "de" = "en") { + mocks.queryData = { + rows: [], + retentionDays: 30, + schedule: healthySchedule, + }; return renderToStaticMarkup( @@ -129,6 +150,68 @@ describe("BackupsSection — empty state", () => { }); }); +/** + * The gap that let a weekly backup stop for a month and a half without anyone + * noticing: the page rendered whatever rows it had, and a copy from six weeks + * ago is indistinguishable from Sunday's until somebody subtracts the dates. + */ +describe("BackupsSection — schedule health", () => { + function withSchedule(schedule: Record) { + mocks.queryData = { rows: [], retentionDays: 30, schedule }; + return renderToStaticMarkup( + + + , + ); + } + + it("stays quiet while the schedule is healthy", () => { + const html = render(); + expect(html).not.toContain('data-slot="backup-schedule-notice"'); + }); + + it("says so when the newest scheduled copy has gone stale", () => { + const html = withSchedule({ + ...healthySchedule, + lastSuccessAt: "2026-07-19T03:00:00.000Z", + lastSuccessAgeDays: 46, + stale: true, + lastRun: null, + lastRunFailed: false, + }); + + expect(html).toContain('data-slot="backup-schedule-notice"'); + expect(html).toContain("The weekly backup has stopped"); + expect(html).toContain("46"); + }); + + it("names the reason when the last scheduled run failed", () => { + const html = withSchedule({ + ...healthySchedule, + lastRun: { + at: "2026-08-30T03:46:00.000Z", + state: "failed", + error: "job timed out", + }, + lastRunFailed: true, + }); + + expect(html).toContain("The last scheduled backup run failed"); + expect(html).toContain("job timed out"); + }); + + it("survives a response that predates the schedule field", () => { + mocks.queryData = { rows: [], retentionDays: 30 }; + expect(() => + renderToStaticMarkup( + + + , + ), + ).not.toThrow(); + }); +}); + describe("BackupsSection — upload outcome", () => { it("includes health profile fact revisions in the restored record total", () => { mocks.toastSuccess.mockClear(); diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index 5e2243a91..6f2c0f1d6 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -48,6 +48,7 @@ import { ListRow } from "@/components/ui/list-row"; import { useFormatters, useTranslations } from "@/lib/i18n/context"; import { queryKeys } from "@/lib/query-keys"; import type { BackupRow, BackupsList } from "@/types/backups"; +import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status"; import type { MissingBackupSection, RestoreSkipSummary, @@ -560,6 +561,71 @@ export function RestoreSkipReport({ ); } +/** + * Says out loud whether the SCHEDULE is still alive. + * + * The weekly pass on the maintainer's own instance stopped producing copies + * and nothing surfaced it for a month and a half: the table listed the rows it + * had, each with an ordinary timestamp, and a copy from six weeks ago reads + * exactly like one made on Sunday until somebody does the subtraction. Two + * independent facts are shown, because they fail at different times — the age + * of the newest scheduled copy never ages out, while the run's own outcome + * carries the reason ("job timed out") and survives only as long as pg-boss + * keeps the row. + * + * Not dismissible: it reflects live state, so the only way to clear it is to + * make a backup succeed. + */ +function ScheduleHealthNotice({ + schedule, +}: { + schedule: BackupScheduleStatus; +}) { + const { t } = useTranslations(); + const fmt = useFormatters(); + if (!schedule.stale && !schedule.lastRunFailed) return null; + + return ( +
+
+
+
+ ); +} + export function BackupsSection() { const { t } = useTranslations(); const fmt = useFormatters(); @@ -833,6 +899,10 @@ export function BackupsSection() { } /> + {data?.schedule ? ( + + ) : null} + {/* Upload card — separate from the table so admins can ingest a backup file independently of any existing rows. The visible button proxies a hidden file input so the layout stays clean diff --git a/src/lib/jobs/__tests__/backup-schedule-status.test.ts b/src/lib/jobs/__tests__/backup-schedule-status.test.ts new file mode 100644 index 000000000..2a45bee31 --- /dev/null +++ b/src/lib/jobs/__tests__/backup-schedule-status.test.ts @@ -0,0 +1,92 @@ +/** + * What the backups page has to be able to say out loud. + * + * The weekly snapshot stopped producing a new copy and nothing surfaced it: the + * page listed the rows it had, each with a perfectly ordinary timestamp, and a + * copy that was a month and a half old looked exactly like one made on Sunday. + * The two facts that would have shown it are the age of the newest scheduled + * copy and the terminal state of the last scheduled run. + */ +import { describe, expect, it } from "vitest"; + +import { + BACKUP_STALE_AFTER_DAYS, + summariseBackupSchedule, +} from "../backup-schedule-status"; + +const NOW = new Date("2026-09-03T09:00:00.000Z"); +const daysAgo = (days: number) => + new Date(NOW.getTime() - days * 24 * 60 * 60 * 1000); + +describe("summariseBackupSchedule", () => { + it("reports a fresh weekly copy as neither stale nor failed", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(2), daysAgo(9)], + lastRun: { + at: daysAgo(2).toISOString(), + state: "completed", + error: null, + }, + now: NOW, + }); + + expect(status.lastSuccessAt).toBe(daysAgo(2).toISOString()); + expect(status.lastSuccessAgeDays).toBe(2); + expect(status.stale).toBe(false); + expect(status.lastRunFailed).toBe(false); + expect(status.staleAfterDays).toBe(BACKUP_STALE_AFTER_DAYS); + }); + + it("calls a copy older than the window stale", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(46)], + lastRun: null, + now: NOW, + }); + + expect(status.lastSuccessAgeDays).toBe(46); + expect(status.stale).toBe(true); + // No run row survives that long — the age is the durable signal. + expect(status.lastRunFailed).toBe(false); + }); + + it("surfaces a failed last run even while the copy is still in window", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(3)], + lastRun: { + at: daysAgo(1).toISOString(), + state: "failed", + error: "job timed out", + }, + now: NOW, + }); + + expect(status.stale).toBe(false); + expect(status.lastRunFailed).toBe(true); + expect(status.lastRun?.error).toBe("job timed out"); + }); + + it("does not call a brand-new instance stale", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [], + lastRun: null, + now: NOW, + }); + + expect(status.lastSuccessAt).toBeNull(); + expect(status.lastSuccessAgeDays).toBeNull(); + expect(status.stale).toBe(false); + }); + + it("ignores manual uploads when judging the scheduled cadence", () => { + // The caller passes only WEEKLY_AUTO rows; a manual upload made today must + // not make a dead cron look alive. Proven here by the shape of the input: + // the summary knows nothing but scheduled copies. + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(30)], + lastRun: null, + now: NOW, + }); + expect(status.stale).toBe(true); + }); +}); diff --git a/src/lib/jobs/backup-schedule-status.ts b/src/lib/jobs/backup-schedule-status.ts new file mode 100644 index 000000000..d7a5ac20d --- /dev/null +++ b/src/lib/jobs/backup-schedule-status.ts @@ -0,0 +1,78 @@ +/** + * Whether the weekly backup is actually still happening, in two facts. + * + * The pass stopped producing a new copy and nothing said so for a month and a + * half. Everything that could have told the operator was either invisible or + * short-lived: the handler's own error never ran (the job was killed, not + * thrown), `recordError()` increments an unnamed counter, and the admin status + * page's failing-queue panel only looks back 72 hours — which for a weekly + * queue means a permanent failure is on screen three days in seven. The backups + * page itself showed a perfectly ordinary row with a perfectly ordinary + * timestamp, because a 46-day-old copy looks exactly like a fresh one until you + * read the date. + * + * So the surface carries both legs, and neither is derived from the other: + * + * - the AGE of the newest scheduled copy, which never ages out of any + * retention window and would have caught this on the second missed Sunday; + * - the terminal state of the last scheduled RUN, which is the only place the + * reason ("job timed out") is written down at all. + * + * Manual uploads are excluded by the shape of the input: the caller passes only + * the scheduled rows, so an admin's ad-hoc upload cannot make a dead cron look + * alive. + */ +import type { LastQueueRun } from "@/lib/jobs/job-failures"; + +/** + * How old the newest scheduled copy may get before the page says so. The cron + * is weekly, so seven days is normal and eight is a rounding error; ten means + * at least one Sunday produced nothing. + */ +export const BACKUP_STALE_AFTER_DAYS = 10; + +const DAY_MS = 24 * 60 * 60 * 1000; + +export interface BackupScheduleStatus { + /** ISO instant of the newest scheduled copy, or null when there is none. */ + lastSuccessAt: string | null; + /** Whole days since that copy was written. */ + lastSuccessAgeDays: number | null; + /** The threshold behind `stale`, so the UI can name it. */ + staleAfterDays: number; + /** A scheduled copy exists and is older than the threshold. */ + stale: boolean; + /** The last scheduled run pg-boss still remembers. */ + lastRun: LastQueueRun | null; + /** That run ended `failed` or `cancelled`. */ + lastRunFailed: boolean; +} + +export function summariseBackupSchedule(args: { + /** `createdAt` of every SCHEDULED backup row; manual uploads excluded. */ + scheduledCreatedAt: readonly Date[]; + lastRun: LastQueueRun | null; + now: Date; +}): BackupScheduleStatus { + const newest = args.scheduledCreatedAt.reduce( + (best, at) => (best === null || at > best ? at : best), + null, + ); + const ageDays = + newest === null + ? null + : Math.floor((args.now.getTime() - newest.getTime()) / DAY_MS); + + return { + lastSuccessAt: newest?.toISOString() ?? null, + lastSuccessAgeDays: ageDays, + staleAfterDays: BACKUP_STALE_AFTER_DAYS, + // An instance that has never produced a scheduled copy is not stale, it is + // new — the page's empty state already says there is nothing there, and + // calling a fresh install "overdue" would train the operator to ignore the + // warning that matters. + stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS, + lastRun: args.lastRun, + lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed", + }; +} diff --git a/src/lib/jobs/job-failures.ts b/src/lib/jobs/job-failures.ts index 48403da09..d846c88be 100644 --- a/src/lib/jobs/job-failures.ts +++ b/src/lib/jobs/job-failures.ts @@ -98,7 +98,11 @@ export async function readFailingQueues( max(completed_on) AS last_failed_at, ( array_agg( - COALESCE(output->>'message', output::text) + COALESCE( + output->>'message', + output->'value'->>'message', + output::text + ) ORDER BY completed_on DESC NULLS LAST ) )[1] AS last_error @@ -124,6 +128,75 @@ export async function readFailingQueues( } } +/** The terminal state pg-boss recorded for the newest run of one queue. */ +export interface LastQueueRun { + /** ISO instant the run reached its terminal state. */ + at: string; + /** `failed` covers both a thrown handler and an expired (killed) job. */ + state: "completed" | "failed" | "cancelled"; + /** The failure text, redacted and truncated. Null when the run succeeded. */ + error: string | null; +} + +/** + * The newest terminal run of one queue, whatever its outcome. + * + * `readFailingQueues` answers "what broke recently" and is deliberately scoped + * to a 72-hour window, which is the right shape for a status page watching + * thirty queues that tick by the minute. It is the wrong shape for a WEEKLY + * pass: a Sunday-night failure has fallen out of the window by Wednesday, so a + * job that fails every single week is visible for three days in seven and + * invisible for four. This reader takes the newest run instead of a window, so + * a weekly queue's last outcome is readable right up until the next one. + * + * Bounded by pg-boss's own retention: terminal rows live `deletion_seconds` + * (seven days by default) past completion, so `null` here means "no run row + * survives", which is not the same as "the pass never ran". Callers pair it + * with a fact that does not age out — for backups, the age of the newest + * stored copy. + */ +export async function readLastQueueRun( + queue: string, +): Promise { + try { + const rows = await prisma.$queryRaw< + Array<{ state: string; completed_on: Date | null; output: string | null }> + >` + SELECT + state, + completed_on, + -- pg-boss wraps a thrown value: an expired job's row reads + -- {"value":{"message":"job timed out"}}, so a plain ->>'message' + -- misses it and the operator gets raw JSON instead of the sentence. + COALESCE( + output->>'message', + output->'value'->>'message', + output::text + ) AS output + FROM pgboss.job + WHERE name = ${queue} + AND state IN ('completed', 'failed', 'cancelled') + AND completed_on IS NOT NULL + ORDER BY completed_on DESC + LIMIT 1 + `; + const row = rows[0]; + if (!row || row.completed_on === null) return null; + const state = + row.state === "failed" || row.state === "cancelled" + ? row.state + : "completed"; + return { + at: row.completed_on.toISOString(), + state, + error: state === "completed" ? null : presentError(row.output), + }; + } catch { + // No `pgboss` schema (web-only deployment) or no permission to read it. + return null; + } +} + export interface QueueFailureForUser { /** ISO instant the user's newest job on this queue gave up. */ lastFailedAt: string; diff --git a/src/types/backups.ts b/src/types/backups.ts index 75b0cc66c..1931628ea 100644 --- a/src/types/backups.ts +++ b/src/types/backups.ts @@ -9,6 +9,7 @@ * both sides a route-handler-independent shared type home; the route * keeps owning the HTTP contract, the component keeps owning the UI. */ +import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status"; export interface BackupRow { id: string; @@ -25,6 +26,12 @@ export interface BackupRow { export interface BackupsList { rows: BackupRow[]; + /** + * Whether the weekly schedule is still producing copies, and how the last + * scheduled run ended. A row's own timestamp cannot answer that — a copy + * made six weeks ago and one made on Sunday look identical in a table. + */ + schedule: BackupScheduleStatus; /** * Soft retention hint — the worker is configured for weekly backups * (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and diff --git a/tests/integration/admin-backups-audit.test.ts b/tests/integration/admin-backups-audit.test.ts index 989399700..fe7755fb4 100644 --- a/tests/integration/admin-backups-audit.test.ts +++ b/tests/integration/admin-backups-audit.test.ts @@ -197,3 +197,69 @@ describe("backup endpoints — AuditLog contract", () => { expect(restore?.details ?? "").toContain(seeded.id); }); }); + +/** + * The listing route is what an operator looks at to answer "are my backups + * still being made". It could not answer that: it returned rows and a + * retention hint, and a copy from six weeks ago carries a timestamp exactly + * like Sunday's. It now carries the schedule's own health, and the size comes + * from `octet_length` in the database rather than from pulling every blob into + * the web process to measure it. + */ +describe("GET /api/admin/backups — schedule health", () => { + it("calls a stalled weekly schedule out, and a fresh manual upload does not hide it", async () => { + const prisma = getPrismaClient(); + const admin = await seedAdminSession(); + const blob = encrypt(JSON.stringify(payloadFor(admin.id))); + + const stalled = new Date(Date.now() - 46 * 24 * 60 * 60 * 1000); + await prisma.dataBackup.create({ + data: { + userId: admin.id, + type: "WEEKLY_AUTO", + data: blob, + createdAt: stalled, + }, + }); + // Uploaded by hand today. It must not make the dead cron look alive. + await prisma.dataBackup.create({ + data: { userId: admin.id, type: "MANUAL_UPLOAD_1", data: blob }, + }); + + // The listing handler declares no parameters; `apiHandler` reads the + // request defensively out of its arguments, so calling it bare is what a + // zero-argument handler supports. + const { GET } = await import("@/app/api/admin/backups/route"); + const res = await GET(); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { + rows: Array<{ type: string; sizeBytes: number; username: string }>; + schedule: { + stale: boolean; + lastSuccessAgeDays: number | null; + lastRun: unknown; + lastRunFailed: boolean; + }; + }; + }; + + expect(body.data.rows).toHaveLength(2); + // Sizes are read in SQL now; a wrong column or a wrong cast shows up here + // as a zero or a NaN rather than as a plausible-looking number. + expect(body.data.rows.every((row) => row.sizeBytes === blob.length)).toBe( + true, + ); + expect(body.data.rows.every((row) => row.username === admin.username)).toBe( + true, + ); + + expect(body.data.schedule.lastSuccessAgeDays).toBe(46); + expect(body.data.schedule.stale).toBe(true); + // No `pgboss` schema in the test database — the run reader fails soft to + // null rather than taking the whole page down, which is the arm a web-only + // deployment lives on permanently. + expect(body.data.schedule.lastRun).toBeNull(); + expect(body.data.schedule.lastRunFailed).toBe(false); + }); +});