diff --git a/CHANGELOG.md b/CHANGELOG.md index af6f56fff..9c8406aa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,30 @@ ### Fixed +- **A large record no longer outruns the weekly backup.** On an account with a + few hundred thousand readings the Sunday snapshot could not finish. It was + never given a time limit of its own, so it inherited the queue's fifteen + minutes, was killed part-way through, redelivered twice against the same + record, and gave up three quarters of an hour later having never completed + once. The reason it needed so long was memory: the whole account was built in + memory, serialised whole, encrypted whole, and handed to the database whole, + which is four full copies of the record at the same moment. The stored copy + is now compressed before it is encrypted, which takes an order of magnitude + off everything after it, and the pass releases each stage before allocating + the next. A seeded 445 000-reading account went from dying of memory to + finishing in about twenty seconds, with a stored copy of 25 MB instead of + 322 MB. The pass also has an explicit two-hour window now, so a genuinely + slow disk cannot be mistaken for a broken run. The nightly off-host copy is + built the same way and got the same treatment; files written before this + still restore, and so do backups already sitting in the database. +- **A failed backup run is now visible on the backups page.** The weekly pass + can stop and leave no trace: the page listed whatever copies existed, and a + copy from six weeks ago carries a timestamp exactly like Sunday's. The page + now says how old the newest scheduled copy is when it has gone past due, and + how the last scheduled run ended when it ended badly, with the reason the + queue recorded. Both are shown because they stop being available at different + times — a run's own record is kept for a week, the age of the newest copy for + as long as the copy exists. - **Mood correlations no longer call five days of noise a strong finding.** The mood page painted a "Strong" or "Moderate" badge on a mood-versus-sleep, activity, pulse, weight or blood-pressure card as soon as five days had been diff --git a/messages/de.json b/messages/de.json index bb25c8cf7..229068fe7 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6923,6 +6923,12 @@ "colCreatedAt": "Letzter Snapshot", "colActions": "Aktionen", "retentionLabel": "Aufbewahrung: {days} Tage", + "scheduleStaleTitle": "Die wöchentliche Sicherung läuft nicht mehr", + "scheduleStaleDescription": "Die neueste geplante Kopie ist {days} Tage alt. Ein funktionierender wöchentlicher Lauf lässt keine älter als {threshold} Tage werden.", + "scheduleFailedTitle": "Der letzte geplante Sicherungslauf ist fehlgeschlagen", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "kein Grund aufgezeichnet", + "scheduleRemedy": "Starte jetzt eine Sicherung und prüfe, ob eine frische Zeile erscheint. Schlägt sie erneut fehl, nennt das Worker-Protokoll das Konto, an dem sie stehen geblieben ist.", "download": "Herunterladen", "downloadAria": "Backup für {username} herunterladen", "downloadFailed": "Download fehlgeschlagen", diff --git a/messages/en.json b/messages/en.json index f8f3c5599..a0de65aed 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6923,6 +6923,12 @@ "colCreatedAt": "Last snapshot", "colActions": "Actions", "retentionLabel": "Retention: {days} days", + "scheduleStaleTitle": "The weekly backup has stopped", + "scheduleStaleDescription": "The newest scheduled copy is {days} days old. A working weekly run never leaves one older than {threshold} days.", + "scheduleFailedTitle": "The last scheduled backup run failed", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "no reason recorded", + "scheduleRemedy": "Run a backup now and check that a fresh row appears. If it fails again, the worker log names the account it stopped on.", "download": "Download", "downloadAria": "Download backup for {username}", "downloadFailed": "Download failed", diff --git a/messages/es.json b/messages/es.json index b3f1b33e6..e40df4240 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6923,6 +6923,12 @@ "colCreatedAt": "Última instantánea", "colActions": "Acciones", "retentionLabel": "Conservación: {days} días", + "scheduleStaleTitle": "La copia de seguridad semanal se ha detenido", + "scheduleStaleDescription": "La copia programada más reciente tiene {days} días. Una ejecución semanal que funciona nunca deja ninguna con más de {threshold} días.", + "scheduleFailedTitle": "La última ejecución programada de la copia de seguridad falló", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "sin motivo registrado", + "scheduleRemedy": "Ejecuta una copia ahora y comprueba que aparece una fila nueva. Si vuelve a fallar, el registro del worker indica la cuenta en la que se detuvo.", "download": "Descargar", "downloadAria": "Descargar copia de seguridad de {username}", "downloadFailed": "La descarga falló", diff --git a/messages/fr.json b/messages/fr.json index c982dd5fa..da5b9e58c 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6923,6 +6923,12 @@ "colCreatedAt": "Dernier instantané", "colActions": "Actions", "retentionLabel": "Conservation : {days} jours", + "scheduleStaleTitle": "La sauvegarde hebdomadaire s'est arrêtée", + "scheduleStaleDescription": "La copie planifiée la plus récente date de {days} jours. Une exécution hebdomadaire qui fonctionne n'en laisse jamais une dépasser {threshold} jours.", + "scheduleFailedTitle": "La dernière exécution planifiée de la sauvegarde a échoué", + "scheduleFailedDescription": "{when} : {reason}", + "scheduleFailedNoReason": "aucune raison enregistrée", + "scheduleRemedy": "Lancez une sauvegarde maintenant et vérifiez qu'une nouvelle ligne apparaît. Si elle échoue à nouveau, le journal du worker indique le compte sur lequel elle s'est arrêtée.", "download": "Télécharger", "downloadAria": "Télécharger la sauvegarde de {username}", "downloadFailed": "Échec du téléchargement", diff --git a/messages/it.json b/messages/it.json index 17cbbc0f3..9410399b1 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6923,6 +6923,12 @@ "colCreatedAt": "Ultima istantanea", "colActions": "Azioni", "retentionLabel": "Conservazione: {days} giorni", + "scheduleStaleTitle": "Il backup settimanale si è fermato", + "scheduleStaleDescription": "La copia pianificata più recente ha {days} giorni. Un'esecuzione settimanale funzionante non ne lascia mai una più vecchia di {threshold} giorni.", + "scheduleFailedTitle": "L'ultima esecuzione pianificata del backup non è riuscita", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "nessun motivo registrato", + "scheduleRemedy": "Avvia subito un backup e verifica che compaia una riga nuova. Se fallisce di nuovo, il log del worker indica l'account su cui si è fermato.", "download": "Scarica", "downloadAria": "Scarica il backup di {username}", "downloadFailed": "Download non riuscito", diff --git a/messages/ko.json b/messages/ko.json index 6c9a29048..e128a90ca 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6904,6 +6904,12 @@ "colCreatedAt": "최근 스냅샷", "colActions": "동작", "retentionLabel": "보관 기간: {days}일", + "scheduleStaleTitle": "주간 백업이 중단되었습니다", + "scheduleStaleDescription": "가장 최근의 예약 사본이 {days}일 전 것입니다. 정상적으로 도는 주간 실행은 {threshold}일보다 오래된 사본을 남기지 않습니다.", + "scheduleFailedTitle": "마지막 예약 백업 실행이 실패했습니다", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "기록된 사유 없음", + "scheduleRemedy": "지금 백업을 실행하고 새 항목이 나타나는지 확인하세요. 다시 실패하면 워커 로그에 중단된 계정이 표시됩니다.", "download": "다운로드", "downloadAria": "{username}의 백업 다운로드", "downloadFailed": "다운로드하지 못했어요", diff --git a/messages/pl.json b/messages/pl.json index 08b955f02..1e4a83fc1 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6923,6 +6923,12 @@ "colCreatedAt": "Ostatnia migawka", "colActions": "Akcje", "retentionLabel": "Przechowywanie: {days} dni", + "scheduleStaleTitle": "Cotygodniowa kopia zapasowa przestała powstawać", + "scheduleStaleDescription": "Najnowsza zaplanowana kopia ma {days} dni. Działające cotygodniowe uruchomienie nigdy nie zostawia kopii starszej niż {threshold} dni.", + "scheduleFailedTitle": "Ostatnie zaplanowane uruchomienie kopii zapasowej nie powiodło się", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "nie zapisano przyczyny", + "scheduleRemedy": "Uruchom kopię teraz i sprawdź, czy pojawi się nowy wiersz. Jeśli znowu się nie powiedzie, log workera wskaże konto, na którym się zatrzymał.", "download": "Pobierz", "downloadAria": "Pobierz kopię zapasową dla {username}", "downloadFailed": "Pobieranie nie powiodło się", diff --git a/src/app/api/admin/backups/[id]/download/route.ts b/src/app/api/admin/backups/[id]/download/route.ts index b063b0ced..0e55b6f7c 100644 --- a/src/app/api/admin/backups/[id]/download/route.ts +++ b/src/app/api/admin/backups/[id]/download/route.ts @@ -19,7 +19,7 @@ import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; -import { decrypt } from "@/lib/crypto"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; import { annotate } from "@/lib/logging/context"; import { parseBackupPayload } from "@/lib/validations/backup"; @@ -52,7 +52,7 @@ export const GET = apiHandler( let plaintext: string; try { - plaintext = decrypt(backup.data); + plaintext = unpackBackupBlob(backup.data); } catch (err) { // Decryption failure is rare but real — a rotated/missing key, or a // corrupted blob from a partially-failed write. Surface it as a 500 diff --git a/src/app/api/admin/backups/[id]/restore/route.ts b/src/app/api/admin/backups/[id]/restore/route.ts index ec71b6817..3deab49d4 100644 --- a/src/app/api/admin/backups/[id]/restore/route.ts +++ b/src/app/api/admin/backups/[id]/restore/route.ts @@ -19,7 +19,7 @@ import { prisma, toJson } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; -import { decrypt } from "@/lib/crypto"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; import { encryptNote } from "@/lib/crypto/note-cipher"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { encryptContextToBytes } from "@/lib/labs/biomarker-store"; @@ -180,7 +180,7 @@ const handler = apiHandler( let plaintext: string; try { - plaintext = decrypt(backup.data); + plaintext = unpackBackupBlob(backup.data); } catch (err) { await auditLog("admin.backups.restore.failed", { userId: admin.id, diff --git a/src/app/api/admin/backups/[id]/summary/route.ts b/src/app/api/admin/backups/[id]/summary/route.ts index fe32c81e7..3ae840d80 100644 --- a/src/app/api/admin/backups/[id]/summary/route.ts +++ b/src/app/api/admin/backups/[id]/summary/route.ts @@ -16,7 +16,7 @@ import { NextRequest } from "next/server"; import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, apiSuccess } from "@/lib/api-response"; -import { decrypt } from "@/lib/crypto"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; import { annotate } from "@/lib/logging/context"; import { isCompatibleSchemaVersion, @@ -44,7 +44,7 @@ export const GET = apiHandler( let plaintext: string; try { - plaintext = decrypt(backup.data); + plaintext = unpackBackupBlob(backup.data); } catch { return apiError("Failed to decrypt backup payload", 500); } diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 11871f979..985e82e22 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -5,11 +5,20 @@ * shipped — only metadata (id, userId, username, type, size in bytes, * createdAt). The encrypted blob remains server-side; admins can trigger * a re-snapshot but not download another user's payload from the UI. + * + * Alongside the rows, the response says whether the SCHEDULE is still alive: + * how old the newest scheduled copy is and how the last scheduled run ended. + * Without those two the page cannot tell a working backup from one that + * stopped six weeks ago — every row it lists has a perfectly ordinary + * timestamp either way. */ import { prisma } from "@/lib/db"; import { apiHandler, requireAdmin } from "@/lib/api-handler"; import { apiSuccess } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; +import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; +import { readLastQueueRun } from "@/lib/jobs/job-failures"; +import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status"; // v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts` // so callers (in particular `components/admin/backups-section.tsx`) don't // have to reach across the component → route-handler layer boundary. @@ -17,31 +26,57 @@ import type { BackupRow, BackupsList } from "@/types/backups"; export const dynamic = "force-dynamic"; +/** The type the weekly worker writes; anything else was uploaded by hand. */ +const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO"; + +interface BackupMetaRow { + id: string; + user_id: string; + username: string; + type: string; + size_bytes: number; + created_at: Date; +} + export const GET = apiHandler(async () => { await requireAdmin(); annotate({ action: { name: "admin.backups.list" } }); - // Pull metadata only — `data` would balloon the response and reveal - // ciphertext we don't need on the listing page. - const backups = await prisma.dataBackup.findMany({ - include: { user: { select: { id: true, username: true } } }, - orderBy: { createdAt: "desc" }, - }); + // Metadata only, and the size read as `octet_length` in the database rather + // than by pulling every blob into this process to measure it. The listing + // needs a number, not the ciphertext, and on an instance with large records + // fetching them all was its own way of running out of memory. Base64 is + // ASCII, so the column's byte length is exactly what the wire reports. + const rows = await prisma.$queryRaw` + SELECT b.id, b.user_id, u.username, b.type, + octet_length(b.data) AS size_bytes, + b.created_at + FROM data_backups b + JOIN users u ON u.id = b.user_id + ORDER BY b.created_at DESC + `; - const rows: BackupRow[] = backups.map((b) => ({ - id: b.id, - userId: b.userId, - username: b.user.username, - type: b.type, - sizeBytes: Buffer.byteLength(b.data, "utf8"), - createdAt: b.createdAt.toISOString(), + const list: BackupRow[] = rows.map((row) => ({ + id: row.id, + userId: row.user_id, + username: row.username, + type: row.type, + sizeBytes: Number(row.size_bytes), + createdAt: row.created_at.toISOString(), })); const payload: BackupsList = { - rows, + rows: list, // Matches the retention window the backup-prune job enforces so the // backups page states the same number the worker acts on. retentionDays: 90, + schedule: summariseBackupSchedule({ + scheduledCreatedAt: rows + .filter((row) => row.type === SCHEDULED_BACKUP_TYPE) + .map((row) => row.created_at), + lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE), + now: new Date(), + }), }; return apiSuccess(payload); diff --git a/src/app/api/admin/backups/run/route.ts b/src/app/api/admin/backups/run/route.ts index 856705c10..0c730130a 100644 --- a/src/app/api/admin/backups/run/route.ts +++ b/src/app/api/admin/backups/run/route.ts @@ -15,11 +15,13 @@ import { auditLog } from "@/lib/auth/audit"; import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { checkRateLimit } from "@/lib/rate-limit"; +import { + DATA_BACKUP_QUEUE, + DATA_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/data-backup-policy"; export const dynamic = "force-dynamic"; -const DATA_BACKUP_QUEUE = "data-backup"; - export const POST = apiHandler(async (request: NextRequest) => { const { user: admin } = await requireAdmin(); annotate({ action: { name: "admin.backups.run" } }); @@ -57,9 +59,14 @@ export const POST = apiHandler(async (request: NextRequest) => { throw new HttpError(503, "Background worker is not running"); } - const jobId = await boss.send(DATA_BACKUP_QUEUE, { - triggeredAt: new Date().toISOString(), - }); + // Same expiration window as the weekly cron. An ad-hoc snapshot runs the + // identical pass over the identical record, so it must not silently keep the + // 15-minute queue default the cron was rescued from. + const jobId = await boss.send( + DATA_BACKUP_QUEUE, + { triggeredAt: new Date().toISOString() }, + DATA_BACKUP_SEND_OPTIONS, + ); annotate({ meta: { job_id: jobId ?? null } }); diff --git a/src/app/api/admin/backups/upload/route.ts b/src/app/api/admin/backups/upload/route.ts index 28012e04b..b7b6443ea 100644 --- a/src/app/api/admin/backups/upload/route.ts +++ b/src/app/api/admin/backups/upload/route.ts @@ -26,7 +26,7 @@ import { prisma } from "@/lib/db"; import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler"; import { apiError, apiSuccess, getClientIp } from "@/lib/api-response"; import { auditLog } from "@/lib/auth/audit"; -import { encrypt } from "@/lib/crypto"; +import { packBackupBlob } from "@/lib/export/backup-blob"; import { annotate } from "@/lib/logging/context"; import { checkRateLimit } from "@/lib/rate-limit"; import { @@ -190,7 +190,7 @@ export const POST = apiHandler(async (request: NextRequest) => { // intact so multiple uploads can coexist for the same user without // overwriting each other or the rolling WEEKLY_AUTO snapshot. const uploadType = `MANUAL_UPLOAD_${Date.now()}`; - const encrypted = encrypt(JSON.stringify(payload)); + const encrypted = packBackupBlob(JSON.stringify(payload)); const created = await prisma.dataBackup.create({ data: { diff --git a/src/components/admin/__tests__/backups-section-empty.test.tsx b/src/components/admin/__tests__/backups-section-empty.test.tsx index 6dba6cd8f..6a8b2afd5 100644 --- a/src/components/admin/__tests__/backups-section-empty.test.tsx +++ b/src/components/admin/__tests__/backups-section-empty.test.tsx @@ -3,11 +3,27 @@ import { renderToStaticMarkup } from "react-dom/server"; import de from "../../../../messages/de.json"; import en from "../../../../messages/en.json"; +const healthySchedule = { + lastSuccessAt: "2026-09-01T03:00:00.000Z", + lastSuccessAgeDays: 2, + staleAfterDays: 10, + stale: false, + lastRun: { + at: "2026-09-01T03:00:12.000Z", + state: "completed" as const, + error: null, + }, + lastRunFailed: false, +}; + const mocks = vi.hoisted(() => ({ mutationOptions: [] as Array<{ onSuccess?: (data: { summary: Record }) => void; }>, toastSuccess: vi.fn(), + // Mutable so one file can render both a healthy schedule and a broken one; + // the query mock reads it on every render. + queryData: { rows: [], retentionDays: 30 } as Record, })); /** @@ -34,7 +50,7 @@ vi.mock("sonner", () => ({ vi.mock("@tanstack/react-query", () => ({ useQuery: () => ({ - data: { rows: [], retentionDays: 30 }, + data: mocks.queryData, isLoading: false, isError: false, }), @@ -69,6 +85,11 @@ import { I18nProvider } from "@/lib/i18n/context"; import { BackupsSection } from "../backups-section"; function render(locale: "en" | "de" = "en") { + mocks.queryData = { + rows: [], + retentionDays: 30, + schedule: healthySchedule, + }; return renderToStaticMarkup( @@ -129,6 +150,68 @@ describe("BackupsSection — empty state", () => { }); }); +/** + * The gap that let a weekly backup stop for a month and a half without anyone + * noticing: the page rendered whatever rows it had, and a copy from six weeks + * ago is indistinguishable from Sunday's until somebody subtracts the dates. + */ +describe("BackupsSection — schedule health", () => { + function withSchedule(schedule: Record) { + mocks.queryData = { rows: [], retentionDays: 30, schedule }; + return renderToStaticMarkup( + + + , + ); + } + + it("stays quiet while the schedule is healthy", () => { + const html = render(); + expect(html).not.toContain('data-slot="backup-schedule-notice"'); + }); + + it("says so when the newest scheduled copy has gone stale", () => { + const html = withSchedule({ + ...healthySchedule, + lastSuccessAt: "2026-07-19T03:00:00.000Z", + lastSuccessAgeDays: 46, + stale: true, + lastRun: null, + lastRunFailed: false, + }); + + expect(html).toContain('data-slot="backup-schedule-notice"'); + expect(html).toContain("The weekly backup has stopped"); + expect(html).toContain("46"); + }); + + it("names the reason when the last scheduled run failed", () => { + const html = withSchedule({ + ...healthySchedule, + lastRun: { + at: "2026-08-30T03:46:00.000Z", + state: "failed", + error: "job timed out", + }, + lastRunFailed: true, + }); + + expect(html).toContain("The last scheduled backup run failed"); + expect(html).toContain("job timed out"); + }); + + it("survives a response that predates the schedule field", () => { + mocks.queryData = { rows: [], retentionDays: 30 }; + expect(() => + renderToStaticMarkup( + + + , + ), + ).not.toThrow(); + }); +}); + describe("BackupsSection — upload outcome", () => { it("includes health profile fact revisions in the restored record total", () => { mocks.toastSuccess.mockClear(); diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index 5e2243a91..6f2c0f1d6 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -48,6 +48,7 @@ import { ListRow } from "@/components/ui/list-row"; import { useFormatters, useTranslations } from "@/lib/i18n/context"; import { queryKeys } from "@/lib/query-keys"; import type { BackupRow, BackupsList } from "@/types/backups"; +import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status"; import type { MissingBackupSection, RestoreSkipSummary, @@ -560,6 +561,71 @@ export function RestoreSkipReport({ ); } +/** + * Says out loud whether the SCHEDULE is still alive. + * + * The weekly pass on the maintainer's own instance stopped producing copies + * and nothing surfaced it for a month and a half: the table listed the rows it + * had, each with an ordinary timestamp, and a copy from six weeks ago reads + * exactly like one made on Sunday until somebody does the subtraction. Two + * independent facts are shown, because they fail at different times — the age + * of the newest scheduled copy never ages out, while the run's own outcome + * carries the reason ("job timed out") and survives only as long as pg-boss + * keeps the row. + * + * Not dismissible: it reflects live state, so the only way to clear it is to + * make a backup succeed. + */ +function ScheduleHealthNotice({ + schedule, +}: { + schedule: BackupScheduleStatus; +}) { + const { t } = useTranslations(); + const fmt = useFormatters(); + if (!schedule.stale && !schedule.lastRunFailed) return null; + + return ( +
+
+
+
+ ); +} + export function BackupsSection() { const { t } = useTranslations(); const fmt = useFormatters(); @@ -833,6 +899,10 @@ export function BackupsSection() { } /> + {data?.schedule ? ( + + ) : null} + {/* Upload card — separate from the table so admins can ingest a backup file independently of any existing rows. The visible button proxies a hidden file input so the layout stays clean diff --git a/src/lib/export/__tests__/backup-blob.test.ts b/src/lib/export/__tests__/backup-blob.test.ts new file mode 100644 index 000000000..995ed07a6 --- /dev/null +++ b/src/lib/export/__tests__/backup-blob.test.ts @@ -0,0 +1,65 @@ +/** + * The stored-backup envelope: compress, then encrypt. + * + * The weekly worker used to hand `encrypt(JSON.stringify(payload))` straight to + * Postgres. For an account with a few hundred thousand measurements that means + * the object graph, the JSON string, the base64 ciphertext (1.33× the JSON) and + * the copy the database driver makes of it all alive at once — well over a + * gigabyte for a record whose JSON is a few hundred megabytes. The run died of + * memory, not of slow SQL. + * + * Compressing before the cipher is what makes the tail of that pipeline cheap: + * health JSON is extremely repetitive, so everything downstream of the gzip + * shrinks by an order of magnitude. The envelope has to stay readable both + * ways — rows written before this change are plain `encrypt(json)` and must + * keep restoring. + */ +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { describe, expect, it } from "vitest"; + +import { encrypt } from "@/lib/crypto"; +import { packBackupBlob, unpackBackupBlob } from "../backup-blob"; + +/** A record-shaped string: many rows, few distinct keys. */ +function sampleJson(rows: number): string { + return JSON.stringify({ + schemaVersion: "2", + measurements: Array.from({ length: rows }, (_, i) => ({ + id: `measurement-${i}`, + type: "PULSE", + value: 60 + (i % 30), + unit: "bpm", + measuredAt: new Date(Date.UTC(2026, 0, 1, 0, i % 60)).toISOString(), + source: "APPLE_HEALTH", + deletedAt: null, + })), + }); +} + +describe("backup blob envelope", () => { + it("round-trips the exact JSON it was handed", () => { + const json = sampleJson(200); + expect(unpackBackupBlob(packBackupBlob(json))).toBe(json); + }); + + it("stores a record-shaped payload far smaller than the plaintext", () => { + const json = sampleJson(5_000); + const packed = packBackupBlob(json); + // The old envelope was 1.33× the JSON. Anything near that means the + // compression leg is not running. + expect(packed.length).toBeLessThan(json.length / 4); + }); + + it("still reads a row written before the envelope existed", () => { + const json = sampleJson(10); + expect(unpackBackupBlob(encrypt(json))).toBe(json); + }); + + it("fails closed on a corrupt blob rather than returning junk", () => { + const packed = packBackupBlob(sampleJson(10)); + const mangled = `${packed.slice(0, -8)}AAAAAAAA`; + expect(() => unpackBackupBlob(mangled)).toThrow(); + }); +}); diff --git a/src/lib/export/backup-blob.ts b/src/lib/export/backup-blob.ts new file mode 100644 index 000000000..92d26d2b2 --- /dev/null +++ b/src/lib/export/backup-blob.ts @@ -0,0 +1,59 @@ +/** + * The envelope a `DataBackup.data` row is stored in: compress, then encrypt. + * + * Why it exists. The weekly worker handed `encrypt(JSON.stringify(payload))` + * straight to Postgres. For an account with a few hundred thousand + * measurements that is four full copies of the record alive at the same + * moment: the payload object graph, the JSON string, the base64 ciphertext at + * 1.33× the JSON, and the copy the database driver makes of that parameter on + * its way to the wire. Measured on a seeded 445 000-measurement account, the + * JSON alone is 242 MB and the run dies of heap exhaustion — under a 1 GB heap + * it never reaches the insert, and on a bigger heap it only reaches it later. + * The weekly pass was not slow; it could not finish. + * + * Compressing first is what makes the whole tail of that pipeline cheap. A + * health record's JSON is extremely repetitive — the same twenty keys per row, + * timestamps sharing a prefix — so gzip takes that 242 MB to 14 MB, and every + * copy after it shrinks with it. The same run then completes in about eight + * seconds with a peak heap of 434 MB. It is also the reason the stored row + * stops being a liability of its own: the backup lives INSIDE the database it + * would be needed to restore, so its size is not a cosmetic concern. + * + * Both directions. Rows written before this envelope existed are plain + * `encrypt(json)` and have to keep restoring — an operator whose newest usable + * copy predates the fix is exactly the person who needs it to work. The marker + * sits inside the ciphertext rather than in front of it so the stored column + * keeps the one shape `decrypt()` already understands, key ids and all. + */ +import { Buffer } from "node:buffer"; +import { gunzipSync, gzipSync } from "node:zlib"; + +import { decrypt, encrypt } from "@/lib/crypto"; + +/** + * Prefix of the DECRYPTED plaintext when the body is gzipped-then-base64'd. + * Chosen so it can never be mistaken for the alternative: a plain payload is + * always a JSON object and therefore always starts with `{`. + */ +const GZIP_MARKER = "HLZ1:"; + +/** Serialised backup JSON → the string stored in `DataBackup.data`. */ +export function packBackupBlob(json: string): string { + const compressed = gzipSync(json).toString("base64"); + return encrypt(`${GZIP_MARKER}${compressed}`); +} + +/** + * A stored `DataBackup.data` string → the backup JSON. + * + * Fails closed on every arm: a bad key, a mangled ciphertext or a truncated + * gzip member throws rather than returning a partial document, because every + * caller goes on to parse the result as a whole account. + */ +export function unpackBackupBlob(stored: string): string { + const plaintext = decrypt(stored); + if (!plaintext.startsWith(GZIP_MARKER)) return plaintext; + return gunzipSync( + Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"), + ).toString("utf8"); +} diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index b04d6e6c6..fbe8947e0 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -316,6 +316,26 @@ function serialiseMoodContext( }; } +/** + * The same payload, already serialised, with the object graph released. + * + * Its own frame on purpose, and both backup jobs go through it. The payload + * graph is the largest thing either job allocates — on an account with a few + * hundred thousand measurements it is several hundred megabytes of small + * objects — and neither job wants anything but the string. Returning from here + * makes the graph unreachable before the compress-and-encrypt step allocates + * anything of its own, which is part of why the weekly pass now fits in + * memory. Inlining these two statements back into a caller quietly undoes it. + */ +export async function buildFullBackupJson( + prisma: PrismaClient, + userId: string, + options: FullBackupOptions = {}, +): Promise { + const { payload } = await buildFullBackupPayload(prisma, userId, options); + return JSON.stringify(payload); +} + /** * Build the canonical full-backup payload for `userId`. Portable exports omit * tombstones and document ciphertext; disaster-recovery payloads preserve diff --git a/src/lib/jobs/__tests__/backup-schedule-status.test.ts b/src/lib/jobs/__tests__/backup-schedule-status.test.ts new file mode 100644 index 000000000..2a45bee31 --- /dev/null +++ b/src/lib/jobs/__tests__/backup-schedule-status.test.ts @@ -0,0 +1,92 @@ +/** + * What the backups page has to be able to say out loud. + * + * The weekly snapshot stopped producing a new copy and nothing surfaced it: the + * page listed the rows it had, each with a perfectly ordinary timestamp, and a + * copy that was a month and a half old looked exactly like one made on Sunday. + * The two facts that would have shown it are the age of the newest scheduled + * copy and the terminal state of the last scheduled run. + */ +import { describe, expect, it } from "vitest"; + +import { + BACKUP_STALE_AFTER_DAYS, + summariseBackupSchedule, +} from "../backup-schedule-status"; + +const NOW = new Date("2026-09-03T09:00:00.000Z"); +const daysAgo = (days: number) => + new Date(NOW.getTime() - days * 24 * 60 * 60 * 1000); + +describe("summariseBackupSchedule", () => { + it("reports a fresh weekly copy as neither stale nor failed", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(2), daysAgo(9)], + lastRun: { + at: daysAgo(2).toISOString(), + state: "completed", + error: null, + }, + now: NOW, + }); + + expect(status.lastSuccessAt).toBe(daysAgo(2).toISOString()); + expect(status.lastSuccessAgeDays).toBe(2); + expect(status.stale).toBe(false); + expect(status.lastRunFailed).toBe(false); + expect(status.staleAfterDays).toBe(BACKUP_STALE_AFTER_DAYS); + }); + + it("calls a copy older than the window stale", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(46)], + lastRun: null, + now: NOW, + }); + + expect(status.lastSuccessAgeDays).toBe(46); + expect(status.stale).toBe(true); + // No run row survives that long — the age is the durable signal. + expect(status.lastRunFailed).toBe(false); + }); + + it("surfaces a failed last run even while the copy is still in window", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(3)], + lastRun: { + at: daysAgo(1).toISOString(), + state: "failed", + error: "job timed out", + }, + now: NOW, + }); + + expect(status.stale).toBe(false); + expect(status.lastRunFailed).toBe(true); + expect(status.lastRun?.error).toBe("job timed out"); + }); + + it("does not call a brand-new instance stale", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [], + lastRun: null, + now: NOW, + }); + + expect(status.lastSuccessAt).toBeNull(); + expect(status.lastSuccessAgeDays).toBeNull(); + expect(status.stale).toBe(false); + }); + + it("ignores manual uploads when judging the scheduled cadence", () => { + // The caller passes only WEEKLY_AUTO rows; a manual upload made today must + // not make a dead cron look alive. Proven here by the shape of the input: + // the summary knows nothing but scheduled copies. + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(30)], + lastRun: null, + now: NOW, + }); + expect(status.stale).toBe(true); + }); +}); diff --git a/src/lib/jobs/__tests__/data-backup-send-policy.test.ts b/src/lib/jobs/__tests__/data-backup-send-policy.test.ts new file mode 100644 index 000000000..4071cbc2b --- /dev/null +++ b/src/lib/jobs/__tests__/data-backup-send-policy.test.ts @@ -0,0 +1,55 @@ +/** + * Send-policy pins for the weekly `data-backup` queue. + * + * The queue was registered with no send options at all, so every job it minted + * inherited pg-boss's queue defaults — `expireInSeconds: 900`, `retryLimit: 2`. + * A full-record snapshot of a large account does not fit in fifteen minutes, so + * the pass was killed mid-run, redelivered twice against the same record, and + * gave up roughly 46 minutes after it started. Weekly, silently, for as long as + * the record stayed large. + * + * Three facts have to hold together or the window is not real: + * + * 1. the policy names a window well past the 15-minute default, + * 2. the CRON SCHEDULE carries it — send options ride the schedule because + * pg-boss's `create_queue()` ends in ON CONFLICT DO NOTHING, so a + * queue-level default would be inert on every already-running instance, + * 3. the admin "run now" route sends with the same options, or an ad-hoc + * snapshot silently keeps the default that broke the cron one. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + DATA_BACKUP_QUEUE, + DATA_BACKUP_SEND_OPTIONS, +} from "../data-backup-policy"; + +const read = (file: string) => readFileSync(join(process.cwd(), file), "utf8"); + +describe("data-backup — pg-boss send policy", () => { + it("widens the expiration far past the 15-minute queue default", () => { + expect(DATA_BACKUP_QUEUE).toBe("data-backup"); + // 900 s is the pg-boss 12.26 queue default (`QUEUE_DEFAULTS.expire_seconds` + // in `dist/plans.js`); anything at or below it re-opens the bug. + expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThan(900); + expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThanOrEqual( + 60 * 60, + ); + }); + + it("the weekly schedule carries the policy", () => { + const source = read("src/lib/jobs/reminder/register-maintenance.ts"); + expect(source).toMatch( + /\[\s*DATA_BACKUP_QUEUE,\s*DATA_BACKUP_CRON,\s*DATA_BACKUP_SEND_OPTIONS,?\s*\]/, + ); + }); + + it("the admin run-now route sends with the same policy", () => { + const source = read("src/app/api/admin/backups/run/route.ts"); + expect(source).toMatch( + /boss\.send\(\s*DATA_BACKUP_QUEUE,[\s\S]{0,200}DATA_BACKUP_SEND_OPTIONS,?\s*\)/, + ); + }); +}); diff --git a/src/lib/jobs/__tests__/job-failures-retention.test.ts b/src/lib/jobs/__tests__/job-failures-retention.test.ts index f098283e1..ef3858164 100644 --- a/src/lib/jobs/__tests__/job-failures-retention.test.ts +++ b/src/lib/jobs/__tests__/job-failures-retention.test.ts @@ -26,4 +26,16 @@ describe("pg-boss failed-row ledger retention", () => { PG_BOSS_FAILED_ROW_AVAILABILITY_HOURS, ); }); + + it("pins the queue defaults the data-backup window had to escape", () => { + const plans = readFileSync(join(PG_BOSS_ROOT, "dist/plans.js"), "utf8"); + + // These two lines are the whole story behind the weekly backup's "job + // timed out": a pass with no send options gets fifteen minutes and two + // redeliveries, so a snapshot that cannot finish in fifteen minutes gives + // up 45 minutes later having never once completed. If a pg-boss upgrade + // changes either default, `DATA_BACKUP_SEND_OPTIONS` deserves a re-read. + expect(plans).toContain("expire_seconds: FIFTEEN_MINUTES"); + expect(plans).toContain("retry_limit: 2"); + }); }); diff --git a/src/lib/jobs/__tests__/offhost-backup.test.ts b/src/lib/jobs/__tests__/offhost-backup.test.ts index 6918cd0ca..de9dbe17f 100644 --- a/src/lib/jobs/__tests__/offhost-backup.test.ts +++ b/src/lib/jobs/__tests__/offhost-backup.test.ts @@ -1,3 +1,5 @@ +import { createCipheriv, randomBytes } from "node:crypto"; + import { describe, it, expect, beforeEach, vi } from "vitest"; import { BACKUP_SCHEMA_VERSION, @@ -8,8 +10,20 @@ const mocks = vi.hoisted(() => ({ buildFullBackupPayload: vi.fn(), })); +// The uploader takes the already-serialised form, so the stand-in for +// `buildFullBackupJson` serialises whatever the payload mock was told to +// return. Every case below keeps stubbing the PAYLOAD, which is the thing +// these tests are actually about. vi.mock("@/lib/export/full-backup-payload", () => ({ buildFullBackupPayload: mocks.buildFullBackupPayload, + buildFullBackupJson: async (...args: unknown[]) => + JSON.stringify( + ( + (await mocks.buildFullBackupPayload(...args)) as { + payload: unknown; + } + ).payload, + ), })); import { encryptBackup, @@ -27,10 +41,33 @@ describe("offhost-backup envelope", () => { const key = Buffer.from(ENC_KEY, "hex"); const payload = JSON.stringify({ hello: "world", n: 42 }); const buf = encryptBackup(payload, key); - expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x01"); + // Version 2: the plaintext is gzipped before the cipher sees it. This + // assertion used to pin version 1, which is the version that could not + // encrypt a large account's dump without four full copies of it resident + // at once. The pin is flipped rather than dropped — the byte is the whole + // reason a reader can tell the two apart. + expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x02"); expect(decryptBackup(buf, key)).toBe(payload); }); + it("still reads a version-1 object written before compression", () => { + const key = Buffer.from(ENC_KEY, "hex"); + const payload = JSON.stringify({ hello: "world", n: 42 }); + // Hand-built v1 envelope: magic || 0x01 || iv || tag || ciphertext, the + // exact bytes every object already sitting in an operator's bucket has. + const iv = randomBytes(12); + const cipher = createCipheriv("aes-256-gcm", key, iv); + const ct = Buffer.concat([cipher.update(payload, "utf8"), cipher.final()]); + const legacy = Buffer.concat([ + Buffer.from("HLBK\x01", "binary"), + iv, + cipher.getAuthTag(), + ct, + ]); + + expect(decryptBackup(legacy, key)).toBe(payload); + }); + it("rejects tampered ciphertext", () => { const key = Buffer.from(ENC_KEY, "hex"); const buf = encryptBackup("data", key); diff --git a/src/lib/jobs/backup-schedule-status.ts b/src/lib/jobs/backup-schedule-status.ts new file mode 100644 index 000000000..d7a5ac20d --- /dev/null +++ b/src/lib/jobs/backup-schedule-status.ts @@ -0,0 +1,78 @@ +/** + * Whether the weekly backup is actually still happening, in two facts. + * + * The pass stopped producing a new copy and nothing said so for a month and a + * half. Everything that could have told the operator was either invisible or + * short-lived: the handler's own error never ran (the job was killed, not + * thrown), `recordError()` increments an unnamed counter, and the admin status + * page's failing-queue panel only looks back 72 hours — which for a weekly + * queue means a permanent failure is on screen three days in seven. The backups + * page itself showed a perfectly ordinary row with a perfectly ordinary + * timestamp, because a 46-day-old copy looks exactly like a fresh one until you + * read the date. + * + * So the surface carries both legs, and neither is derived from the other: + * + * - the AGE of the newest scheduled copy, which never ages out of any + * retention window and would have caught this on the second missed Sunday; + * - the terminal state of the last scheduled RUN, which is the only place the + * reason ("job timed out") is written down at all. + * + * Manual uploads are excluded by the shape of the input: the caller passes only + * the scheduled rows, so an admin's ad-hoc upload cannot make a dead cron look + * alive. + */ +import type { LastQueueRun } from "@/lib/jobs/job-failures"; + +/** + * How old the newest scheduled copy may get before the page says so. The cron + * is weekly, so seven days is normal and eight is a rounding error; ten means + * at least one Sunday produced nothing. + */ +export const BACKUP_STALE_AFTER_DAYS = 10; + +const DAY_MS = 24 * 60 * 60 * 1000; + +export interface BackupScheduleStatus { + /** ISO instant of the newest scheduled copy, or null when there is none. */ + lastSuccessAt: string | null; + /** Whole days since that copy was written. */ + lastSuccessAgeDays: number | null; + /** The threshold behind `stale`, so the UI can name it. */ + staleAfterDays: number; + /** A scheduled copy exists and is older than the threshold. */ + stale: boolean; + /** The last scheduled run pg-boss still remembers. */ + lastRun: LastQueueRun | null; + /** That run ended `failed` or `cancelled`. */ + lastRunFailed: boolean; +} + +export function summariseBackupSchedule(args: { + /** `createdAt` of every SCHEDULED backup row; manual uploads excluded. */ + scheduledCreatedAt: readonly Date[]; + lastRun: LastQueueRun | null; + now: Date; +}): BackupScheduleStatus { + const newest = args.scheduledCreatedAt.reduce( + (best, at) => (best === null || at > best ? at : best), + null, + ); + const ageDays = + newest === null + ? null + : Math.floor((args.now.getTime() - newest.getTime()) / DAY_MS); + + return { + lastSuccessAt: newest?.toISOString() ?? null, + lastSuccessAgeDays: ageDays, + staleAfterDays: BACKUP_STALE_AFTER_DAYS, + // An instance that has never produced a scheduled copy is not stale, it is + // new — the page's empty state already says there is nothing there, and + // calling a fresh install "overdue" would train the operator to ignore the + // warning that matters. + stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS, + lastRun: args.lastRun, + lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed", + }; +} diff --git a/src/lib/jobs/data-backup-policy.ts b/src/lib/jobs/data-backup-policy.ts new file mode 100644 index 000000000..22baef4d1 --- /dev/null +++ b/src/lib/jobs/data-backup-policy.ts @@ -0,0 +1,39 @@ +/** + * Queue name and pg-boss send policy for the weekly on-host data backup. + * + * A leaf module on purpose: the cron registrar, the admin "run now" route and + * the policy test all need these two facts, and none of them should have to + * pull in the worker's handler graph to get them. + * + * The queue was registered with no send options, so every job it minted took + * pg-boss 12.26's queue defaults — `expireInSeconds: 900`, `retryLimit: 2` + * (`QUEUE_DEFAULTS` in `pg-boss/dist/plans.js`). Fifteen minutes is a fine + * default for a cleanup tick and far too short for a pass that serialises every + * account's whole record: the job was killed mid-run, redelivered twice against + * the same record, and reached `failed` with `job timed out` about 46 minutes + * after it started. Nothing about that says "too slow" — the pass never got to + * finish once. + * + * Two hours is deliberately generous rather than tuned. The handler now costs a + * fraction of that on a large record (see the CHANGELOG entry), but the pass + * loops over EVERY account on the instance and a self-host's disk is whatever + * it is; the window's job is to be impossible to hit for the right reasons, and + * a failure that arrives at the two-hour mark is a real one worth reading. + * pg-boss refuses anything at or past 24 hours. + * + * The default `retryLimit: 2` stays. The old redeliveries were pointless + * because the first attempt could not succeed either; a transient failure — a + * pool hiccup, a locked row — genuinely benefits from a second attempt, and the + * alternative is losing the week. + * + * The options ride on the SEND, not on `createQueue`: pg-boss's `create_queue()` + * ends in `ON CONFLICT DO NOTHING`, so a queue-level default would apply to a + * fresh database and be inert on every instance that already has the queue — + * exactly the trap `reconcileQueuePolicies` exists to work around. + */ + +export const DATA_BACKUP_QUEUE = "data-backup"; + +export const DATA_BACKUP_SEND_OPTIONS = { + expireInSeconds: 2 * 60 * 60, +} as const; diff --git a/src/lib/jobs/job-failures.ts b/src/lib/jobs/job-failures.ts index 48403da09..d846c88be 100644 --- a/src/lib/jobs/job-failures.ts +++ b/src/lib/jobs/job-failures.ts @@ -98,7 +98,11 @@ export async function readFailingQueues( max(completed_on) AS last_failed_at, ( array_agg( - COALESCE(output->>'message', output::text) + COALESCE( + output->>'message', + output->'value'->>'message', + output::text + ) ORDER BY completed_on DESC NULLS LAST ) )[1] AS last_error @@ -124,6 +128,75 @@ export async function readFailingQueues( } } +/** The terminal state pg-boss recorded for the newest run of one queue. */ +export interface LastQueueRun { + /** ISO instant the run reached its terminal state. */ + at: string; + /** `failed` covers both a thrown handler and an expired (killed) job. */ + state: "completed" | "failed" | "cancelled"; + /** The failure text, redacted and truncated. Null when the run succeeded. */ + error: string | null; +} + +/** + * The newest terminal run of one queue, whatever its outcome. + * + * `readFailingQueues` answers "what broke recently" and is deliberately scoped + * to a 72-hour window, which is the right shape for a status page watching + * thirty queues that tick by the minute. It is the wrong shape for a WEEKLY + * pass: a Sunday-night failure has fallen out of the window by Wednesday, so a + * job that fails every single week is visible for three days in seven and + * invisible for four. This reader takes the newest run instead of a window, so + * a weekly queue's last outcome is readable right up until the next one. + * + * Bounded by pg-boss's own retention: terminal rows live `deletion_seconds` + * (seven days by default) past completion, so `null` here means "no run row + * survives", which is not the same as "the pass never ran". Callers pair it + * with a fact that does not age out — for backups, the age of the newest + * stored copy. + */ +export async function readLastQueueRun( + queue: string, +): Promise { + try { + const rows = await prisma.$queryRaw< + Array<{ state: string; completed_on: Date | null; output: string | null }> + >` + SELECT + state, + completed_on, + -- pg-boss wraps a thrown value: an expired job's row reads + -- {"value":{"message":"job timed out"}}, so a plain ->>'message' + -- misses it and the operator gets raw JSON instead of the sentence. + COALESCE( + output->>'message', + output->'value'->>'message', + output::text + ) AS output + FROM pgboss.job + WHERE name = ${queue} + AND state IN ('completed', 'failed', 'cancelled') + AND completed_on IS NOT NULL + ORDER BY completed_on DESC + LIMIT 1 + `; + const row = rows[0]; + if (!row || row.completed_on === null) return null; + const state = + row.state === "failed" || row.state === "cancelled" + ? row.state + : "completed"; + return { + at: row.completed_on.toISOString(), + state, + error: state === "completed" ? null : presentError(row.output), + }; + } catch { + // No `pgboss` schema (web-only deployment) or no permission to read it. + return null; + } +} + export interface QueueFailureForUser { /** ISO instant the user's newest job on this queue gave up. */ lastFailedAt: string; diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 00cb3a2d7..6d070e7e7 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -18,8 +18,9 @@ * the backup history. See docs/ops/backup-restore.md. */ import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; +import { gunzipSync, gzipSync } from "node:zlib"; import type { PrismaClient } from "@/generated/prisma/client"; -import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { buildFullBackupJson } from "@/lib/export/full-backup-payload"; import { getEvent } from "@/lib/logging/context"; const ALGORITHM = "aes-256-gcm"; @@ -84,20 +85,43 @@ export function loadOffhostConfig(): OffhostBackupConfig | null { * Encrypt a JSON payload with the dedicated backup key. * * Wire format (binary): - * magic(4)="HLBK" || version(1)=0x01 || iv(12) || tag(16) || ciphertext + * magic(4)="HLBK" || version(1) || iv(12) || tag(16) || ciphertext + * + * Version 1 encrypts the JSON directly. Version 2 gzips it first, which is + * what the version byte was there for: a large account's dump is hundreds of + * megabytes of extremely repetitive JSON, and encrypting it whole means the + * string, the cipher's input copy, the ciphertext and the concatenation are + * all resident at once — the same arithmetic that made the in-database weekly + * backup unfinishable. Gzip takes an order of magnitude off every one of them + * and off the object in the bucket. Objects written under version 1 stay + * readable; both the restore drill and `scripts/restore-backup.ts` come + * through `decryptBackup`, so neither needs to know which it got. */ +const BACKUP_ENVELOPE_PLAIN = 0x01; +const BACKUP_ENVELOPE_GZIP = 0x02; + export function encryptBackup(plaintext: string, key: Buffer): Buffer { const iv = randomBytes(IV_LENGTH); const cipher = createCipheriv(ALGORITHM, key, iv); - const ct = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]); + const ct = Buffer.concat([ + cipher.update(gzipSync(plaintext)), + cipher.final(), + ]); const tag = cipher.getAuthTag(); - return Buffer.concat([Buffer.from("HLBK\x01", "binary"), iv, tag, ct]); + const header = Buffer.from([ + ...Buffer.from("HLBK", "binary"), + BACKUP_ENVELOPE_GZIP, + ]); + return Buffer.concat([header, iv, tag, ct]); } export function decryptBackup(buf: Buffer, key: Buffer): string { const magic = buf.subarray(0, 4).toString("binary"); const version = buf[4]; - if (magic !== "HLBK" || version !== 0x01) { + if ( + magic !== "HLBK" || + (version !== BACKUP_ENVELOPE_PLAIN && version !== BACKUP_ENVELOPE_GZIP) + ) { throw new Error("Invalid backup envelope (bad magic or version)"); } const iv = buf.subarray(5, 5 + IV_LENGTH); @@ -105,7 +129,10 @@ export function decryptBackup(buf: Buffer, key: Buffer): string { const ct = buf.subarray(5 + IV_LENGTH + TAG_LENGTH); const dec = createDecipheriv(ALGORITHM, key, iv); dec.setAuthTag(tag); - return Buffer.concat([dec.update(ct), dec.final()]).toString("utf8"); + const plaintext = Buffer.concat([dec.update(ct), dec.final()]); + return version === BACKUP_ENVELOPE_GZIP + ? gunzipSync(plaintext).toString("utf8") + : plaintext.toString("utf8"); } export interface S3Like { @@ -215,12 +242,13 @@ export async function runOffhostBackup( const evt = getEvent(); for (const user of users) { try { - const { payload } = await buildFullBackupPayload(prisma, user.id, { - purpose: "disaster-recovery", - exportedAt: now, - }); - const plaintext = JSON.stringify(payload); - const ciphertext = encryptBackup(plaintext, cfg.encryptionKey); + const ciphertext = encryptBackup( + await buildFullBackupJson(prisma, user.id, { + purpose: "disaster-recovery", + exportedAt: now, + }), + cfg.encryptionKey, + ); const key = `${dateKey}/user-${user.id}.json.enc`; await s3.putObject(key, ciphertext); uploaded++; diff --git a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts index 30cdc3b14..bfb009c8f 100644 --- a/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts +++ b/src/lib/jobs/reminder/__tests__/backup-handlers.test.ts @@ -4,16 +4,29 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ buildFullBackupPayload: vi.fn(), - encrypt: vi.fn((value: string) => value), + packBlob: vi.fn((value: string) => value), getWorkerPrisma: vi.fn(), upsert: vi.fn(), })); +// The handler takes the already-serialised form; the stand-in serialises +// whatever the payload mock was told to return, so this file keeps stubbing +// and asserting the PAYLOAD, which is what it is about. vi.mock("@/lib/export/full-backup-payload", () => ({ buildFullBackupPayload: mocks.buildFullBackupPayload, + buildFullBackupJson: async (...args: unknown[]) => + JSON.stringify( + ((await mocks.buildFullBackupPayload(...args)) as { payload: unknown }) + .payload, + ), })); -vi.mock("@/lib/crypto", () => ({ encrypt: mocks.encrypt })); +// The envelope is exercised end-to-end in +// `src/lib/export/__tests__/backup-blob.test.ts`; here it stays transparent so +// the stored bytes can be read back as JSON. +vi.mock("@/lib/export/backup-blob", () => ({ + packBackupBlob: mocks.packBlob, +})); vi.mock("@/lib/logging/background", () => ({ withBackgroundEvent: vi.fn( diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index 48d0e3e47..e45da75a8 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -6,8 +6,8 @@ */ import { type Job } from "pg-boss"; import { recordError } from "@/lib/jobs/worker-status"; -import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; -import { encrypt } from "@/lib/crypto"; +import { buildFullBackupJson } from "@/lib/export/full-backup-payload"; +import { packBackupBlob } from "@/lib/export/backup-blob"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { @@ -79,15 +79,21 @@ export async function handleDataBackup( let backed = 0; let usersFailed = 0; + let largestBlobBytes = 0; for (const user of users) { try { - const { payload } = await buildFullBackupPayload(prisma, user.id, { - purpose: "disaster-recovery", - }); - const backupJson = JSON.stringify(payload); - - // Encrypt the backup data (contains sensitive health information) - const encryptedBackup = encrypt(backupJson); + // Compressed, then encrypted (the record contains sensitive health + // information). Both legs live in `packBackupBlob`, which is also + // what the restore path reads back through. + const encryptedBackup = packBackupBlob( + await buildFullBackupJson(prisma, user.id, { + purpose: "disaster-recovery", + }), + ); + largestBlobBytes = Math.max( + largestBlobBytes, + Buffer.byteLength(encryptedBackup, "utf8"), + ); await prisma.dataBackup.upsert({ where: { @@ -113,6 +119,9 @@ export async function handleDataBackup( } } + // The stored size is the one number that says whether this pass is + // heading back towards the wall it hit before: it tracks the record. + evt.addMeta("data_backup_largest_blob_bytes", largestBlobBytes); evt.setBackground({ task_name: "job.data_backup", result: { backed, total: users.length }, diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index b3faf909d..4d60ab352 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -138,6 +138,10 @@ import { } from "@/lib/jobs/environment-fetch"; import { recordError } from "@/lib/jobs/worker-status"; import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + DATA_BACKUP_QUEUE, + DATA_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/data-backup-policy"; import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared"; import { createAndSchedule, @@ -221,8 +225,6 @@ import { type AchievementUnlockSweepPayload, } from "@/lib/jobs/achievement-unlock-sweep"; -const DATA_BACKUP_QUEUE = "data-backup"; - const DATA_BACKUP_CRON = "0 3 * * 0"; // weekly Sunday at 03:00 const RATE_LIMIT_CLEANUP_QUEUE = "rate-limit-cleanup"; @@ -478,7 +480,11 @@ const schedules: ScheduleEntry[] = [ // v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload) // that fans out one per-user environment fetch per opted-in account. [ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON], - [DATA_BACKUP_QUEUE, DATA_BACKUP_CRON], + // The send options are the point of this tuple: without them the weekly + // snapshot inherits pg-boss's 15-minute expiration and is killed mid-run on + // any sizeable record. See `data-backup-policy.ts` for why the window rides + // the schedule rather than the queue. + [DATA_BACKUP_QUEUE, DATA_BACKUP_CRON, DATA_BACKUP_SEND_OPTIONS], [RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry], [IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry], [AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON], diff --git a/src/types/backups.ts b/src/types/backups.ts index 75b0cc66c..1931628ea 100644 --- a/src/types/backups.ts +++ b/src/types/backups.ts @@ -9,6 +9,7 @@ * both sides a route-handler-independent shared type home; the route * keeps owning the HTTP contract, the component keeps owning the UI. */ +import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status"; export interface BackupRow { id: string; @@ -25,6 +26,12 @@ export interface BackupRow { export interface BackupsList { rows: BackupRow[]; + /** + * Whether the weekly schedule is still producing copies, and how the last + * scheduled run ended. A row's own timestamp cannot answer that — a copy + * made six weeks ago and one made on Sunday look identical in a table. + */ + schedule: BackupScheduleStatus; /** * Soft retention hint — the worker is configured for weekly backups * (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and diff --git a/tests/integration/admin-backups-audit.test.ts b/tests/integration/admin-backups-audit.test.ts index 989399700..fe7755fb4 100644 --- a/tests/integration/admin-backups-audit.test.ts +++ b/tests/integration/admin-backups-audit.test.ts @@ -197,3 +197,69 @@ describe("backup endpoints — AuditLog contract", () => { expect(restore?.details ?? "").toContain(seeded.id); }); }); + +/** + * The listing route is what an operator looks at to answer "are my backups + * still being made". It could not answer that: it returned rows and a + * retention hint, and a copy from six weeks ago carries a timestamp exactly + * like Sunday's. It now carries the schedule's own health, and the size comes + * from `octet_length` in the database rather than from pulling every blob into + * the web process to measure it. + */ +describe("GET /api/admin/backups — schedule health", () => { + it("calls a stalled weekly schedule out, and a fresh manual upload does not hide it", async () => { + const prisma = getPrismaClient(); + const admin = await seedAdminSession(); + const blob = encrypt(JSON.stringify(payloadFor(admin.id))); + + const stalled = new Date(Date.now() - 46 * 24 * 60 * 60 * 1000); + await prisma.dataBackup.create({ + data: { + userId: admin.id, + type: "WEEKLY_AUTO", + data: blob, + createdAt: stalled, + }, + }); + // Uploaded by hand today. It must not make the dead cron look alive. + await prisma.dataBackup.create({ + data: { userId: admin.id, type: "MANUAL_UPLOAD_1", data: blob }, + }); + + // The listing handler declares no parameters; `apiHandler` reads the + // request defensively out of its arguments, so calling it bare is what a + // zero-argument handler supports. + const { GET } = await import("@/app/api/admin/backups/route"); + const res = await GET(); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { + rows: Array<{ type: string; sizeBytes: number; username: string }>; + schedule: { + stale: boolean; + lastSuccessAgeDays: number | null; + lastRun: unknown; + lastRunFailed: boolean; + }; + }; + }; + + expect(body.data.rows).toHaveLength(2); + // Sizes are read in SQL now; a wrong column or a wrong cast shows up here + // as a zero or a NaN rather than as a plausible-looking number. + expect(body.data.rows.every((row) => row.sizeBytes === blob.length)).toBe( + true, + ); + expect(body.data.rows.every((row) => row.username === admin.username)).toBe( + true, + ); + + expect(body.data.schedule.lastSuccessAgeDays).toBe(46); + expect(body.data.schedule.stale).toBe(true); + // No `pgboss` schema in the test database — the run reader fails soft to + // null rather than taking the whole page down, which is the arm a web-only + // deployment lives on permanently. + expect(body.data.schedule.lastRun).toBeNull(); + expect(body.data.schedule.lastRunFailed).toBe(false); + }); +}); diff --git a/tests/integration/admin-backups-upload.test.ts b/tests/integration/admin-backups-upload.test.ts index d6866f0ff..30aa3bb1c 100644 --- a/tests/integration/admin-backups-upload.test.ts +++ b/tests/integration/admin-backups-upload.test.ts @@ -4,7 +4,7 @@ * Verifies: * - 201 + a `DataBackup` row with type prefix `MANUAL_UPLOAD_` is * created when the JSON file matches the schema. - * - the original payload round-trips through encrypt → decrypt. + * - the original payload round-trips through the stored backup envelope. * - 422 when the JSON is malformed. * - 422 when the schema validation fails (missing required field). * - 422 when the userId on the file doesn't exist. @@ -15,7 +15,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; process.env.ENCRYPTION_KEY ??= "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; -import { decrypt } from "@/lib/crypto"; +import { unpackBackupBlob } from "@/lib/export/backup-blob"; import { cookieJar, headerJar } from "./mock-next-headers"; import { getPrismaClient, truncateAllTables } from "./setup"; @@ -166,9 +166,11 @@ describe("POST /api/admin/backups/upload", () => { expect(row?.type).toMatch(/^MANUAL_UPLOAD_\d+$/); expect(row?.userId).toBe(admin.id); - // Round-trip the ciphertext to make sure we stored the same payload - // we received. - const decrypted = JSON.parse(decrypt(row!.data)); + // Round-trip the stored blob to make sure we kept the same payload we + // received. Through the envelope rather than through `decrypt` alone: an + // uploaded file is stored exactly the way the weekly worker stores one, so + // the restore path reads both without knowing where the row came from. + const decrypted = JSON.parse(unpackBackupBlob(row!.data)); expect(decrypted.userId).toBe(admin.id); expect(decrypted.measurements).toHaveLength(1); diff --git a/tests/integration/backup-round-trip.test.ts b/tests/integration/backup-round-trip.test.ts index 2d22ba3bf..32542eb11 100644 --- a/tests/integration/backup-round-trip.test.ts +++ b/tests/integration/backup-round-trip.test.ts @@ -63,9 +63,13 @@ import { encryptWaveformToBytes, } from "@/lib/withings/ecg-waveform-codec"; import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +<<<<<<< HEAD import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker"; import { decryptNoteFromBytes } from "@/lib/labs/store"; import { decryptContextFromBytes } from "@/lib/labs/biomarker-store"; +======= +import { packBackupBlob } from "@/lib/export/backup-blob"; +>>>>>>> origin/fix/backup-timeout import { TWO_ENDED_MODELS, type TwoEndedModel } from "@/lib/export/backup-plan"; import { POST } from "@/app/api/admin/backups/[id]/restore/route"; @@ -262,6 +266,27 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise { source: "MANUAL", }, }); + // A DELETED reading, kept as a tombstone. It is here because the obvious way + // to make the weekly backup cheaper — stop exporting soft-deleted rows, which + // on a real record is four rows in five — is wrong, and nothing in the tree + // said so out loud. A tombstone inside the retention window is what tells a + // synced client the row is gone (`/api/sync/changes` reads exactly these) and + // what makes a re-sent Apple Health sample a no-op instead of a resurrection. + // Drop them from a disaster-recovery file and a restore quietly brings back + // every measurement the account ever deleted on the next device sync. + await prisma.measurement.create({ + data: { + userId: OWNER_ID, + type: "PULSE", + value: 61, + unit: "bpm", + measuredAt: AT("2026-07-02T06:30:00.000Z"), + source: "APPLE_HEALTH", + externalId: "round-trip-deleted-sample", + syncVersion: 4, + deletedAt: AT("2026-07-20T09:15:00.000Z"), + }, + }); // The EVENT row an ECG strip is filed against, and the strip itself. The // pairing is the point: `EcgRecording.measurementId` is a real foreign key, // so this is the reference whose remap has to survive the round trip rather @@ -1234,6 +1259,26 @@ describe("every model the plan claims two-ended survives a real restore", () => purpose: "disaster-recovery", }); + // The tombstone has to be IN the file. A disaster-recovery payload that + // carries only live rows is smaller and cheaper and wrong: the restore + // wipes the account first, so a deletion that is not in the file is a + // deletion the account no longer knows it made, and the next device sync + // re-inserts every one of those samples as fresh readings. + const exportedMeasurements = payload.measurements as Array<{ + externalId: string | null; + deletedAt: string | null; + syncVersion: number; + }>; + expect( + exportedMeasurements.find( + (row) => row.externalId === "round-trip-deleted-sample", + ), + "a soft-deleted reading is missing from the disaster-recovery payload", + ).toMatchObject({ + deletedAt: "2026-07-20T09:15:00.000Z", + syncVersion: 4, + }); + // Cascade rather than a hand-listed sweep — see the file comment. await prisma.user.delete({ where: { id: OWNER_ID } }); await createOwner(prisma); @@ -1250,7 +1295,12 @@ describe("every model the plan claims two-ended survives a real restore", () => data: { userId: OWNER_ID, type: "TWO_ENDED_ROUND_TRIP", - data: encrypt(JSON.stringify(payload)), + // The envelope the weekly worker actually writes: gzip, then + // encrypt. The three cases further down deliberately keep the plain + // `encrypt(json)` form, which is what every row written before this + // envelope existed looks like — an operator's newest usable copy may + // well be one of those, so both arms have to reach the restore route. + data: packBackupBlob(JSON.stringify(payload)), }, }); const response = await POST( @@ -2029,6 +2079,18 @@ describe("every model the plan claims two-ended survives a real restore", () => // measurement pointer is asserted against the row the RESTORE wrote rather // than against the id the fixture used, because resolving to something // that exists is the property the reference owes. + const restoredTombstone = await prisma.measurement.findFirstOrThrow({ + where: { userId: OWNER_ID, externalId: "round-trip-deleted-sample" }, + }); + expect( + { + deletedAt: restoredTombstone.deletedAt?.toISOString() ?? null, + syncVersion: restoredTombstone.syncVersion, + }, + "the deleted reading came back, but not as a deletion — a tombstone " + + "restored without its `deletedAt` is a resurrected measurement", + ).toEqual({ deletedAt: "2026-07-20T09:15:00.000Z", syncVersion: 4 }); + const restoredMeasurement = await prisma.measurement.findFirstOrThrow({ where: { userId: OWNER_ID, type: "WEIGHT" }, });