diff --git a/CHANGELOG.md b/CHANGELOG.md index 48b6ec02c..91e6c38b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## [Unreleased] + +### Fixed + +- **A large record no longer outruns the weekly backup.** On an account with a + few hundred thousand readings the Sunday snapshot could not finish. It was + never given a time limit of its own, so it inherited the queue's fifteen + minutes, was killed part-way through, redelivered twice against the same + record, and gave up three quarters of an hour later having never completed + once. The reason it needed so long was memory: the whole account was built in + memory, serialised whole, encrypted whole, and handed to the database whole, + which is four full copies of the record at the same moment. The stored copy + is now compressed before it is encrypted, which takes an order of magnitude + off everything after it, and the pass releases each stage before allocating + the next. A seeded 445 000-reading account went from dying of memory to + finishing in about twenty seconds, with a stored copy of 25 MB instead of + 322 MB. The pass also has an explicit two-hour window now, so a genuinely + slow disk cannot be mistaken for a broken run. The nightly off-host copy is + built the same way and got the same treatment; files written before this + still restore, and so do backups already sitting in the database. +- **A failed backup run is now visible on the backups page.** The weekly pass + can stop and leave no trace: the page listed whatever copies existed, and a + copy from six weeks ago carries a timestamp exactly like Sunday's. The page + now says how old the newest scheduled copy is when it has gone past due, and + how the last scheduled run ended when it ended badly, with the reason the + queue recorded. Both are shown because they stop being available at different + times — a run's own record is kept for a week, the age of the newest copy for + as long as the copy exists. + ## [1.38.4] — 2026-09-02 The shared AI key reaches the provider its address names, and the image diff --git a/messages/de.json b/messages/de.json index ae260abd6..66f499faa 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Letzter Snapshot", "colActions": "Aktionen", "retentionLabel": "Aufbewahrung: {days} Tage", + "scheduleStaleTitle": "Die wöchentliche Sicherung läuft nicht mehr", + "scheduleStaleDescription": "Die neueste geplante Kopie ist {days} Tage alt. Ein funktionierender wöchentlicher Lauf lässt keine älter als {threshold} Tage werden.", + "scheduleFailedTitle": "Der letzte geplante Sicherungslauf ist fehlgeschlagen", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "kein Grund aufgezeichnet", + "scheduleRemedy": "Starte jetzt eine Sicherung und prüfe, ob eine frische Zeile erscheint. Schlägt sie erneut fehl, nennt das Worker-Protokoll das Konto, an dem sie stehen geblieben ist.", "download": "Herunterladen", "downloadAria": "Backup für {username} herunterladen", "downloadFailed": "Download fehlgeschlagen", diff --git a/messages/en.json b/messages/en.json index bfbb0ab32..f368799ab 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Last snapshot", "colActions": "Actions", "retentionLabel": "Retention: {days} days", + "scheduleStaleTitle": "The weekly backup has stopped", + "scheduleStaleDescription": "The newest scheduled copy is {days} days old. A working weekly run never leaves one older than {threshold} days.", + "scheduleFailedTitle": "The last scheduled backup run failed", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "no reason recorded", + "scheduleRemedy": "Run a backup now and check that a fresh row appears. If it fails again, the worker log names the account it stopped on.", "download": "Download", "downloadAria": "Download backup for {username}", "downloadFailed": "Download failed", diff --git a/messages/es.json b/messages/es.json index 2ecddf478..882d57a3f 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Última instantánea", "colActions": "Acciones", "retentionLabel": "Conservación: {days} días", + "scheduleStaleTitle": "La copia de seguridad semanal se ha detenido", + "scheduleStaleDescription": "La copia programada más reciente tiene {days} días. Una ejecución semanal que funciona nunca deja ninguna con más de {threshold} días.", + "scheduleFailedTitle": "La última ejecución programada de la copia de seguridad falló", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "sin motivo registrado", + "scheduleRemedy": "Ejecuta una copia ahora y comprueba que aparece una fila nueva. Si vuelve a fallar, el registro del worker indica la cuenta en la que se detuvo.", "download": "Descargar", "downloadAria": "Descargar copia de seguridad de {username}", "downloadFailed": "La descarga falló", diff --git a/messages/fr.json b/messages/fr.json index 63b62b5ff..cc826c2ee 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Dernier instantané", "colActions": "Actions", "retentionLabel": "Conservation : {days} jours", + "scheduleStaleTitle": "La sauvegarde hebdomadaire s'est arrêtée", + "scheduleStaleDescription": "La copie planifiée la plus récente date de {days} jours. Une exécution hebdomadaire qui fonctionne n'en laisse jamais une dépasser {threshold} jours.", + "scheduleFailedTitle": "La dernière exécution planifiée de la sauvegarde a échoué", + "scheduleFailedDescription": "{when} : {reason}", + "scheduleFailedNoReason": "aucune raison enregistrée", + "scheduleRemedy": "Lancez une sauvegarde maintenant et vérifiez qu'une nouvelle ligne apparaît. Si elle échoue à nouveau, le journal du worker indique le compte sur lequel elle s'est arrêtée.", "download": "Télécharger", "downloadAria": "Télécharger la sauvegarde de {username}", "downloadFailed": "Échec du téléchargement", diff --git a/messages/it.json b/messages/it.json index 3cc1dd08e..0e5e5c624 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Ultima istantanea", "colActions": "Azioni", "retentionLabel": "Conservazione: {days} giorni", + "scheduleStaleTitle": "Il backup settimanale si è fermato", + "scheduleStaleDescription": "La copia pianificata più recente ha {days} giorni. Un'esecuzione settimanale funzionante non ne lascia mai una più vecchia di {threshold} giorni.", + "scheduleFailedTitle": "L'ultima esecuzione pianificata del backup non è riuscita", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "nessun motivo registrato", + "scheduleRemedy": "Avvia subito un backup e verifica che compaia una riga nuova. Se fallisce di nuovo, il log del worker indica l'account su cui si è fermato.", "download": "Scarica", "downloadAria": "Scarica il backup di {username}", "downloadFailed": "Download non riuscito", diff --git a/messages/ko.json b/messages/ko.json index 062b3e109..85a8c19af 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6901,6 +6901,12 @@ "colCreatedAt": "최근 스냅샷", "colActions": "동작", "retentionLabel": "보관 기간: {days}일", + "scheduleStaleTitle": "주간 백업이 중단되었습니다", + "scheduleStaleDescription": "가장 최근의 예약 사본이 {days}일 전 것입니다. 정상적으로 도는 주간 실행은 {threshold}일보다 오래된 사본을 남기지 않습니다.", + "scheduleFailedTitle": "마지막 예약 백업 실행이 실패했습니다", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "기록된 사유 없음", + "scheduleRemedy": "지금 백업을 실행하고 새 항목이 나타나는지 확인하세요. 다시 실패하면 워커 로그에 중단된 계정이 표시됩니다.", "download": "다운로드", "downloadAria": "{username}의 백업 다운로드", "downloadFailed": "다운로드하지 못했어요", diff --git a/messages/pl.json b/messages/pl.json index de783600b..bb784d314 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6920,6 +6920,12 @@ "colCreatedAt": "Ostatnia migawka", "colActions": "Akcje", "retentionLabel": "Przechowywanie: {days} dni", + "scheduleStaleTitle": "Cotygodniowa kopia zapasowa przestała powstawać", + "scheduleStaleDescription": "Najnowsza zaplanowana kopia ma {days} dni. Działające cotygodniowe uruchomienie nigdy nie zostawia kopii starszej niż {threshold} dni.", + "scheduleFailedTitle": "Ostatnie zaplanowane uruchomienie kopii zapasowej nie powiodło się", + "scheduleFailedDescription": "{when}: {reason}", + "scheduleFailedNoReason": "nie zapisano przyczyny", + "scheduleRemedy": "Uruchom kopię teraz i sprawdź, czy pojawi się nowy wiersz. Jeśli znowu się nie powiedzie, log workera wskaże konto, na którym się zatrzymał.", "download": "Pobierz", "downloadAria": "Pobierz kopię zapasową dla {username}", "downloadFailed": "Pobieranie nie powiodło się", diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 11871f979..985e82e22 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -5,11 +5,20 @@ * shipped — only metadata (id, userId, username, type, size in bytes, * createdAt). The encrypted blob remains server-side; admins can trigger * a re-snapshot but not download another user's payload from the UI. + * + * Alongside the rows, the response says whether the SCHEDULE is still alive: + * how old the newest scheduled copy is and how the last scheduled run ended. + * Without those two the page cannot tell a working backup from one that + * stopped six weeks ago — every row it lists has a perfectly ordinary + * timestamp either way. */ import { prisma } from "@/lib/db"; import { apiHandler, requireAdmin } from "@/lib/api-handler"; import { apiSuccess } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; +import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; +import { readLastQueueRun } from "@/lib/jobs/job-failures"; +import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status"; // v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts` // so callers (in particular `components/admin/backups-section.tsx`) don't // have to reach across the component → route-handler layer boundary. @@ -17,31 +26,57 @@ import type { BackupRow, BackupsList } from "@/types/backups"; export const dynamic = "force-dynamic"; +/** The type the weekly worker writes; anything else was uploaded by hand. */ +const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO"; + +interface BackupMetaRow { + id: string; + user_id: string; + username: string; + type: string; + size_bytes: number; + created_at: Date; +} + export const GET = apiHandler(async () => { await requireAdmin(); annotate({ action: { name: "admin.backups.list" } }); - // Pull metadata only — `data` would balloon the response and reveal - // ciphertext we don't need on the listing page. - const backups = await prisma.dataBackup.findMany({ - include: { user: { select: { id: true, username: true } } }, - orderBy: { createdAt: "desc" }, - }); + // Metadata only, and the size read as `octet_length` in the database rather + // than by pulling every blob into this process to measure it. The listing + // needs a number, not the ciphertext, and on an instance with large records + // fetching them all was its own way of running out of memory. Base64 is + // ASCII, so the column's byte length is exactly what the wire reports. + const rows = await prisma.$queryRaw` + SELECT b.id, b.user_id, u.username, b.type, + octet_length(b.data) AS size_bytes, + b.created_at + FROM data_backups b + JOIN users u ON u.id = b.user_id + ORDER BY b.created_at DESC + `; - const rows: BackupRow[] = backups.map((b) => ({ - id: b.id, - userId: b.userId, - username: b.user.username, - type: b.type, - sizeBytes: Buffer.byteLength(b.data, "utf8"), - createdAt: b.createdAt.toISOString(), + const list: BackupRow[] = rows.map((row) => ({ + id: row.id, + userId: row.user_id, + username: row.username, + type: row.type, + sizeBytes: Number(row.size_bytes), + createdAt: row.created_at.toISOString(), })); const payload: BackupsList = { - rows, + rows: list, // Matches the retention window the backup-prune job enforces so the // backups page states the same number the worker acts on. retentionDays: 90, + schedule: summariseBackupSchedule({ + scheduledCreatedAt: rows + .filter((row) => row.type === SCHEDULED_BACKUP_TYPE) + .map((row) => row.created_at), + lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE), + now: new Date(), + }), }; return apiSuccess(payload); diff --git a/src/components/admin/__tests__/backups-section-empty.test.tsx b/src/components/admin/__tests__/backups-section-empty.test.tsx index 6dba6cd8f..6a8b2afd5 100644 --- a/src/components/admin/__tests__/backups-section-empty.test.tsx +++ b/src/components/admin/__tests__/backups-section-empty.test.tsx @@ -3,11 +3,27 @@ import { renderToStaticMarkup } from "react-dom/server"; import de from "../../../../messages/de.json"; import en from "../../../../messages/en.json"; +const healthySchedule = { + lastSuccessAt: "2026-09-01T03:00:00.000Z", + lastSuccessAgeDays: 2, + staleAfterDays: 10, + stale: false, + lastRun: { + at: "2026-09-01T03:00:12.000Z", + state: "completed" as const, + error: null, + }, + lastRunFailed: false, +}; + const mocks = vi.hoisted(() => ({ mutationOptions: [] as Array<{ onSuccess?: (data: { summary: Record }) => void; }>, toastSuccess: vi.fn(), + // Mutable so one file can render both a healthy schedule and a broken one; + // the query mock reads it on every render. + queryData: { rows: [], retentionDays: 30 } as Record, })); /** @@ -34,7 +50,7 @@ vi.mock("sonner", () => ({ vi.mock("@tanstack/react-query", () => ({ useQuery: () => ({ - data: { rows: [], retentionDays: 30 }, + data: mocks.queryData, isLoading: false, isError: false, }), @@ -69,6 +85,11 @@ import { I18nProvider } from "@/lib/i18n/context"; import { BackupsSection } from "../backups-section"; function render(locale: "en" | "de" = "en") { + mocks.queryData = { + rows: [], + retentionDays: 30, + schedule: healthySchedule, + }; return renderToStaticMarkup( @@ -129,6 +150,68 @@ describe("BackupsSection — empty state", () => { }); }); +/** + * The gap that let a weekly backup stop for a month and a half without anyone + * noticing: the page rendered whatever rows it had, and a copy from six weeks + * ago is indistinguishable from Sunday's until somebody subtracts the dates. + */ +describe("BackupsSection — schedule health", () => { + function withSchedule(schedule: Record) { + mocks.queryData = { rows: [], retentionDays: 30, schedule }; + return renderToStaticMarkup( + + + , + ); + } + + it("stays quiet while the schedule is healthy", () => { + const html = render(); + expect(html).not.toContain('data-slot="backup-schedule-notice"'); + }); + + it("says so when the newest scheduled copy has gone stale", () => { + const html = withSchedule({ + ...healthySchedule, + lastSuccessAt: "2026-07-19T03:00:00.000Z", + lastSuccessAgeDays: 46, + stale: true, + lastRun: null, + lastRunFailed: false, + }); + + expect(html).toContain('data-slot="backup-schedule-notice"'); + expect(html).toContain("The weekly backup has stopped"); + expect(html).toContain("46"); + }); + + it("names the reason when the last scheduled run failed", () => { + const html = withSchedule({ + ...healthySchedule, + lastRun: { + at: "2026-08-30T03:46:00.000Z", + state: "failed", + error: "job timed out", + }, + lastRunFailed: true, + }); + + expect(html).toContain("The last scheduled backup run failed"); + expect(html).toContain("job timed out"); + }); + + it("survives a response that predates the schedule field", () => { + mocks.queryData = { rows: [], retentionDays: 30 }; + expect(() => + renderToStaticMarkup( + + + , + ), + ).not.toThrow(); + }); +}); + describe("BackupsSection — upload outcome", () => { it("includes health profile fact revisions in the restored record total", () => { mocks.toastSuccess.mockClear(); diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index 5e2243a91..6f2c0f1d6 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -48,6 +48,7 @@ import { ListRow } from "@/components/ui/list-row"; import { useFormatters, useTranslations } from "@/lib/i18n/context"; import { queryKeys } from "@/lib/query-keys"; import type { BackupRow, BackupsList } from "@/types/backups"; +import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status"; import type { MissingBackupSection, RestoreSkipSummary, @@ -560,6 +561,71 @@ export function RestoreSkipReport({ ); } +/** + * Says out loud whether the SCHEDULE is still alive. + * + * The weekly pass on the maintainer's own instance stopped producing copies + * and nothing surfaced it for a month and a half: the table listed the rows it + * had, each with an ordinary timestamp, and a copy from six weeks ago reads + * exactly like one made on Sunday until somebody does the subtraction. Two + * independent facts are shown, because they fail at different times — the age + * of the newest scheduled copy never ages out, while the run's own outcome + * carries the reason ("job timed out") and survives only as long as pg-boss + * keeps the row. + * + * Not dismissible: it reflects live state, so the only way to clear it is to + * make a backup succeed. + */ +function ScheduleHealthNotice({ + schedule, +}: { + schedule: BackupScheduleStatus; +}) { + const { t } = useTranslations(); + const fmt = useFormatters(); + if (!schedule.stale && !schedule.lastRunFailed) return null; + + return ( +
+
+
+
+ ); +} + export function BackupsSection() { const { t } = useTranslations(); const fmt = useFormatters(); @@ -833,6 +899,10 @@ export function BackupsSection() { } /> + {data?.schedule ? ( + + ) : null} + {/* Upload card — separate from the table so admins can ingest a backup file independently of any existing rows. The visible button proxies a hidden file input so the layout stays clean diff --git a/src/lib/jobs/__tests__/backup-schedule-status.test.ts b/src/lib/jobs/__tests__/backup-schedule-status.test.ts new file mode 100644 index 000000000..2a45bee31 --- /dev/null +++ b/src/lib/jobs/__tests__/backup-schedule-status.test.ts @@ -0,0 +1,92 @@ +/** + * What the backups page has to be able to say out loud. + * + * The weekly snapshot stopped producing a new copy and nothing surfaced it: the + * page listed the rows it had, each with a perfectly ordinary timestamp, and a + * copy that was a month and a half old looked exactly like one made on Sunday. + * The two facts that would have shown it are the age of the newest scheduled + * copy and the terminal state of the last scheduled run. + */ +import { describe, expect, it } from "vitest"; + +import { + BACKUP_STALE_AFTER_DAYS, + summariseBackupSchedule, +} from "../backup-schedule-status"; + +const NOW = new Date("2026-09-03T09:00:00.000Z"); +const daysAgo = (days: number) => + new Date(NOW.getTime() - days * 24 * 60 * 60 * 1000); + +describe("summariseBackupSchedule", () => { + it("reports a fresh weekly copy as neither stale nor failed", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(2), daysAgo(9)], + lastRun: { + at: daysAgo(2).toISOString(), + state: "completed", + error: null, + }, + now: NOW, + }); + + expect(status.lastSuccessAt).toBe(daysAgo(2).toISOString()); + expect(status.lastSuccessAgeDays).toBe(2); + expect(status.stale).toBe(false); + expect(status.lastRunFailed).toBe(false); + expect(status.staleAfterDays).toBe(BACKUP_STALE_AFTER_DAYS); + }); + + it("calls a copy older than the window stale", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(46)], + lastRun: null, + now: NOW, + }); + + expect(status.lastSuccessAgeDays).toBe(46); + expect(status.stale).toBe(true); + // No run row survives that long — the age is the durable signal. + expect(status.lastRunFailed).toBe(false); + }); + + it("surfaces a failed last run even while the copy is still in window", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(3)], + lastRun: { + at: daysAgo(1).toISOString(), + state: "failed", + error: "job timed out", + }, + now: NOW, + }); + + expect(status.stale).toBe(false); + expect(status.lastRunFailed).toBe(true); + expect(status.lastRun?.error).toBe("job timed out"); + }); + + it("does not call a brand-new instance stale", () => { + const status = summariseBackupSchedule({ + scheduledCreatedAt: [], + lastRun: null, + now: NOW, + }); + + expect(status.lastSuccessAt).toBeNull(); + expect(status.lastSuccessAgeDays).toBeNull(); + expect(status.stale).toBe(false); + }); + + it("ignores manual uploads when judging the scheduled cadence", () => { + // The caller passes only WEEKLY_AUTO rows; a manual upload made today must + // not make a dead cron look alive. Proven here by the shape of the input: + // the summary knows nothing but scheduled copies. + const status = summariseBackupSchedule({ + scheduledCreatedAt: [daysAgo(30)], + lastRun: null, + now: NOW, + }); + expect(status.stale).toBe(true); + }); +}); diff --git a/src/lib/jobs/backup-schedule-status.ts b/src/lib/jobs/backup-schedule-status.ts new file mode 100644 index 000000000..d7a5ac20d --- /dev/null +++ b/src/lib/jobs/backup-schedule-status.ts @@ -0,0 +1,78 @@ +/** + * Whether the weekly backup is actually still happening, in two facts. + * + * The pass stopped producing a new copy and nothing said so for a month and a + * half. Everything that could have told the operator was either invisible or + * short-lived: the handler's own error never ran (the job was killed, not + * thrown), `recordError()` increments an unnamed counter, and the admin status + * page's failing-queue panel only looks back 72 hours — which for a weekly + * queue means a permanent failure is on screen three days in seven. The backups + * page itself showed a perfectly ordinary row with a perfectly ordinary + * timestamp, because a 46-day-old copy looks exactly like a fresh one until you + * read the date. + * + * So the surface carries both legs, and neither is derived from the other: + * + * - the AGE of the newest scheduled copy, which never ages out of any + * retention window and would have caught this on the second missed Sunday; + * - the terminal state of the last scheduled RUN, which is the only place the + * reason ("job timed out") is written down at all. + * + * Manual uploads are excluded by the shape of the input: the caller passes only + * the scheduled rows, so an admin's ad-hoc upload cannot make a dead cron look + * alive. + */ +import type { LastQueueRun } from "@/lib/jobs/job-failures"; + +/** + * How old the newest scheduled copy may get before the page says so. The cron + * is weekly, so seven days is normal and eight is a rounding error; ten means + * at least one Sunday produced nothing. + */ +export const BACKUP_STALE_AFTER_DAYS = 10; + +const DAY_MS = 24 * 60 * 60 * 1000; + +export interface BackupScheduleStatus { + /** ISO instant of the newest scheduled copy, or null when there is none. */ + lastSuccessAt: string | null; + /** Whole days since that copy was written. */ + lastSuccessAgeDays: number | null; + /** The threshold behind `stale`, so the UI can name it. */ + staleAfterDays: number; + /** A scheduled copy exists and is older than the threshold. */ + stale: boolean; + /** The last scheduled run pg-boss still remembers. */ + lastRun: LastQueueRun | null; + /** That run ended `failed` or `cancelled`. */ + lastRunFailed: boolean; +} + +export function summariseBackupSchedule(args: { + /** `createdAt` of every SCHEDULED backup row; manual uploads excluded. */ + scheduledCreatedAt: readonly Date[]; + lastRun: LastQueueRun | null; + now: Date; +}): BackupScheduleStatus { + const newest = args.scheduledCreatedAt.reduce( + (best, at) => (best === null || at > best ? at : best), + null, + ); + const ageDays = + newest === null + ? null + : Math.floor((args.now.getTime() - newest.getTime()) / DAY_MS); + + return { + lastSuccessAt: newest?.toISOString() ?? null, + lastSuccessAgeDays: ageDays, + staleAfterDays: BACKUP_STALE_AFTER_DAYS, + // An instance that has never produced a scheduled copy is not stale, it is + // new — the page's empty state already says there is nothing there, and + // calling a fresh install "overdue" would train the operator to ignore the + // warning that matters. + stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS, + lastRun: args.lastRun, + lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed", + }; +} diff --git a/src/lib/jobs/job-failures.ts b/src/lib/jobs/job-failures.ts index 48403da09..d846c88be 100644 --- a/src/lib/jobs/job-failures.ts +++ b/src/lib/jobs/job-failures.ts @@ -98,7 +98,11 @@ export async function readFailingQueues( max(completed_on) AS last_failed_at, ( array_agg( - COALESCE(output->>'message', output::text) + COALESCE( + output->>'message', + output->'value'->>'message', + output::text + ) ORDER BY completed_on DESC NULLS LAST ) )[1] AS last_error @@ -124,6 +128,75 @@ export async function readFailingQueues( } } +/** The terminal state pg-boss recorded for the newest run of one queue. */ +export interface LastQueueRun { + /** ISO instant the run reached its terminal state. */ + at: string; + /** `failed` covers both a thrown handler and an expired (killed) job. */ + state: "completed" | "failed" | "cancelled"; + /** The failure text, redacted and truncated. Null when the run succeeded. */ + error: string | null; +} + +/** + * The newest terminal run of one queue, whatever its outcome. + * + * `readFailingQueues` answers "what broke recently" and is deliberately scoped + * to a 72-hour window, which is the right shape for a status page watching + * thirty queues that tick by the minute. It is the wrong shape for a WEEKLY + * pass: a Sunday-night failure has fallen out of the window by Wednesday, so a + * job that fails every single week is visible for three days in seven and + * invisible for four. This reader takes the newest run instead of a window, so + * a weekly queue's last outcome is readable right up until the next one. + * + * Bounded by pg-boss's own retention: terminal rows live `deletion_seconds` + * (seven days by default) past completion, so `null` here means "no run row + * survives", which is not the same as "the pass never ran". Callers pair it + * with a fact that does not age out — for backups, the age of the newest + * stored copy. + */ +export async function readLastQueueRun( + queue: string, +): Promise { + try { + const rows = await prisma.$queryRaw< + Array<{ state: string; completed_on: Date | null; output: string | null }> + >` + SELECT + state, + completed_on, + -- pg-boss wraps a thrown value: an expired job's row reads + -- {"value":{"message":"job timed out"}}, so a plain ->>'message' + -- misses it and the operator gets raw JSON instead of the sentence. + COALESCE( + output->>'message', + output->'value'->>'message', + output::text + ) AS output + FROM pgboss.job + WHERE name = ${queue} + AND state IN ('completed', 'failed', 'cancelled') + AND completed_on IS NOT NULL + ORDER BY completed_on DESC + LIMIT 1 + `; + const row = rows[0]; + if (!row || row.completed_on === null) return null; + const state = + row.state === "failed" || row.state === "cancelled" + ? row.state + : "completed"; + return { + at: row.completed_on.toISOString(), + state, + error: state === "completed" ? null : presentError(row.output), + }; + } catch { + // No `pgboss` schema (web-only deployment) or no permission to read it. + return null; + } +} + export interface QueueFailureForUser { /** ISO instant the user's newest job on this queue gave up. */ lastFailedAt: string; diff --git a/src/types/backups.ts b/src/types/backups.ts index 75b0cc66c..1931628ea 100644 --- a/src/types/backups.ts +++ b/src/types/backups.ts @@ -9,6 +9,7 @@ * both sides a route-handler-independent shared type home; the route * keeps owning the HTTP contract, the component keeps owning the UI. */ +import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status"; export interface BackupRow { id: string; @@ -25,6 +26,12 @@ export interface BackupRow { export interface BackupsList { rows: BackupRow[]; + /** + * Whether the weekly schedule is still producing copies, and how the last + * scheduled run ended. A row's own timestamp cannot answer that — a copy + * made six weeks ago and one made on Sunday look identical in a table. + */ + schedule: BackupScheduleStatus; /** * Soft retention hint — the worker is configured for weekly backups * (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and diff --git a/tests/integration/admin-backups-audit.test.ts b/tests/integration/admin-backups-audit.test.ts index 989399700..fe7755fb4 100644 --- a/tests/integration/admin-backups-audit.test.ts +++ b/tests/integration/admin-backups-audit.test.ts @@ -197,3 +197,69 @@ describe("backup endpoints — AuditLog contract", () => { expect(restore?.details ?? "").toContain(seeded.id); }); }); + +/** + * The listing route is what an operator looks at to answer "are my backups + * still being made". It could not answer that: it returned rows and a + * retention hint, and a copy from six weeks ago carries a timestamp exactly + * like Sunday's. It now carries the schedule's own health, and the size comes + * from `octet_length` in the database rather than from pulling every blob into + * the web process to measure it. + */ +describe("GET /api/admin/backups — schedule health", () => { + it("calls a stalled weekly schedule out, and a fresh manual upload does not hide it", async () => { + const prisma = getPrismaClient(); + const admin = await seedAdminSession(); + const blob = encrypt(JSON.stringify(payloadFor(admin.id))); + + const stalled = new Date(Date.now() - 46 * 24 * 60 * 60 * 1000); + await prisma.dataBackup.create({ + data: { + userId: admin.id, + type: "WEEKLY_AUTO", + data: blob, + createdAt: stalled, + }, + }); + // Uploaded by hand today. It must not make the dead cron look alive. + await prisma.dataBackup.create({ + data: { userId: admin.id, type: "MANUAL_UPLOAD_1", data: blob }, + }); + + // The listing handler declares no parameters; `apiHandler` reads the + // request defensively out of its arguments, so calling it bare is what a + // zero-argument handler supports. + const { GET } = await import("@/app/api/admin/backups/route"); + const res = await GET(); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { + rows: Array<{ type: string; sizeBytes: number; username: string }>; + schedule: { + stale: boolean; + lastSuccessAgeDays: number | null; + lastRun: unknown; + lastRunFailed: boolean; + }; + }; + }; + + expect(body.data.rows).toHaveLength(2); + // Sizes are read in SQL now; a wrong column or a wrong cast shows up here + // as a zero or a NaN rather than as a plausible-looking number. + expect(body.data.rows.every((row) => row.sizeBytes === blob.length)).toBe( + true, + ); + expect(body.data.rows.every((row) => row.username === admin.username)).toBe( + true, + ); + + expect(body.data.schedule.lastSuccessAgeDays).toBe(46); + expect(body.data.schedule.stale).toBe(true); + // No `pgboss` schema in the test database — the run reader fails soft to + // null rather than taking the whole page down, which is the arm a web-only + // deployment lives on permanently. + expect(body.data.schedule.lastRun).toBeNull(); + expect(body.data.schedule.lastRunFailed).toBe(false); + }); +});