diff --git a/docs/ops/backup-restore.md b/docs/ops/backup-restore.md index 4abb38e46..4130a2277 100644 --- a/docs/ops/backup-restore.md +++ b/docs/ops/backup-restore.md @@ -225,8 +225,16 @@ failed` each time, which looks like weather. host last put that account's encrypted copy in the bucket, how big it was, and how that compares with the nightly schedule. **Fresh** is inside one run, **due** means one run produced nothing for that account, **stale** means two -did not, and **never** means no object has ever landed. On a host without -`BACKUP_S3_*` the card says off-host backup is not configured and stops there. +did not, **never** means a run walked the account and no object has ever +landed for it, and **no record yet** means no run has recorded the account at +all. On a host without `BACKUP_S3_*` the card says off-host backup is not +configured and stops there. + +The ledger starts empty on the release that adds it, so on a host that has +been uploading nightly for months every account reads **no record yet** until +the first run after the upgrade. That is the ledger having no history, not the +bucket being empty — which is why it is a separate word from **never**, and +why the row says so rather than claiming nothing has ever reached the bucket. Each verdict carries six hours of slack on top of the schedule's own 24, and that slack is load-bearing on two nights. The cron runs at 02:30 Europe/Berlin, diff --git a/e2e/a11y.spec.ts b/e2e/a11y.spec.ts index 3e4509716..63a0d5fb0 100644 --- a/e2e/a11y.spec.ts +++ b/e2e/a11y.spec.ts @@ -717,6 +717,7 @@ async function installA11yMocks(page: Page) { { userId: "a11y-user", username: "a11y-backup-user", + lastAttemptAt: "2026-07-20T03:00:00.000Z", lastSuccessAt: "2026-07-20T03:00:00.000Z", sizeBytes: 4096, ageHours: 3, @@ -725,6 +726,7 @@ async function installA11yMocks(page: Page) { { userId: "a11y-user-2", username: "a11y-backup-user-2", + lastAttemptAt: "2026-07-14T03:00:00.000Z", lastSuccessAt: "2026-07-14T03:00:00.000Z", sizeBytes: 2048, ageHours: 150, @@ -733,11 +735,21 @@ async function installA11yMocks(page: Page) { { userId: "a11y-user-3", username: "a11y-backup-user-3", + lastAttemptAt: "2026-07-20T03:00:00.000Z", lastSuccessAt: null, sizeBytes: null, ageHours: null, freshness: "never", }, + { + userId: "a11y-user-4", + username: "a11y-backup-user-4", + lastAttemptAt: null, + lastSuccessAt: null, + sizeBytes: null, + ageHours: null, + freshness: "unknown", + }, ], }, }), diff --git a/messages/de.json b/messages/de.json index d2b1c9cbb..581a7792e 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7026,7 +7026,9 @@ "stateDue": "Fällig", "stateStale": "Veraltet", "stateNever": "Nie", - "neverDetail": "Für dieses Konto ist noch nie eine Kopie im Bucket angekommen.", + "stateUnknown": "Noch kein Eintrag", + "neverDetail": "Für dieses Konto ist nichts angekommen. Zuletzt geprüft {when}.", + "unknownDetail": "Noch kein nächtlicher Lauf hat dieses Konto erfasst.", "age": "vor {hours} h", "scheduleNote": "Der Upload läuft alle {hours} h. Frisch heißt innerhalb eines Laufs, fällig heißt ein Lauf blieb ohne Ergebnis, veraltet heißt zwei." } diff --git a/messages/en.json b/messages/en.json index 2f4a06eb6..ba0b8ac2b 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7026,7 +7026,9 @@ "stateDue": "Due", "stateStale": "Stale", "stateNever": "Never", - "neverDetail": "No copy has ever reached the bucket for this account.", + "stateUnknown": "No record yet", + "neverDetail": "Nothing has landed for this account. Last checked {when}.", + "unknownDetail": "No nightly run has recorded this account yet.", "age": "{hours} h ago", "scheduleNote": "The upload runs every {hours} h. Fresh means inside one run, due means one run produced nothing, stale means two did not." } diff --git a/messages/es.json b/messages/es.json index 28aab6313..ffb0affc5 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7026,7 +7026,9 @@ "stateDue": "Pendiente", "stateStale": "Obsoleta", "stateNever": "Nunca", - "neverDetail": "Para esta cuenta nunca ha llegado una copia al bucket.", + "stateUnknown": "Sin registro aún", + "neverDetail": "No ha llegado nada para esta cuenta. Última comprobación {when}.", + "unknownDetail": "Ninguna ejecución nocturna ha registrado esta cuenta todavía.", "age": "hace {hours} h", "scheduleNote": "La subida se ejecuta cada {hours} h. Reciente significa dentro de una ejecución, pendiente que una no produjo nada, obsoleta que fueron dos." } diff --git a/messages/fr.json b/messages/fr.json index 2e40628d5..2de62005f 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7026,7 +7026,9 @@ "stateDue": "Attendue", "stateStale": "Périmée", "stateNever": "Jamais", - "neverDetail": "Aucune copie n'est jamais arrivée dans le bucket pour ce compte.", + "stateUnknown": "Pas encore d'enregistrement", + "neverDetail": "Rien n'est arrivé pour ce compte. Dernière vérification {when}.", + "unknownDetail": "Aucun cycle nocturne n'a encore enregistré ce compte.", "age": "il y a {hours} h", "scheduleNote": "L'envoi tourne toutes les {hours} h. Récente signifie dans un cycle, attendue qu'un cycle n'a rien produit, périmée que deux n'ont rien produit." } diff --git a/messages/it.json b/messages/it.json index d883b2f5d..ddefd908c 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7026,7 +7026,9 @@ "stateDue": "Attesa", "stateStale": "Obsoleta", "stateNever": "Mai", - "neverDetail": "Per questo account nessuna copia è mai arrivata nel bucket.", + "stateUnknown": "Nessuna registrazione", + "neverDetail": "Per questo account non è arrivato nulla. Ultimo controllo {when}.", + "unknownDetail": "Nessun giro notturno ha ancora registrato questo account.", "age": "{hours} h fa", "scheduleNote": "Il caricamento gira ogni {hours} h. Recente significa entro un giro, attesa che un giro non ha prodotto nulla, obsoleta che non lo hanno fatto due." } diff --git a/messages/ko.json b/messages/ko.json index 549935a1a..7e66cb7e3 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7006,7 +7006,9 @@ "stateDue": "지연", "stateStale": "오래됨", "stateNever": "없음", - "neverDetail": "이 계정의 사본이 버킷에 도착한 적이 없습니다.", + "stateUnknown": "기록 없음", + "neverDetail": "이 계정으로 도착한 것이 없습니다. 마지막 확인 {when}.", + "unknownDetail": "아직 어떤 야간 실행도 이 계정을 기록하지 않았습니다.", "age": "{hours}시간 전", "scheduleNote": "업로드는 {hours}시간마다 실행됩니다. 최신은 한 번의 실행 이내, 지연은 한 번이 아무것도 만들지 못한 상태, 오래됨은 두 번이 그런 상태입니다." } diff --git a/messages/pl.json b/messages/pl.json index 6bd2d6b15..a8af51993 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7026,7 +7026,9 @@ "stateDue": "Zaległa", "stateStale": "Przeterminowana", "stateNever": "Nigdy", - "neverDetail": "Dla tego konta żadna kopia nigdy nie trafiła do bucketu.", + "stateUnknown": "Brak wpisu", + "neverDetail": "Dla tego konta nic nie dotarło. Ostatnie sprawdzenie {when}.", + "unknownDetail": "Żaden nocny przebieg nie zarejestrował jeszcze tego konta.", "age": "{hours} h temu", "scheduleNote": "Wysyłka działa co {hours} h. Świeża znaczy w obrębie jednego przebiegu, zaległa że jeden nic nie dał, przeterminowana że dwa." } diff --git a/prisma/migrations/0336_offhost_backup_ledger/migration.sql b/prisma/migrations/0336_offhost_backup_ledger/migration.sql index dd0ad6f39..e0a66eaa8 100644 --- a/prisma/migrations/0336_offhost_backup_ledger/migration.sql +++ b/prisma/migrations/0336_offhost_backup_ledger/migration.sql @@ -4,11 +4,16 @@ -- without the page ever talking to the bucket. CREATE TABLE "offhost_backup_state" ( "user_id" TEXT NOT NULL, - "last_success_at" TIMESTAMP(3) NOT NULL, + -- Written on every account the run walks, success or not. A row's mere + -- existence is what lets the console tell "no run has reached this + -- account yet" from "a run reached it and nothing landed". + "last_attempt_at" TIMESTAMP(3) NOT NULL, + -- NULL until an object lands for this account. + "last_success_at" TIMESTAMP(3), -- BIGINT, not INTEGER: a single object may occupy 8 MiB x 10 000 multipart -- parts, so the uploader permits 80 GB and a 32-bit column would refuse -- every account past 2.1 GB. - "size_bytes" BIGINT NOT NULL, + "size_bytes" BIGINT, CONSTRAINT "offhost_backup_state_pkey" PRIMARY KEY ("user_id") ); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 540af229c..35d9336e6 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -4953,17 +4953,28 @@ model DataBackup { model OffhostBackupState { userId String @id @map("user_id") - /// When the newest object for this account finished uploading. Consumer: - /// `classifyOffhostBackup()` behind `GET /api/admin/backups`. - lastSuccessAt DateTime @map("last_success_at") + /// When the last nightly run walked this account, whether or not an object + /// came of it. It is what separates "no run has reached this account yet" + /// from "a run reached it and put nothing there" — two facts that look + /// identical in a ledger keyed only on success, and the first of which is + /// what every account reads on the day this table ships. Consumer: + /// `classifyOffhostBackup()` behind `GET /api/admin/backups`, and the + /// card's "last checked" line. + lastAttemptAt DateTime @map("last_attempt_at") + + /// When the newest object for this account finished uploading. NULL means + /// a run has looked at this account and nothing has ever landed for it. + /// Consumer: `classifyOffhostBackup()` behind `GET /api/admin/backups`. + lastSuccessAt DateTime? @map("last_success_at") /// Size of that object in bytes — the same number the run reports as its /// largest, per account. `BigInt` because the uploader permits an object of /// up to 8 MiB × 10 000 parts = 80 GB and a 32-bit column stops at 2.1 GB: /// the biggest account on the host, the one an operator cares most about, /// is exactly the one whose row would refuse to be written. + /// NULL alongside a NULL `lastSuccessAt`. /// Consumer: the off-host card's size column. - sizeBytes BigInt @map("size_bytes") + sizeBytes BigInt? @map("size_bytes") user User @relation(fields: [userId], references: [id], onDelete: Cascade) diff --git a/src/app/api/admin/backups/__tests__/route.test.ts b/src/app/api/admin/backups/__tests__/route.test.ts index c1dcd9a11..e182e8e17 100644 --- a/src/app/api/admin/backups/__tests__/route.test.ts +++ b/src/app/api/admin/backups/__tests__/route.test.ts @@ -63,6 +63,7 @@ describe("GET /api/admin/backups — off-host freshness", () => { id: "u1", username: "account-one", offhostBackupState: { + lastAttemptAt: new Date(now - 3 * HOUR), lastSuccessAt: new Date(now - 3 * HOUR), // The column is BigInt — an object may be 80 GB — and the wire type // is a plain number. @@ -73,6 +74,7 @@ describe("GET /api/admin/backups — off-host freshness", () => { id: "u2", username: "account-two", offhostBackupState: { + lastAttemptAt: new Date(now - 31 * HOUR), lastSuccessAt: new Date(now - 31 * HOUR), sizeBytes: BigInt(8192), }, @@ -81,11 +83,24 @@ describe("GET /api/admin/backups — off-host freshness", () => { id: "u3", username: "account-three", offhostBackupState: { + lastAttemptAt: new Date(now - 100 * HOUR), lastSuccessAt: new Date(now - 100 * HOUR), sizeBytes: BigInt(128), }, }, - { id: "u4", username: "account-four", offhostBackupState: null }, + { + id: "u4", + username: "account-four", + // Walked by a run, nothing landed. + offhostBackupState: { + lastAttemptAt: new Date(now - 2 * HOUR), + lastSuccessAt: null, + sizeBytes: null, + }, + }, + // No run has recorded this one at all — the state of every account on + // the morning the ledger ships. + { id: "u5", username: "account-five", offhostBackupState: null }, ]); const body = await read(); @@ -99,12 +114,17 @@ describe("GET /api/admin/backups — off-host freshness", () => { ["account-two", "due"], ["account-three", "stale"], ["account-four", "never"], + ["account-five", "unknown"], ]); expect(body.offhost.rows[0]?.sizeBytes).toBe(4096); expect(typeof body.offhost.rows[0]?.sizeBytes).toBe("number"); expect(body.offhost.rows[0]?.ageHours).toBe(3); expect(body.offhost.rows[3]?.lastSuccessAt).toBeNull(); expect(body.offhost.rows[3]?.sizeBytes).toBeNull(); + expect(body.offhost.rows[3]?.lastAttemptAt).not.toBeNull(); + // `unknown` is the honest one: the ledger has no history for this + // account, which is not the same claim as "the bucket is empty". + expect(body.offhost.rows[4]?.lastAttemptAt).toBeNull(); }); it("says so and reads no account when off-host backup is not configured", async () => { @@ -128,6 +148,7 @@ describe("GET /api/admin/backups — off-host freshness", () => { id: "u1", username: "account-one", offhostBackupState: { + lastAttemptAt: new Date(), lastSuccessAt: new Date(), sizeBytes: BigInt(1), }, @@ -139,6 +160,7 @@ describe("GET /api/admin/backups — off-host freshness", () => { expect(Object.keys(body.offhost.rows[0] ?? {}).sort()).toEqual([ "ageHours", "freshness", + "lastAttemptAt", "lastSuccessAt", "sizeBytes", "userId", diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index bebfffa5b..b8f38b14e 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -95,7 +95,11 @@ export const GET = apiHandler(async () => { id: true, username: true, offhostBackupState: { - select: { lastSuccessAt: true, sizeBytes: true }, + select: { + lastAttemptAt: true, + lastSuccessAt: true, + sizeBytes: true, + }, }, }, orderBy: { username: "asc" }, @@ -105,17 +109,22 @@ export const GET = apiHandler(async () => { const offhostRows: OffhostAccountRow[] = offhostAccounts.map((account) => { const state = account.offhostBackupState; const verdict = classifyOffhostBackup({ + lastAttemptAt: state?.lastAttemptAt ?? null, lastSuccessAt: state?.lastSuccessAt ?? null, now, }); return { userId: account.id, username: account.username, - lastSuccessAt: state?.lastSuccessAt.toISOString() ?? null, + lastAttemptAt: state?.lastAttemptAt.toISOString() ?? null, + lastSuccessAt: state?.lastSuccessAt?.toISOString() ?? null, // BigInt column -> number on the wire. The value is bytes of one // object, capped by the uploader at 80 GB, so it is nowhere near // Number.MAX_SAFE_INTEGER and JSON has no BigInt. - sizeBytes: state ? Number(state.sizeBytes) : null, + sizeBytes: + state?.sizeBytes === null || state?.sizeBytes === undefined + ? null + : Number(state.sizeBytes), ageHours: verdict.ageHours, freshness: verdict.freshness, }; diff --git a/src/components/admin/__tests__/offhost-backups-section.test.tsx b/src/components/admin/__tests__/offhost-backups-section.test.tsx index 7d7847414..d3dd46985 100644 --- a/src/components/admin/__tests__/offhost-backups-section.test.tsx +++ b/src/components/admin/__tests__/offhost-backups-section.test.tsx @@ -59,6 +59,7 @@ describe("", () => { { userId: "u1", username: "account-one", + lastAttemptAt: "2026-09-10T06:00:00.000Z", lastSuccessAt: "2026-09-10T06:00:00.000Z", sizeBytes: 4096, ageHours: 3, @@ -67,6 +68,7 @@ describe("", () => { { userId: "u2", username: "account-two", + lastAttemptAt: "2026-09-06T06:00:00.000Z", lastSuccessAt: "2026-09-06T06:00:00.000Z", sizeBytes: 2048, ageHours: 99, @@ -75,20 +77,35 @@ describe("", () => { { userId: "u3", username: "account-three", + lastAttemptAt: "2026-09-10T02:30:00.000Z", lastSuccessAt: null, sizeBytes: null, ageHours: null, freshness: "never", }, + { + userId: "u4", + username: "account-four", + lastAttemptAt: null, + lastSuccessAt: null, + sizeBytes: null, + ageHours: null, + freshness: "unknown", + }, ], }); expect(html).toContain('data-offhost-freshness="fresh"'); expect(html).toContain('data-offhost-freshness="stale"'); expect(html).toContain('data-offhost-freshness="never"'); + expect(html).toContain('data-offhost-freshness="unknown"'); expect(html).toContain("account-two"); - // The account with no object says so instead of showing a blank cell. - expect(html).toContain("No copy has ever reached the bucket"); + // The account a run walked and found nothing for says that, with the + // instant it was checked. + expect(html).toContain("Nothing has landed for this account"); + // The account no run has recorded says THAT — not "never", which would + // claim the bucket is empty on the strength of an empty ledger. + expect(html).toContain("No nightly run has recorded this account yet"); // Never the bucket's own coordinates. expect(html).not.toMatch(/secret|access[- ]key/i); }); diff --git a/src/components/admin/offhost-backups-section.tsx b/src/components/admin/offhost-backups-section.tsx index 1084a2510..cf3de6837 100644 --- a/src/components/admin/offhost-backups-section.tsx +++ b/src/components/admin/offhost-backups-section.tsx @@ -34,16 +34,17 @@ import type { BackupsList, OffhostAccountRow } from "@/types/backups"; import { formatBytes } from "./backups-section"; /** - * Status wash per verdict. `never` is deliberately neutral rather than - * alarming: a host that turned the off-host job on this afternoon has not - * failed at anything yet, and painting every row red on day one is how an - * operator learns to stop reading the colour. + * Status wash per verdict. `never` and `unknown` are deliberately neutral + * rather than alarming: a host that turned the off-host job on this afternoon + * has not failed at anything yet, and painting every row red on day one is how + * an operator learns to stop reading the colour. */ const FRESHNESS_STYLE: Record = { fresh: "border-success/30 bg-success/10 text-success", due: "border-warning/30 bg-warning/10 text-warning", stale: "border-destructive/30 bg-destructive/10 text-destructive", never: "border-border bg-muted text-muted-foreground", + unknown: "border-border bg-muted text-muted-foreground", }; function freshnessLabel( @@ -61,6 +62,8 @@ function freshnessLabel( return t("admin.section.backups.offhost.stateStale"); case "never": return t("admin.section.backups.offhost.stateNever"); + case "unknown": + return t("admin.section.backups.offhost.stateUnknown"); } } @@ -87,14 +90,22 @@ function AccountRow({ row }: { row: OffhostAccountRow }) {

- {row.lastSuccessAt === null || row.sizeBytes === null - ? t("admin.section.backups.offhost.neverDetail") - : `${fmt.dateTime(row.lastSuccessAt)} · ${formatBytes( - row.sizeBytes, - fmt, - )} · ${t("admin.section.backups.offhost.age", { - hours: row.ageHours ?? 0, - })}`} + {row.lastAttemptAt === null + ? // No run has recorded this account. Saying "never" here would + // claim the bucket is empty when what is empty is the ledger, + // which is exactly what every account looks like between the + // upgrade that adds it and the first nightly run after that. + t("admin.section.backups.offhost.unknownDetail") + : row.lastSuccessAt === null || row.sizeBytes === null + ? t("admin.section.backups.offhost.neverDetail", { + when: fmt.dateTime(row.lastAttemptAt), + }) + : `${fmt.dateTime(row.lastSuccessAt)} · ${formatBytes( + row.sizeBytes, + fmt, + )} · ${t("admin.section.backups.offhost.age", { + hours: row.ageHours ?? 0, + })}`}

diff --git a/src/lib/jobs/__tests__/offhost-backup-freshness.test.ts b/src/lib/jobs/__tests__/offhost-backup-freshness.test.ts index 77be636b1..7de8e0fa4 100644 --- a/src/lib/jobs/__tests__/offhost-backup-freshness.test.ts +++ b/src/lib/jobs/__tests__/offhost-backup-freshness.test.ts @@ -16,20 +16,40 @@ function ago(ms: number): Date { } describe("classifyOffhostBackup", () => { - it("says never when the worker has put nothing there for this account", () => { - expect(classifyOffhostBackup({ lastSuccessAt: null, now: NOW })).toEqual({ - freshness: "never", - ageHours: null, - }); + it("says unknown, not never, when no run has recorded this account", () => { + // The ledger starts empty. On the first morning after the upgrade every + // account on a perfectly healthy host looks like this, and `never` would + // assert something about the bucket the host has not looked at. + expect( + classifyOffhostBackup({ + lastAttemptAt: null, + lastSuccessAt: null, + now: NOW, + }), + ).toEqual({ freshness: "unknown", ageHours: null }); + }); + + it("says never once a run has walked the account and put nothing there", () => { + expect( + classifyOffhostBackup({ + lastAttemptAt: ago(HOUR), + lastSuccessAt: null, + now: NOW, + }), + ).toEqual({ freshness: "never", ageHours: null }); }); it("is fresh right up to one period plus the grace and due one millisecond past it", () => { expect( - classifyOffhostBackup({ lastSuccessAt: ago(PERIOD + GRACE), now: NOW }) - .freshness, + classifyOffhostBackup({ + lastAttemptAt: ago(0), + lastSuccessAt: ago(PERIOD + GRACE), + now: NOW, + }).freshness, ).toBe("fresh"); expect( classifyOffhostBackup({ + lastAttemptAt: ago(0), lastSuccessAt: ago(PERIOD + GRACE + 1), now: NOW, }).freshness, @@ -39,12 +59,14 @@ describe("classifyOffhostBackup", () => { it("is due right up to two periods plus the grace and stale one millisecond past it", () => { expect( classifyOffhostBackup({ + lastAttemptAt: ago(0), lastSuccessAt: ago(PERIOD * 2 + GRACE), now: NOW, }).freshness, ).toBe("due"); expect( classifyOffhostBackup({ + lastAttemptAt: ago(0), lastSuccessAt: ago(PERIOD * 2 + GRACE + 1), now: NOW, }).freshness, @@ -66,6 +88,7 @@ describe("classifyOffhostBackup", () => { // on every host, with nothing wrong anywhere. expect( classifyOffhostBackup({ + lastAttemptAt: ago(0), lastSuccessAt: beforeTheChange, now: afterTheChange, }).freshness, @@ -76,14 +99,21 @@ describe("classifyOffhostBackup", () => { // 31 hours: past one period even with six hours of slack, so a run that // produced nothing for this account still shows. expect( - classifyOffhostBackup({ lastSuccessAt: ago(31 * HOUR), now: NOW }) - .freshness, + classifyOffhostBackup({ + lastAttemptAt: ago(0), + lastSuccessAt: ago(31 * HOUR), + now: NOW, + }).freshness, ).toBe("due"); }); it("reports whole hours of age", () => { expect( - classifyOffhostBackup({ lastSuccessAt: ago(90 * 60_000), now: NOW }), + classifyOffhostBackup({ + lastAttemptAt: ago(0), + lastSuccessAt: ago(90 * 60_000), + now: NOW, + }), ).toEqual({ freshness: "fresh", ageHours: 1 }); }); @@ -93,10 +123,15 @@ describe("classifyOffhostBackup", () => { // as zero so the thresholds under test are the caller's period alone. const twelveHours = ago(12 * HOUR); expect( - classifyOffhostBackup({ lastSuccessAt: twelveHours, now: NOW }).freshness, + classifyOffhostBackup({ + lastAttemptAt: ago(0), + lastSuccessAt: twelveHours, + now: NOW, + }).freshness, ).toBe("fresh"); expect( classifyOffhostBackup({ + lastAttemptAt: ago(0), lastSuccessAt: twelveHours, now: NOW, periodHours: 6, @@ -105,6 +140,7 @@ describe("classifyOffhostBackup", () => { ).toBe("due"); expect( classifyOffhostBackup({ + lastAttemptAt: ago(0), lastSuccessAt: twelveHours, now: NOW, periodHours: 5, @@ -116,6 +152,7 @@ describe("classifyOffhostBackup", () => { it("does not call a copy stale because the host clock ran backwards", () => { expect( classifyOffhostBackup({ + lastAttemptAt: ago(0), lastSuccessAt: new Date(NOW.getTime() + 2 * HOUR), now: NOW, }), diff --git a/src/lib/jobs/__tests__/offhost-backup.test.ts b/src/lib/jobs/__tests__/offhost-backup.test.ts index 508e28bab..a9d07113c 100644 --- a/src/lib/jobs/__tests__/offhost-backup.test.ts +++ b/src/lib/jobs/__tests__/offhost-backup.test.ts @@ -334,16 +334,24 @@ describe("runOffhostBackup", () => { expect(report.uploaded).toBe(1); expect(report.failed).toBe(1); + // Both accounts are recorded as walked — that is what separates "a run + // looked and found nothing" from "no run has looked at this account" — + // but only the one whose object landed carries a success. + type Upsert = { + where: { userId: string }; + update: Record; + create: Record; + }; + const calls = upsert.mock.calls.map((call) => call[0] as Upsert); + expect(calls.map((call) => call.where.userId)).toEqual(["u1", "u2"]); + expect(calls[0]?.update).toHaveProperty("lastSuccessAt"); // The guarantee an operator leans on: an account whose upload failed keeps - // whatever `lastSuccessAt` it already had. A failed upload writes no - // success into the ledger, so yesterday's good copy is still what the - // console reports — the run does not overwrite it with today's failure. - expect(upsert).toHaveBeenCalledTimes(1); - expect( - upsert.mock.calls.map( - (call) => (call[0] as { where: { userId: string } }).where.userId, - ), - ).toEqual(["u1"]); + // whatever `lastSuccessAt` it already had. The failed arm names only the + // walk instant, so yesterday's good copy is still what the console + // reports — the run does not overwrite it with today's failure. + expect(calls[1]?.update).not.toHaveProperty("lastSuccessAt"); + expect(calls[1]?.update).not.toHaveProperty("sizeBytes"); + expect(calls[1]?.create).not.toHaveProperty("lastSuccessAt"); }); it("still reports a landed object as uploaded when the ledger write fails", async () => { @@ -368,6 +376,7 @@ describe("runOffhostBackup", () => { expect(report.failed).toBe(0); expect(report.failures).toEqual([]); expect(s3.store.has("2026-05-08/user-u1.json.enc")).toBe(true); + expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(1); }); it("refuses an account whose object outgrows one multipart upload", async () => { diff --git a/src/lib/jobs/offhost-backup-freshness.ts b/src/lib/jobs/offhost-backup-freshness.ts index 869b5c1b0..29d2f5a3d 100644 --- a/src/lib/jobs/offhost-backup-freshness.ts +++ b/src/lib/jobs/offhost-backup-freshness.ts @@ -14,10 +14,16 @@ * same whether or not the app process holds a read grant on the bucket, and a * listing of somebody's backups never crosses the network to show a page. * - * Four states, because the run is nightly and one missed night is not the same - * event as four: + * Five states, because the run is nightly and one missed night is not the same + * event as four — and because "we have not looked yet" is not a verdict: * - * never — the worker has never put an object there for this account. + * unknown — no nightly run has recorded this account. The ledger starts + * empty, so this is what every account on a perfectly healthy host + * reads between the upgrade that adds the table and the first run + * after it. Saying "never" there would assert something the host + * cannot know. + * never — a run walked this account and put no object there, and no earlier + * run ever did either. * fresh — inside one schedule period plus the grace below. This is what * every account looks like on a host where the cron is doing its * job. @@ -54,7 +60,8 @@ export const OFFHOST_BACKUP_PERIOD_HOURS = 24; */ export const OFFHOST_BACKUP_GRACE_HOURS = 6; -export type OffhostBackupFreshness = "never" | "fresh" | "due" | "stale"; +export type OffhostBackupFreshness = + "unknown" | "never" | "fresh" | "due" | "stale"; export interface OffhostBackupVerdict { freshness: OffhostBackupFreshness; @@ -63,6 +70,11 @@ export interface OffhostBackupVerdict { } export function classifyOffhostBackup(args: { + /** + * When a run last walked this account, or null when none has. Null is the + * only thing that separates `unknown` from `never`. + */ + lastAttemptAt: Date | null; lastSuccessAt: Date | null; now: Date; /** Defaults to the nightly schedule. Tests pass their own. */ @@ -70,7 +82,8 @@ export function classifyOffhostBackup(args: { /** Defaults to `OFFHOST_BACKUP_GRACE_HOURS`. Tests pass their own. */ graceHours?: number; }): OffhostBackupVerdict { - const { lastSuccessAt, now } = args; + const { lastAttemptAt, lastSuccessAt, now } = args; + if (lastAttemptAt === null) return { freshness: "unknown", ageHours: null }; if (lastSuccessAt === null) return { freshness: "never", ageHours: null }; const periodMs = diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index e46e5cc96..3627b1d2b 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -615,32 +615,49 @@ export async function runOffhostBackup( // by here the object is durably in the bucket and this account's backup // has succeeded whatever the database does next. A pool timeout on the // row would otherwise un-count a copy that exists — the run would report - // one failure too many and the console would read `never` for an account + // one failure too many and the console would read stale for an account // whose disaster-recovery object is fine, which is exactly the inversion // the ledger exists to remove. // + // The row is written whether or not an object came of the walk, because a + // ledger keyed only on success cannot tell "no run has reached this + // account" from "a run reached it and nothing landed" — and the first of + // those is what every account on a running host looks like the day this + // table ships. A failed walk touches `lastAttemptAt` and leaves + // `lastSuccessAt` exactly as it was, so yesterday's good copy still reads + // as the copy this account has. + // // `new Date()` rather than the run's `now`, because on a large cohort the - // two are hours apart and the row is meant to say when this account's - // object landed. - if (objectBytes !== null) { - const bytes = BigInt(objectBytes); - try { - await prisma.offhostBackupState.upsert({ - where: { userId: user.id }, - update: { lastSuccessAt: new Date(), sizeBytes: bytes }, - create: { - userId: user.id, - lastSuccessAt: new Date(), - sizeBytes: bytes, - }, - }); - } catch (err) { - ledgerWriteFailures++; - const message = (err as Error).message ?? "unknown"; - evt?.addWarning( - `offhost-backup ledger write failed for ${user.id}: ${message.slice(0, 200)}`, - ); - } + // two are hours apart and the row is meant to say when this account was + // reached. + const at = new Date(); + try { + await prisma.offhostBackupState.upsert({ + where: { userId: user.id }, + update: + objectBytes === null + ? { lastAttemptAt: at } + : { + lastAttemptAt: at, + lastSuccessAt: at, + sizeBytes: BigInt(objectBytes), + }, + create: + objectBytes === null + ? { userId: user.id, lastAttemptAt: at } + : { + userId: user.id, + lastAttemptAt: at, + lastSuccessAt: at, + sizeBytes: BigInt(objectBytes), + }, + }); + } catch (err) { + ledgerWriteFailures++; + const message = (err as Error).message ?? "unknown"; + evt?.addWarning( + `offhost-backup ledger write failed for ${user.id}: ${message.slice(0, 200)}`, + ); } } if (ledgerWriteFailures > 0) { diff --git a/src/types/backups.ts b/src/types/backups.ts index 8411f688c..0cd2a20ad 100644 --- a/src/types/backups.ts +++ b/src/types/backups.ts @@ -29,6 +29,12 @@ export interface BackupRow { export interface OffhostAccountRow { userId: string; username: string; + /** + * ISO instant a run last walked this account, or null when none has. Null + * is what the whole cohort reads until the first run after the upgrade that + * adds the ledger, and the card says so rather than claiming `never`. + */ + lastAttemptAt: string | null; /** ISO instant the newest object landed, or null when there is none. */ lastSuccessAt: string | null; /** Size of that object in bytes, or null when there is none. */ diff --git a/tests/integration/offhost-backup-ledger.test.ts b/tests/integration/offhost-backup-ledger.test.ts index 27f70d178..f2047fc9a 100644 --- a/tests/integration/offhost-backup-ledger.test.ts +++ b/tests/integration/offhost-backup-ledger.test.ts @@ -109,6 +109,7 @@ describe("off-host backup ledger (real Postgres)", () => { expect( classifyOffhostBackup({ + lastAttemptAt: row.lastAttemptAt, lastSuccessAt: row.lastSuccessAt, now: new Date(), }).freshness, @@ -144,12 +145,80 @@ describe("off-host backup ledger (real Postgres)", () => { }); expect( classifyOffhostBackup({ + lastAttemptAt: row.lastAttemptAt, lastSuccessAt: row.lastSuccessAt, now: new Date(), }).freshness, ).toBe("stale"); }); + it("separates an account no run has recorded from one a run found nothing for", async () => { + const prisma = getPrismaClient(); + await seedUser("account-one"); + + // Before any run: the ledger is empty, which is what every account on a + // host that has been uploading for months looks like the moment this + // table ships. It must not read as "nothing has ever reached the bucket". + expect( + classifyOffhostBackup({ + lastAttemptAt: null, + lastSuccessAt: null, + now: new Date(), + }).freshness, + ).toBe("unknown"); + + // A run that reaches the account and produces no object records the walk + // and leaves the success columns alone. + const bucket = makeBucket(); + bucket.putStream = async () => { + throw new Error("bucket refused the signature"); + }; + const report = await runOffhostBackup(prisma, bucket, new Date()); + expect(report.uploaded).toBe(0); + expect(report.failed).toBe(1); + + const row = await prisma.offhostBackupState.findUniqueOrThrow({ + where: { userId: "account-one" }, + }); + expect(row.lastSuccessAt).toBeNull(); + expect(row.sizeBytes).toBeNull(); + expect( + classifyOffhostBackup({ + lastAttemptAt: row.lastAttemptAt, + lastSuccessAt: row.lastSuccessAt, + now: new Date(), + }).freshness, + ).toBe("never"); + }); + + it("keeps the last good copy on the row when a later run fails", async () => { + const prisma = getPrismaClient(); + await seedUser("account-one"); + + const bucket = makeBucket(); + await runOffhostBackup(prisma, bucket, new Date()); + const good = await prisma.offhostBackupState.findUniqueOrThrow({ + where: { userId: "account-one" }, + }); + expect(good.lastSuccessAt).not.toBeNull(); + + bucket.putStream = async () => { + throw new Error("bucket refused the signature"); + }; + await runOffhostBackup(prisma, bucket, new Date()); + + const after = await prisma.offhostBackupState.findUniqueOrThrow({ + where: { userId: "account-one" }, + }); + // The failed run moved the walk instant and nothing else: yesterday's + // copy is still the copy this account has off-host. + expect(after.lastSuccessAt?.getTime()).toBe(good.lastSuccessAt?.getTime()); + expect(after.sizeBytes).toBe(good.sizeBytes); + expect(after.lastAttemptAt.getTime()).toBeGreaterThanOrEqual( + good.lastAttemptAt.getTime(), + ); + }); + it("goes with the account it describes", async () => { const prisma = getPrismaClient(); await seedUser("account-one");