diff --git a/docs/ops/backup-restore.md b/docs/ops/backup-restore.md index 2b6fe2af9..12b834de3 100644 --- a/docs/ops/backup-restore.md +++ b/docs/ops/backup-restore.md @@ -215,6 +215,26 @@ A run where SOME account got a copy still succeeds, with the rest counted in `offhost_backup_failed`. Failing the whole queue over one account's object would re-upload everybody's on every retry. +### Which account has no copy + +The counts above say how many accounts were uploaded, never which. An account +whose object has failed every night for a month reads as `99 uploaded, 1 +failed` each time, which looks like weather. + +**Admin console → Backups → Off-host copies** answers it per account: when this +host last put that account's encrypted copy in the bucket, how big it was, and +how that compares with the nightly schedule. **Fresh** is inside one run, +**due** means one run produced nothing for that account, **stale** means two +did not, and **never** means no object has ever landed. On a host without +`BACKUP_S3_*` the card says off-host backup is not configured and stops there. + +The verdicts come from a ledger the worker writes when the object lands +(`offhost_backup_state`), not from a listing of the bucket. Keep the worker's +grant as documented — PutObject, GetObject, AbortMultipartUpload — and the card +still tells the truth. A row that goes stale while the run reports success is +that account's object being refused, and the reason is in the run's +`offhost_backup_failures` meta. + ## The weekly in-database backup (`data-backup`) Separate from the off-host job above, and easy to confuse with it. A second diff --git a/messages/de.json b/messages/de.json index fca6667b9..84d2a27b9 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7012,7 +7012,21 @@ "restoreMissingDocuments": "Dokumente", "restoreMissingWorkouts": "Workouts", "restoreMissingMentalHealth": "Psychische Fragebögen", - "restoreMissingConsent": "Einwilligungsnachweise" + "restoreMissingConsent": "Einwilligungsnachweise", + "offhost": { + "title": "Kopien außer Haus", + "description": "Wann dieser Host zuletzt die verschlüsselte Kopie jedes Kontos in deinen Bucket gelegt hat.", + "loadError": "Der Status außer Haus konnte nicht geladen werden.", + "notConfiguredTitle": "Sicherung außer Haus ist nicht eingerichtet", + "notConfiguredDescription": "Es sind weder Bucket noch Schlüssel gesetzt, also verlässt nichts diesen Host. Setze BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY, BACKUP_S3_SECRET_KEY und BACKUP_ENCRYPTION_KEY, um den nächtlichen Upload einzuschalten.", + "stateFresh": "Frisch", + "stateDue": "Fällig", + "stateStale": "Veraltet", + "stateNever": "Nie", + "neverDetail": "Für dieses Konto ist noch nie eine Kopie im Bucket angekommen.", + "age": "vor {hours} h", + "scheduleNote": "Der Upload läuft alle {hours} h. Frisch heißt innerhalb eines Laufs, fällig heißt ein Lauf blieb ohne Ergebnis, veraltet heißt zwei." + } }, "danger-zone": { "title": "Gefahrenzone", diff --git a/messages/en.json b/messages/en.json index 5891d7ddf..dcd603d32 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7012,7 +7012,21 @@ "restoreMissingDocuments": "Documents", "restoreMissingWorkouts": "Workouts", "restoreMissingMentalHealth": "Mental-health screeners", - "restoreMissingConsent": "Consent receipts" + "restoreMissingConsent": "Consent receipts", + "offhost": { + "title": "Off-host copies", + "description": "When this host last put each account's encrypted copy in your bucket.", + "loadError": "Could not load the off-host status.", + "notConfiguredTitle": "Off-host backup is not configured", + "notConfiguredDescription": "No bucket and no key are set, so nothing leaves this host. Set BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY, BACKUP_S3_SECRET_KEY and BACKUP_ENCRYPTION_KEY to turn the nightly upload on.", + "stateFresh": "Fresh", + "stateDue": "Due", + "stateStale": "Stale", + "stateNever": "Never", + "neverDetail": "No copy has ever reached the bucket for this account.", + "age": "{hours} h ago", + "scheduleNote": "The upload runs every {hours} h. Fresh means inside one run, due means one run produced nothing, stale means two did not." + } }, "danger-zone": { "title": "Danger Zone", diff --git a/messages/es.json b/messages/es.json index 990f5c414..0188dd089 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7012,7 +7012,21 @@ "restoreMissingDocuments": "Documentos", "restoreMissingWorkouts": "Entrenamientos", "restoreMissingMentalHealth": "Cuestionarios de salud mental", - "restoreMissingConsent": "Comprobantes de consentimiento" + "restoreMissingConsent": "Comprobantes de consentimiento", + "offhost": { + "title": "Copias fuera del host", + "description": "Cuándo puso este host por última vez la copia cifrada de cada cuenta en tu bucket.", + "loadError": "No se pudo cargar el estado fuera del host.", + "notConfiguredTitle": "La copia fuera del host no está configurada", + "notConfiguredDescription": "No hay bucket ni clave definidos, así que nada sale de este host. Define BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY, BACKUP_S3_SECRET_KEY y BACKUP_ENCRYPTION_KEY para activar la subida nocturna.", + "stateFresh": "Reciente", + "stateDue": "Pendiente", + "stateStale": "Obsoleta", + "stateNever": "Nunca", + "neverDetail": "Para esta cuenta nunca ha llegado una copia al bucket.", + "age": "hace {hours} h", + "scheduleNote": "La subida se ejecuta cada {hours} h. Reciente significa dentro de una ejecución, pendiente que una no produjo nada, obsoleta que fueron dos." + } }, "danger-zone": { "title": "Zona de peligro", diff --git a/messages/fr.json b/messages/fr.json index eee2b3cbd..38dc08eb9 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7012,7 +7012,21 @@ "restoreMissingDocuments": "Documents", "restoreMissingWorkouts": "Séances", "restoreMissingMentalHealth": "Questionnaires de santé mentale", - "restoreMissingConsent": "Preuves de consentement" + "restoreMissingConsent": "Preuves de consentement", + "offhost": { + "title": "Copies hors hôte", + "description": "Quand cet hôte a déposé pour la dernière fois la copie chiffrée de chaque compte dans votre bucket.", + "loadError": "Impossible de charger l'état hors hôte.", + "notConfiguredTitle": "La sauvegarde hors hôte n'est pas configurée", + "notConfiguredDescription": "Aucun bucket ni clé ne sont définis, donc rien ne quitte cet hôte. Définissez BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY, BACKUP_S3_SECRET_KEY et BACKUP_ENCRYPTION_KEY pour activer l'envoi nocturne.", + "stateFresh": "Récente", + "stateDue": "Attendue", + "stateStale": "Périmée", + "stateNever": "Jamais", + "neverDetail": "Aucune copie n'est jamais arrivée dans le bucket pour ce compte.", + "age": "il y a {hours} h", + "scheduleNote": "L'envoi tourne toutes les {hours} h. Récente signifie dans un cycle, attendue qu'un cycle n'a rien produit, périmée que deux n'ont rien produit." + } }, "danger-zone": { "title": "Zone de danger", diff --git a/messages/it.json b/messages/it.json index f550c93a0..973619366 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7012,7 +7012,21 @@ "restoreMissingDocuments": "Documenti", "restoreMissingWorkouts": "Allenamenti", "restoreMissingMentalHealth": "Questionari sulla salute mentale", - "restoreMissingConsent": "Ricevute di consenso" + "restoreMissingConsent": "Ricevute di consenso", + "offhost": { + "title": "Copie fuori host", + "description": "Quando questo host ha messo per l'ultima volta la copia cifrata di ogni account nel tuo bucket.", + "loadError": "Impossibile caricare lo stato fuori host.", + "notConfiguredTitle": "Il backup fuori host non è configurato", + "notConfiguredDescription": "Non sono impostati né bucket né chiave, quindi nulla lascia questo host. Imposta BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY, BACKUP_S3_SECRET_KEY e BACKUP_ENCRYPTION_KEY per attivare il caricamento notturno.", + "stateFresh": "Recente", + "stateDue": "Attesa", + "stateStale": "Obsoleta", + "stateNever": "Mai", + "neverDetail": "Per questo account nessuna copia è mai arrivata nel bucket.", + "age": "{hours} h fa", + "scheduleNote": "Il caricamento gira ogni {hours} h. Recente significa entro un giro, attesa che un giro non ha prodotto nulla, obsoleta che non lo hanno fatto due." + } }, "danger-zone": { "title": "Zona pericolosa", diff --git a/messages/ko.json b/messages/ko.json index 641082b67..800afd943 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6992,7 +6992,21 @@ "restoreMissingDocuments": "문서", "restoreMissingWorkouts": "운동", "restoreMissingMentalHealth": "정신 건강 선별 검사", - "restoreMissingConsent": "동의 기록" + "restoreMissingConsent": "동의 기록", + "offhost": { + "title": "외부 보관 사본", + "description": "이 호스트가 각 계정의 암호화 사본을 버킷에 마지막으로 올린 시점입니다.", + "loadError": "외부 보관 상태를 불러오지 못했습니다.", + "notConfiguredTitle": "외부 보관 백업이 설정되지 않았습니다", + "notConfiguredDescription": "버킷과 키가 모두 설정되지 않아 이 호스트를 벗어나는 것이 없습니다. 야간 업로드를 켜려면 BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY, BACKUP_S3_SECRET_KEY, BACKUP_ENCRYPTION_KEY를 설정하세요.", + "stateFresh": "최신", + "stateDue": "지연", + "stateStale": "오래됨", + "stateNever": "없음", + "neverDetail": "이 계정의 사본이 버킷에 도착한 적이 없습니다.", + "age": "{hours}시간 전", + "scheduleNote": "업로드는 {hours}시간마다 실행됩니다. 최신은 한 번의 실행 이내, 지연은 한 번이 아무것도 만들지 못한 상태, 오래됨은 두 번이 그런 상태입니다." + } }, "danger-zone": { "title": "위험 구역", diff --git a/messages/pl.json b/messages/pl.json index 1caaad0a3..698a2e2c5 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7012,7 +7012,21 @@ "restoreMissingDocuments": "Dokumenty", "restoreMissingWorkouts": "Treningi", "restoreMissingMentalHealth": "Kwestionariusze zdrowia psychicznego", - "restoreMissingConsent": "Potwierdzenia zgody" + "restoreMissingConsent": "Potwierdzenia zgody", + "offhost": { + "title": "Kopie poza hostem", + "description": "Kiedy ten host ostatnio umieścił zaszyfrowaną kopię każdego konta w Twoim buckecie.", + "loadError": "Nie udało się wczytać stanu poza hostem.", + "notConfiguredTitle": "Kopia poza hostem nie jest skonfigurowana", + "notConfiguredDescription": "Nie ustawiono ani bucketu, ani klucza, więc nic nie opuszcza tego hosta. Ustaw BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY, BACKUP_S3_SECRET_KEY i BACKUP_ENCRYPTION_KEY, aby włączyć nocne wysyłanie.", + "stateFresh": "Świeża", + "stateDue": "Zaległa", + "stateStale": "Przeterminowana", + "stateNever": "Nigdy", + "neverDetail": "Dla tego konta żadna kopia nigdy nie trafiła do bucketu.", + "age": "{hours} h temu", + "scheduleNote": "Wysyłka działa co {hours} h. Świeża znaczy w obrębie jednego przebiegu, zaległa że jeden nic nie dał, przeterminowana że dwa." + } }, "danger-zone": { "title": "Strefa zagrożenia", diff --git a/prisma/migrations/0336_offhost_backup_ledger/migration.sql b/prisma/migrations/0336_offhost_backup_ledger/migration.sql new file mode 100644 index 000000000..9239d5eaf --- /dev/null +++ b/prisma/migrations/0336_offhost_backup_ledger/migration.sql @@ -0,0 +1,16 @@ +-- One row per account, written by the nightly off-host backup job when an +-- object lands in the bucket. The admin console reads it to say, per account, +-- whether the newest off-host copy is as fresh as the schedule promises, +-- without the page ever talking to the bucket. +CREATE TABLE "offhost_backup_state" ( + "user_id" TEXT NOT NULL, + "last_success_at" TIMESTAMP(3) NOT NULL, + "size_bytes" INTEGER NOT NULL, + + CONSTRAINT "offhost_backup_state_pkey" PRIMARY KEY ("user_id") +); + +ALTER TABLE "offhost_backup_state" + ADD CONSTRAINT "offhost_backup_state_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users"("id") + ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 0c6e0a00a..5d38625d1 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -770,6 +770,7 @@ model User { notificationChannels NotificationChannel[] pushSubscriptions PushSubscription[] dataBackups DataBackup[] + offhostBackupState OffhostBackupState? achievements UserAchievement[] recommendationFeedback RecommendationFeedback[] idempotencyKeys IdempotencyKey[] @@ -4937,6 +4938,34 @@ model DataBackup { @@map("data_backups") } +// ─── Off-host backup ledger ────────────────────────────────── +// +// One row per account, written by the nightly off-host job the moment an +// object lands in the bucket. It exists because "the job ran" and "this +// account has a recent copy off-host" are different facts: the run reports +// counts, and a cohort of a hundred accounts where one has silently failed +// every night for a month reads exactly like a healthy one. +// +// The admin console's off-host card is the reader, and it is the only reader +// there will be: the page computes staleness from this timestamp rather than +// listing the bucket, so an operator who has revoked the worker's read grant +// still sees the truth. +model OffhostBackupState { + userId String @id @map("user_id") + + /// When the newest object for this account finished uploading. Consumer: + /// `classifyOffhostBackup()` behind `GET /api/admin/backups`. + lastSuccessAt DateTime @map("last_success_at") + + /// Size of that object in bytes — the same number the run reports as its + /// largest, per account. Consumer: the off-host card's size column. + sizeBytes Int @map("size_bytes") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@map("offhost_backup_state") +} + // ─── Apple Health export.zip import (v1.4.34) ─────────────── // // Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the diff --git a/src/__tests__/success-affordance-guard.test.ts b/src/__tests__/success-affordance-guard.test.ts index b991b8cab..6df0bd37e 100644 --- a/src/__tests__/success-affordance-guard.test.ts +++ b/src/__tests__/success-affordance-guard.test.ts @@ -109,6 +109,10 @@ const PINNED_AFFORDANCES: Record< "src/components/admin/module-availability-section.tsx": { "toast.success": 1, }, + // Reads a ledger, writes nothing: the wash marks how fresh one account's + // off-host copy is, and the state it paints green came out of the database + // rather than out of a request this card just made. + "src/components/admin/offhost-backups-section.tsx": { "text-success": 1 }, "src/components/admin/recent-audit-preview.tsx": { CheckCircle2: 2, "text-success": 1, diff --git a/src/app/admin/[section]/renderer.tsx b/src/app/admin/[section]/renderer.tsx index f9548687e..24ac6042a 100644 --- a/src/app/admin/[section]/renderer.tsx +++ b/src/app/admin/[section]/renderer.tsx @@ -25,6 +25,7 @@ import { CoachFeedbackSection } from "@/components/admin/coach-feedback-section" import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-section"; import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section"; import { BackupsSection } from "@/components/admin/backups-section"; +import { OffhostBackupsSection } from "@/components/admin/offhost-backups-section"; import { DangerZoneSection } from "@/components/admin/danger-zone-section"; import { EncryptionSection } from "@/components/admin/encryption-section"; import { GeneralSettingsSection } from "@/components/admin/general-settings-section"; @@ -144,9 +145,12 @@ export function AdminSectionRenderer({ ); case "backups": + // Two cards, two independent failure modes: the copies held in this + // database, then the copies that reached the operator's bucket. return ( + ); case "encryption": diff --git a/src/app/api/admin/backups/__tests__/route.test.ts b/src/app/api/admin/backups/__tests__/route.test.ts new file mode 100644 index 000000000..e96b759e4 --- /dev/null +++ b/src/app/api/admin/backups/__tests__/route.test.ts @@ -0,0 +1,146 @@ +/** + * `GET /api/admin/backups` — the off-host leg. + * + * The rows the weekly pass writes into this database were already covered by + * the page that lists them. What was not covered is the question this response + * now answers: per account, when did an encrypted copy last reach the + * operator's bucket, and is that inside what the nightly schedule promises. + * + * The bucket is never touched here, and that is the assertion that matters — + * the verdicts come from the ledger the worker writes, so an operator whose + * worker holds PutObject and nothing else still gets a true answer. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/api-handler", () => ({ + apiHandler: unknown>(fn: T) => fn, + requireAdmin: vi.fn(async () => ({ user: { id: "admin-1" } })), +})); + +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +vi.mock("@/lib/jobs/job-failures", () => ({ + readLastQueueRun: vi.fn(async () => null), +})); + +const queryRawMock = vi.fn(); +const findManyMock = vi.fn(); +vi.mock("@/lib/db", () => ({ + prisma: { + $queryRaw: (...a: unknown[]) => queryRawMock(...a), + user: { findMany: (...a: unknown[]) => findManyMock(...a) }, + }, +})); + +const configuredMock = vi.fn(() => true); +vi.mock("@/lib/jobs/offhost-backup", () => ({ + offhostBackupConfigured: () => configuredMock(), +})); + +import { GET } from "../route"; +import type { BackupsList } from "@/types/backups"; + +const HOUR = 3_600_000; + +async function read(): Promise { + const res = await (GET as unknown as () => Promise)(); + expect(res.status).toBe(200); + return (await res.json()).data as BackupsList; +} + +beforeEach(() => { + vi.clearAllMocks(); + configuredMock.mockReturnValue(true); + queryRawMock.mockResolvedValue([]); + findManyMock.mockResolvedValue([]); +}); + +describe("GET /api/admin/backups — off-host freshness", () => { + it("rates every account against the nightly schedule without listing the bucket", async () => { + const now = Date.now(); + findManyMock.mockResolvedValue([ + { + id: "u1", + username: "account-one", + offhostBackupState: { + lastSuccessAt: new Date(now - 3 * HOUR), + sizeBytes: 4096, + }, + }, + { + id: "u2", + username: "account-two", + offhostBackupState: { + lastSuccessAt: new Date(now - 30 * HOUR), + sizeBytes: 8192, + }, + }, + { + id: "u3", + username: "account-three", + offhostBackupState: { + lastSuccessAt: new Date(now - 100 * HOUR), + sizeBytes: 128, + }, + }, + { id: "u4", username: "account-four", offhostBackupState: null }, + ]); + + const body = await read(); + + expect(body.offhost.configured).toBe(true); + expect(body.offhost.periodHours).toBe(24); + expect( + body.offhost.rows.map((row) => [row.username, row.freshness]), + ).toEqual([ + ["account-one", "fresh"], + ["account-two", "due"], + ["account-three", "stale"], + ["account-four", "never"], + ]); + expect(body.offhost.rows[0]?.sizeBytes).toBe(4096); + expect(body.offhost.rows[0]?.ageHours).toBe(3); + expect(body.offhost.rows[3]?.lastSuccessAt).toBeNull(); + expect(body.offhost.rows[3]?.sizeBytes).toBeNull(); + }); + + it("says so and reads no account when off-host backup is not configured", async () => { + configuredMock.mockReturnValue(false); + + const body = await read(); + + expect(body.offhost).toEqual({ + configured: false, + periodHours: 24, + rows: [], + }); + // Not merely empty output — the query never runs, so an unconfigured host + // does not pay for a table scan to be told nothing is set up. + expect(findManyMock).not.toHaveBeenCalled(); + }); + + it("never ships a credential or an object key with the verdicts", async () => { + findManyMock.mockResolvedValue([ + { + id: "u1", + username: "account-one", + offhostBackupState: { + lastSuccessAt: new Date(), + sizeBytes: 1, + }, + }, + ]); + + const body = await read(); + + expect(Object.keys(body.offhost.rows[0] ?? {}).sort()).toEqual([ + "ageHours", + "freshness", + "lastSuccessAt", + "sizeBytes", + "userId", + "username", + ]); + expect(JSON.stringify(body.offhost)).not.toMatch(/bucket|endpoint|secret/i); + }); +}); diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 04a8c2139..b1885683f 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -13,6 +13,12 @@ * Without those two the page cannot tell a working backup from one that * stopped six weeks ago — every row it lists has a perfectly ordinary * timestamp either way. + * + * And alongside both, the off-host leg: one row per account saying when this + * host last put that account's encrypted copy in the operator's bucket and how + * stale that is against the nightly schedule. It comes from the ledger the + * worker writes, never from a bucket listing — the answer has to hold on a host + * whose worker grant is PutObject and nothing else. */ import { prisma } from "@/lib/db"; import { apiHandler, requireAdmin } from "@/lib/api-handler"; @@ -21,10 +27,19 @@ import { annotate } from "@/lib/logging/context"; import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; import { readLastQueueRun } from "@/lib/jobs/job-failures"; import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status"; +import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; +import { + classifyOffhostBackup, + OFFHOST_BACKUP_PERIOD_HOURS, +} from "@/lib/jobs/offhost-backup-freshness"; // v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts` // so callers (in particular `components/admin/backups-section.tsx`) don't // have to reach across the component → route-handler layer boundary. -import type { BackupRow, BackupsList } from "@/types/backups"; +import type { + BackupRow, + BackupsList, + OffhostAccountRow, +} from "@/types/backups"; export const dynamic = "force-dynamic"; @@ -67,6 +82,42 @@ export const GET = apiHandler(async () => { createdAt: row.created_at.toISOString(), })); + // The off-host leg, per account, read from the ledger the nightly worker + // writes rather than from the bucket. The page must be able to answer "which + // account has no recent copy off-host" on a host whose worker grant is + // PutObject and nothing else, and a listing call from a page render would + // put the operator's credentials on the request path of every page view. + const now = new Date(); + const configured = offhostBackupConfigured(); + const offhostAccounts = configured + ? await prisma.user.findMany({ + select: { + id: true, + username: true, + offhostBackupState: { + select: { lastSuccessAt: true, sizeBytes: true }, + }, + }, + orderBy: { username: "asc" }, + }) + : []; + + const offhostRows: OffhostAccountRow[] = offhostAccounts.map((account) => { + const state = account.offhostBackupState; + const verdict = classifyOffhostBackup({ + lastSuccessAt: state?.lastSuccessAt ?? null, + now, + }); + return { + userId: account.id, + username: account.username, + lastSuccessAt: state?.lastSuccessAt.toISOString() ?? null, + sizeBytes: state?.sizeBytes ?? null, + ageHours: verdict.ageHours, + freshness: verdict.freshness, + }; + }); + const payload: BackupsList = { rows: list, // Matches the retention window the backup-prune job enforces so the @@ -77,8 +128,13 @@ export const GET = apiHandler(async () => { .filter((row) => row.type === SCHEDULED_BACKUP_TYPE) .map((row) => row.created_at), lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE), - now: new Date(), + now, }), + offhost: { + configured, + periodHours: OFFHOST_BACKUP_PERIOD_HOURS, + rows: offhostRows, + }, }; return apiSuccess(payload); diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index f6f448a0a..d776da699 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -322,7 +322,14 @@ function RestoreRowDialog({ ); } -function formatBytes(bytes: number, fmt: ReturnType) { +/** + * Exported for the off-host card, which reads the same sizes out of the same + * response and must not spell "1.25 MB" a second way two cards apart. + */ +export function formatBytes( + bytes: number, + fmt: ReturnType, +) { if (bytes < 1024) return `${bytes} B`; if (bytes < 1024 * 1024) { return `${fmt.integer(Math.round(bytes / 1024))} KB`; diff --git a/src/components/admin/offhost-backups-section.tsx b/src/components/admin/offhost-backups-section.tsx new file mode 100644 index 000000000..280819a0f --- /dev/null +++ b/src/components/admin/offhost-backups-section.tsx @@ -0,0 +1,162 @@ +"use client"; + +/** + * `` — per-account freshness of the off-host copies. + * + * The card above it lists the copies that live in THIS database. This one + * answers the different question: for each account, how long ago did this host + * put an encrypted copy in the operator's bucket, and is that inside what the + * nightly schedule promises. The two fail independently — a database whose + * weekly rows are current tells an operator nothing about a bucket that has + * been refusing the worker's signature since March. + * + * Every verdict is computed server-side from the ledger the worker writes. The + * page never lists the bucket: the worker's grant is deliberately PutObject + + * GetObject, and putting a credentialed listing call on the render path of an + * admin page would be a poor trade for a number the database already holds. + */ + +import { useQuery } from "@tanstack/react-query"; +import { CloudUpload } from "lucide-react"; +import { SettingsCard } from "@/components/settings/settings-card"; +import { SettingsCardHeader } from "@/components/settings/_card-header"; +import { Badge } from "@/components/ui/badge"; +import { EmptyState } from "@/components/ui/empty-state"; +import { ListRow } from "@/components/ui/list-row"; +import { QueryErrorRow } from "@/components/ui/query-error-row"; +import { Skeleton } from "@/components/ui/skeleton"; +import { apiGet } from "@/lib/api/api-fetch"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import { queryKeys } from "@/lib/query-keys"; +import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness"; +import type { BackupsList, OffhostAccountRow } from "@/types/backups"; +import { formatBytes } from "./backups-section"; + +/** + * Status wash per verdict. `never` is deliberately neutral rather than + * alarming: a host that turned the off-host job on this afternoon has not + * failed at anything yet, and painting every row red on day one is how an + * operator learns to stop reading the colour. + */ +const FRESHNESS_STYLE: Record = { + fresh: "border-success/30 bg-success/10 text-success", + due: "border-warning/30 bg-warning/10 text-warning", + stale: "border-destructive/30 bg-destructive/10 text-destructive", + never: "border-border bg-muted text-muted-foreground", +}; + +function freshnessLabel( + freshness: OffhostBackupFreshness, + t: ReturnType["t"], +): string { + // Literal keys, one per arm: a template-built key resolves at runtime and is + // invisible to the call-site coverage guard. + switch (freshness) { + case "fresh": + return t("admin.section.backups.offhost.stateFresh"); + case "due": + return t("admin.section.backups.offhost.stateDue"); + case "stale": + return t("admin.section.backups.offhost.stateStale"); + case "never": + return t("admin.section.backups.offhost.stateNever"); + } +} + +function AccountRow({ row }: { row: OffhostAccountRow }) { + const { t } = useTranslations(); + const fmt = useFormatters(); + + return ( + +
  • +
    + + {row.username} + + + {freshnessLabel(row.freshness, t)} + +
    +

    + {row.lastSuccessAt === null || row.sizeBytes === null + ? t("admin.section.backups.offhost.neverDetail") + : `${fmt.dateTime(row.lastSuccessAt)} · ${formatBytes( + row.sizeBytes, + fmt, + )} · ${t("admin.section.backups.offhost.age", { + hours: row.ageHours ?? 0, + })}`} +

    +
    • +     + ); +} + +export function OffhostBackupsSection() { + const { t } = useTranslations(); + + const { data, isLoading, isError, refetch } = useQuery({ + queryKey: queryKeys.adminBackups(), + queryFn: async () => apiGet("/api/admin/backups"), + }); + + const offhost = data?.offhost; + + return ( + + + {offhost.rows.length} + + ) : null + } + /> + + {isLoading ? ( +
    + + +
    + ) : isError ? ( + void refetch()} + /> + ) : offhost && !offhost.configured ? ( + } + title={t("admin.section.backups.offhost.notConfiguredTitle")} + description={t( + "admin.section.backups.offhost.notConfiguredDescription", + )} + /> + ) : offhost ? ( +
    +
      + {offhost.rows.map((row) => ( + + ))} +
    +

    + {t("admin.section.backups.offhost.scheduleNote", { + hours: offhost.periodHours, + })} +

    +
    + ) : null} +     + ); +} diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 43fab3294..17b3e7dd0 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -210,6 +210,11 @@ export const WIPE_MODELS = [ // just returned. "IdempotencyKey", "DataBackup", + // What this host last put in the operator's bucket for this account. The + // objects themselves stay — the worker holds no DeleteObject grant and the + // bucket's lifecycle rule owns their retirement — but the row saying when + // they were written is about this account and goes with it. + "OffhostBackupState", "AuditLog", "ConsentReceipt", "UserKnownDevice", diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index 5f47cadf2..8b216df9b 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -642,6 +642,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly> = { "Replay protection for requests this host already answered. Restoring it would suppress a legitimate new request.", DataBackup: "The backup catalogue itself. A backup that contains the list of backups is a recursion with no reader.", + OffhostBackupState: + "When this host last put this account's copy in this operator's bucket, and how big it was. It describes one deployment's relationship with one bucket, so restoring it elsewhere would assert an off-host copy that host has never written.", ImportJob: "A job record pointing at an uploaded file that the backup does not carry, so restoring it would resurrect a task with nothing to work on.", InviteToken: diff --git a/src/lib/jobs/__tests__/offhost-backup-freshness.test.ts b/src/lib/jobs/__tests__/offhost-backup-freshness.test.ts new file mode 100644 index 000000000..f364d9e69 --- /dev/null +++ b/src/lib/jobs/__tests__/offhost-backup-freshness.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; +import { + classifyOffhostBackup, + OFFHOST_BACKUP_PERIOD_HOURS, +} from "@/lib/jobs/offhost-backup-freshness"; + +const NOW = new Date("2026-09-10T09:00:00.000Z"); +const HOUR = 3_600_000; +const PERIOD = OFFHOST_BACKUP_PERIOD_HOURS * HOUR; + +/** `age` hours before NOW. */ +function ago(ms: number): Date { + return new Date(NOW.getTime() - ms); +} + +describe("classifyOffhostBackup", () => { + it("says never when the worker has put nothing there for this account", () => { + expect(classifyOffhostBackup({ lastSuccessAt: null, now: NOW })).toEqual({ + freshness: "never", + ageHours: null, + }); + }); + + it("is fresh right up to the one-period boundary and due one millisecond past it", () => { + expect( + classifyOffhostBackup({ lastSuccessAt: ago(PERIOD), now: NOW }).freshness, + ).toBe("fresh"); + expect( + classifyOffhostBackup({ lastSuccessAt: ago(PERIOD + 1), now: NOW }) + .freshness, + ).toBe("due"); + }); + + it("is due right up to the two-period boundary and stale one millisecond past it", () => { + expect( + classifyOffhostBackup({ lastSuccessAt: ago(PERIOD * 2), now: NOW }) + .freshness, + ).toBe("due"); + expect( + classifyOffhostBackup({ lastSuccessAt: ago(PERIOD * 2 + 1), now: NOW }) + .freshness, + ).toBe("stale"); + }); + + it("reports whole hours of age", () => { + expect( + classifyOffhostBackup({ lastSuccessAt: ago(90 * 60_000), now: NOW }), + ).toEqual({ freshness: "fresh", ageHours: 1 }); + }); + + it("honours a caller's own period rather than the nightly default", () => { + // A six-hour schedule: the same 12-hour-old object that is fresh under the + // nightly cron is two periods behind under this one. + const twelveHours = ago(12 * HOUR); + expect( + classifyOffhostBackup({ lastSuccessAt: twelveHours, now: NOW }).freshness, + ).toBe("fresh"); + expect( + classifyOffhostBackup({ + lastSuccessAt: twelveHours, + now: NOW, + periodHours: 6, + }).freshness, + ).toBe("due"); + expect( + classifyOffhostBackup({ + lastSuccessAt: twelveHours, + now: NOW, + periodHours: 5, + }).freshness, + ).toBe("stale"); + }); + + it("does not call a copy stale because the host clock ran backwards", () => { + expect( + classifyOffhostBackup({ + lastSuccessAt: new Date(NOW.getTime() + 2 * HOUR), + now: NOW, + }), + ).toEqual({ freshness: "fresh", ageHours: 0 }); + }); +}); diff --git a/src/lib/jobs/__tests__/offhost-backup.test.ts b/src/lib/jobs/__tests__/offhost-backup.test.ts index 5a630b088..3bd97c4df 100644 --- a/src/lib/jobs/__tests__/offhost-backup.test.ts +++ b/src/lib/jobs/__tests__/offhost-backup.test.ts @@ -184,6 +184,7 @@ describe("runOffhostBackup", () => { s3.store.set("2020-01-01/user-old.json.enc", Buffer.from([0])); const prisma = { + offhostBackupState: { upsert: vi.fn() }, user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]), }, @@ -229,11 +230,26 @@ describe("runOffhostBackup", () => { // Nothing went through the whole-buffer arm. expect(s3.putObject).not.toHaveBeenCalled(); expect(s3.putStream).toHaveBeenCalledTimes(2); + + // Every account that got an object gets a ledger row, because the run's + // own counts cannot say WHICH account has a copy off-host and which one + // has been failing quietly since March. + expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(2); + const ledgered = prisma.offhostBackupState.upsert.mock.calls.map( + (call) => (call[0] as { where: { userId: string } }).where.userId, + ); + expect(ledgered).toEqual(["u1", "u2"]); + const first = prisma.offhostBackupState.upsert.mock.calls[0]?.[0] as { + create: { sizeBytes: number; lastSuccessAt: Date }; + }; + expect(first.create.sizeBytes).toBe(ct.byteLength); + expect(first.create.lastSuccessAt).toBeInstanceOf(Date); }); it("uploads the canonical builder output without reshaping it", async () => { const s3 = makeS3Mock(); const prisma = { + offhostBackupState: { upsert: vi.fn() }, user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, }; const canonicalPayload = { @@ -291,6 +307,7 @@ describe("runOffhostBackup", () => { it("counts per-user failures without aborting the whole run", async () => { const s3 = makeS3Mock(); const prisma = { + offhostBackupState: { upsert: vi.fn() }, user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]), }, @@ -319,6 +336,7 @@ describe("runOffhostBackup", () => { it("refuses an account whose object outgrows one multipart upload", async () => { const s3 = makeS3Mock(); const prisma = { + offhostBackupState: { upsert: vi.fn() }, user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, }; mocks.buildFullBackupPayload.mockResolvedValue({ @@ -355,6 +373,7 @@ describe("runOffhostBackup", () => { it("reports the largest object it wrote", async () => { const s3 = makeS3Mock(); const prisma = { + offhostBackupState: { upsert: vi.fn() }, user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, }; const report = await runOffhostBackup( diff --git a/src/lib/jobs/offhost-backup-freshness.ts b/src/lib/jobs/offhost-backup-freshness.ts new file mode 100644 index 000000000..fe1fee86c --- /dev/null +++ b/src/lib/jobs/offhost-backup-freshness.ts @@ -0,0 +1,65 @@ +/** + * How fresh one account's newest off-host copy is. + * + * The nightly off-host job reports counts — uploaded, failed, total — and a + * count is exactly the shape that hides a single account. A cohort where one + * account's object has failed every night for a month reports "99 uploaded, 1 + * failed" every night, which reads like weather rather than like an account + * that has no disaster-recovery copy at all. The per-account ledger the worker + * writes (`OffhostBackupState`) turns that into a fact the console can state + * per row, and this function is the only place that fact is judged. + * + * Deliberately pure and deliberately not talking to the bucket: the admin page + * computes staleness from the timestamp the worker wrote, so the answer is the + * same whether or not the app process holds a read grant on the bucket, and a + * listing of somebody's backups never crosses the network to show a page. + * + * Four states, because the run is nightly and one missed night is not the same + * event as four: + * + * never — the worker has never put an object there for this account. + * fresh — inside one schedule period. This is what every account looks like + * on a host where the cron is doing its job. + * due — past one period, inside two. One night produced nothing: a retry, a + * long run, a restart during the window. Worth showing, not worth + * alarming about. + * stale — past two periods. Two consecutive nights produced nothing for this + * account, which no ordinary hiccup explains. + */ + +/** + * Hours between two scheduled off-host runs. Tracks `OFFHOST_BACKUP_CRON` + * (`30 2 * * *`) in `src/lib/jobs/reminder/register-maintenance.ts` — nightly. + */ +export const OFFHOST_BACKUP_PERIOD_HOURS = 24; + +export type OffhostBackupFreshness = "never" | "fresh" | "due" | "stale"; + +export interface OffhostBackupVerdict { + freshness: OffhostBackupFreshness; + /** Whole hours since the newest object landed; null when there is none. */ + ageHours: number | null; +} + +export function classifyOffhostBackup(args: { + lastSuccessAt: Date | null; + now: Date; + /** Defaults to the nightly schedule. Tests pass their own. */ + periodHours?: number; +}): OffhostBackupVerdict { + const { lastSuccessAt, now } = args; + if (lastSuccessAt === null) return { freshness: "never", ageHours: null }; + + const periodMs = + (args.periodHours ?? OFFHOST_BACKUP_PERIOD_HOURS) * 3_600_000; + const ageMs = now.getTime() - lastSuccessAt.getTime(); + // A clock that ran backwards between the upload and this read reports a + // negative age. That is a host-clock problem, not a backup problem, and + // calling the copy stale over it would point the operator at the wrong + // thing. + const ageHours = Math.max(0, Math.floor(ageMs / 3_600_000)); + + if (ageMs <= periodMs) return { freshness: "fresh", ageHours }; + if (ageMs <= periodMs * 2) return { freshness: "due", ageHours }; + return { freshness: "stale", ageHours }; +} diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 0f55273c0..4f3a2cc62 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -61,6 +61,27 @@ function decodeBackupKey(raw: string): Buffer { ); } +/** + * Whether this host has the five variables the nightly job needs. + * + * Separate from `loadOffhostConfig()` because a surface that only wants to say + * "off-host backup is not set up here" must not decode the encryption key to + * find out — `decodeBackupKey` throws on a malformed one, and an operator + * whose key has a typo should see the admin page, not a 500. + */ +export function offhostBackupConfigured(): boolean { + return OFFHOST_ENV_VARS.every((name) => Boolean(process.env[name])); +} + +/** The five variables the nightly job needs, in one place. */ +const OFFHOST_ENV_VARS = [ + "BACKUP_S3_ENDPOINT", + "BACKUP_S3_BUCKET", + "BACKUP_S3_ACCESS_KEY", + "BACKUP_S3_SECRET_KEY", + "BACKUP_ENCRYPTION_KEY", +] as const; + export function loadOffhostConfig(): OffhostBackupConfig | null { const endpoint = process.env.BACKUP_S3_ENDPOINT; const bucket = process.env.BACKUP_S3_BUCKET; @@ -573,6 +594,21 @@ export async function runOffhostBackup( }), options, ); + // The per-account ledger, written the moment the object exists. The run + // itself only ever reported counts, and a count cannot say WHICH account + // has no copy — which is the one question an operator asks about a + // backup. `new Date()` rather than the run's `now`, because on a large + // cohort the two are hours apart and the row is meant to say when this + // account's object landed. + await prisma.offhostBackupState.upsert({ + where: { userId: user.id }, + update: { lastSuccessAt: new Date(), sizeBytes: objectBytes }, + create: { + userId: user.id, + lastSuccessAt: new Date(), + sizeBytes: objectBytes, + }, + }); largestObjectBytes = Math.max(largestObjectBytes, objectBytes); uploaded++; } catch (err) { diff --git a/src/types/backups.ts b/src/types/backups.ts index 1931628ea..8411f688c 100644 --- a/src/types/backups.ts +++ b/src/types/backups.ts @@ -10,6 +10,7 @@ * keeps owning the HTTP contract, the component keeps owning the UI. */ import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status"; +import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness"; export interface BackupRow { id: string; @@ -24,6 +25,35 @@ export interface BackupRow { createdAt: string; } +/** One account's newest object in the operator's off-host bucket. */ +export interface OffhostAccountRow { + userId: string; + username: string; + /** ISO instant the newest object landed, or null when there is none. */ + lastSuccessAt: string | null; + /** Size of that object in bytes, or null when there is none. */ + sizeBytes: number | null; + /** Whole hours since it landed, or null when there is none. */ + ageHours: number | null; + freshness: OffhostBackupFreshness; +} + +/** + * The off-host picture, per account. + * + * Computed from the ledger the nightly worker writes, never from the bucket: + * the page must be able to answer "does this account have a recent copy + * off-host" on a host whose worker holds PutObject and nothing else. + */ +export interface OffhostBackupOverview { + /** Whether this host has the variables the nightly job needs. */ + configured: boolean; + /** Hours between two scheduled runs — the number behind the verdicts. */ + periodHours: number; + /** One row per account. Empty when off-host backup is not configured. */ + rows: OffhostAccountRow[]; +} + export interface BackupsList { rows: BackupRow[]; /** @@ -32,6 +62,12 @@ export interface BackupsList { * made six weeks ago and one made on Sunday look identical in a table. */ schedule: BackupScheduleStatus; + /** + * The off-host leg. A weekly row in this table says the copy that lives in + * this database is current; it says nothing about whether anything reached + * the operator's bucket, and those two fail independently. + */ + offhost: OffhostBackupOverview; /** * Soft retention hint — the worker is configured for weekly backups * (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and