feat(admin): judge every backup signal and push changes to admins

Every backup failure was a pull, on a page nobody opens until the day it
matters: the weekly copy stopped for 46 days and showed an ordinary row,
a failed monthly drill was on the failing-jobs card three days in thirty
(job-failures.ts reads 72 hours), and an off-host bucket that was never
set up showed only on the backups page.

One readiness model (src/lib/recovery/readiness-model.ts) judges six
signals as ok, attention or failing, with a reason code, the time it is
about and where to fix it:

- database dump: failing once the newest good dump is older than 36 h
  while dumps are on, or when it can be kept nowhere; read through a
  provider that reports "not available" until the dump lands;
- weekly account copies: worst account decides, attention past 9 days,
  failing past 14; an account without a copy ages from its creation;
- off-host copies: attention when any account is two nights behind;
- restore drill: failing until a later drill passes, with no time
  window; the drill now keeps its verdict in restore_drill_outcomes;
- off-host setup: attention 7 days after the first admin exists;
- recovery kit: attention when a needed key is not confirmed, by id,
  7 days after setup or as soon as a confirmed kit is out of date.

A "Recovery readiness" card shows them on the admin overview and the
backups page. An hourly job (recovery-readiness, :33) sends each change
to every admin through the localised admin path, under a new
admin-only BACKUP_HEALTH event that the channel preferences route: once
when a signal turns to attention or failing, once a day while it stays
failing, once when it recovers. Each notice is claimed with a
compare-and-set before it is sent, so two workers never send the same
one, and nothing is claimed while no admin exists.

The backups page calls the weekly copy stale at the same instant as the
model (past nine days rather than past ten).

src/__tests__/success-affordance-guard.test.ts

@@ -135,6 +135,13 @@ const PINNED_AFFORDANCES: Record<
   // off-host copy is, and the state it paints green came out of the database
   // rather than out of a request this card just made.
   "src/components/admin/offhost-backups-section.tsx": { "text-success": 1 },
+  // Reads the server's verdict, writes nothing: the check and the green mark
+  // say a backup signal is ok as the readiness model judged it, not that a
+  // request this card made succeeded.
+  "src/components/admin/recovery-readiness-card.tsx": {
+    CheckCircle2: 2,
+    "text-success": 1,
+  },
   "src/components/admin/recent-audit-preview.tsx": {
     CheckCircle2: 2,
     "text-success": 1,

src/app/admin/[section]/renderer.tsx

@@ -28,6 +28,7 @@ import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-s
 import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section";
 import { BackupsSection } from "@/components/admin/backups-section";
 import { OffhostBackupsSection } from "@/components/admin/offhost-backups-section";
+import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card";
 import { DangerZoneSection } from "@/components/admin/danger-zone-section";
 import { EncryptionSection } from "@/components/admin/encryption-section";
 import { GeneralSettingsSection } from "@/components/admin/general-settings-section";
@@ -166,10 +167,12 @@ export function AdminSectionRenderer({
         </SectionFrame>
       );
     case "backups":
-      // Two cards, two independent failure modes: the copies held in this
-      // database, then the copies that reached the operator's bucket.
+      // The verdict across every backup first, then two cards, two
+      // independent failure modes: the copies held in this database, then
+      // the copies that reached the operator's bucket.
       return (
         <SectionFrame>
+          <RecoveryReadinessCard />
           <BackupsSection />
           <OffhostBackupsSection />
         </SectionFrame>

src/app/admin/page.tsx

@@ -4,6 +4,7 @@ import { useAuth } from "@/hooks/use-auth";
 import { useTranslations } from "@/lib/i18n/context";
 import { QueryErrorRow } from "@/components/ui/query-error-row";
 import { AdminShell } from "@/components/admin/admin-shell";
+import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card";
 import { SystemStatusSummary } from "@/components/admin/system-status-summary";
 import { VersionTileSection } from "@/components/admin/version-tile-section";
 import { RecentAuditPreview } from "@/components/admin/recent-audit-preview";
@@ -49,6 +50,9 @@ export default function AdminOverviewPage() {
 
         <SystemStatusSummary />
 
+        {/* v1.46 — every backup signal, the recovery kit included. */}
+        <RecoveryReadinessCard />
+
         <RecentAuditPreview />
       </div>
     </AdminShell>

src/app/api/admin/recovery-readiness/route.ts

@@ -0,0 +1,24 @@
+/**
+ * GET /api/admin/recovery-readiness
+ *
+ * Every backup signal with its verdict, its time and where to fix it: the
+ * database dump, the weekly copy, the off-host copies, the last restore drill
+ * (no time window: a failed drill stays failed until a later one passes), the
+ * off-host setup and the recovery kit. The same judgement the hourly
+ * evaluation pushes to the admins (`src/lib/recovery/readiness-model.ts`).
+ * Cookie-only admin.
+ */
+import { apiHandler, requireAdmin } from "@/lib/api-handler";
+import { apiSuccess } from "@/lib/api-response";
+import { annotate } from "@/lib/logging/context";
+import { readRecoveryReadiness } from "@/lib/recovery/readiness";
+
+export const dynamic = "force-dynamic";
+
+export const GET = apiHandler(async () => {
+  await requireAdmin();
+  annotate({ action: { name: "admin.recoveryReadiness.status" } });
+  const readiness = await readRecoveryReadiness();
+  annotate({ meta: { recovery_readiness_state: readiness.state } });
+  return apiSuccess(readiness);
+});

src/app/api/notifications/preferences/route.ts

@@ -9,7 +9,10 @@ import {
   sanitiseZodIssues,
 } from "@/lib/api-response";
 import { notificationPreferenceSchema } from "@/lib/validations/notifications";
-import { EVENT_TYPES, CHANNEL_TYPE_LABELS } from "@/lib/notifications/types";
+import {
+  CHANNEL_TYPE_LABELS,
+  eventTypesForRole,
+} from "@/lib/notifications/types";
 import type { ChannelType } from "@/lib/notifications/types";
 import { decrypt, encrypt } from "@/lib/crypto";
 import { resolveEventDefaults } from "@/lib/notifications/event-defaults";
@@ -92,7 +95,9 @@ export const GET = apiHandler(async () => {
   return apiSuccess({
     channels: channelData,
     preferences,
-    eventTypes: EVENT_TYPES as unknown as string[],
+    // Admin-only events (backup health) are listed for admins only: nobody
+    // else is ever sent one, so a switch for it would switch nothing.
+    eventTypes: eventTypesForRole(user.role),
     // What an event does on a channel with no stored row, resolved for this
     // account by the same function the dispatcher asks. A client renders a
     // switch without a row from this map, never from its own copy of the

src/components/admin/__tests__/backups-restore-preview.test.tsx

@@ -53,7 +53,7 @@ vi.mock("@tanstack/react-query", () => ({
         schedule: {
           lastSuccessAt: "2026-09-01T03:00:00.000Z",
           lastSuccessAgeDays: 2,
-          staleAfterDays: 10,
+          staleAfterDays: 9,
           stale: false,
           lastRun: null,
           lastRunFailed: false,

src/components/admin/__tests__/backups-section-empty.test.tsx

@@ -6,7 +6,7 @@ import en from "../../../../messages/en.json";
 const healthySchedule = {
   lastSuccessAt: "2026-09-01T03:00:00.000Z",
   lastSuccessAgeDays: 2,
-  staleAfterDays: 10,
+  staleAfterDays: 9,
   stale: false,
   lastRun: {
     at: "2026-09-01T03:00:12.000Z",

src/components/admin/__tests__/recovery-readiness-card.test.tsx

@@ -0,0 +1,154 @@
+/**
+ * The recovery readiness card.
+ *
+ *   1. every signal renders as its own row with the server's state, its
+ *      sentence and its time;
+ *   2. a row that needs work links to where it is fixed, unless that is the
+ *      page already open; an ok or inactive row carries no link;
+ *   3. a grace period shows its due date rather than a time of an event;
+ *   4. a failed drill shows its reason with no time window: a failure months
+ *      old is still on the card;
+ *   5. the overall badge carries the worst state; the copy resolves in German.
+ *
+ * Mutation check: show the fix link on every row and case 2 goes red; drop
+ * the `detail` line and case 4 goes red.
+ */
+import { describe, expect, it, vi } from "vitest";
+import { renderToStaticMarkup } from "react-dom/server";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+
+import { I18nProvider } from "@/lib/i18n/context";
+import { queryKeys } from "@/lib/query-keys";
+import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card";
+import type {
+  ReadinessSignal,
+  RecoveryReadiness,
+} from "@/components/admin/use-recovery-readiness";
+import { READINESS_FIX_PATHS } from "@/lib/recovery/readiness-model";
+import { primeLocales } from "@/__tests__/helpers/prime-locales";
+
+primeLocales("de");
+
+const pathname = vi.hoisted(() => ({ current: "/admin" }));
+vi.mock("next/navigation", () => ({
+  useRouter: () => ({ push: vi.fn(), replace: vi.fn() }),
+  usePathname: () => pathname.current,
+}));
+
+function sig(
+  id: ReadinessSignal["id"],
+  state: ReadinessSignal["state"],
+  reason: string,
+  over: Partial<ReadinessSignal> = {},
+): ReadinessSignal {
+  return {
+    id,
+    state,
+    reason,
+    at: null,
+    params: {},
+    detail: null,
+    fixPath: READINESS_FIX_PATHS[id],
+    ...over,
+  };
+}
+
+const READINESS: RecoveryReadiness = {
+  state: "failing",
+  evaluatedAt: "2026-10-25T12:00:00.000Z",
+  signals: [
+    sig("database_dump", "inactive", "not_available"),
+    sig("weekly_copy", "attention", "stale", {
+      at: "2026-10-14T03:00:00.000Z",
+      params: { days: 11, accounts: 2 },
+    }),
+    sig("offhost_copies", "ok", "fresh", { at: "2026-10-25T02:40:00.000Z" }),
+    sig("restore_drill", "failing", "failed", {
+      at: "2026-07-01T04:20:00.000Z",
+      params: { failed: 1, checked: 4 },
+      detail: "1 of 4 backup objects failed. Restore drill: bad tag",
+    }),
+    sig("offhost_setup", "ok", "configured"),
+    sig("recovery_kit", "ok", "not_confirmed_grace", {
+      at: "2026-10-30T09:00:00.000Z",
+      params: { keys: "v2, API_TOKEN_HMAC_KEY", count: 2 },
+    }),
+  ],
+};
+
+function render(readiness: RecoveryReadiness, locale: "en" | "de" = "en") {
+  const client = new QueryClient();
+  client.setQueryData(queryKeys.adminRecoveryReadiness(), readiness);
+  return renderToStaticMarkup(
+    <QueryClientProvider client={client}>
+      <I18nProvider initialLocale={locale}>
+        <RecoveryReadinessCard />
+      </I18nProvider>
+    </QueryClientProvider>,
+  );
+}
+
+/** The markup of one signal's row. */
+function row(html: string, id: string): string {
+  const start = html.indexOf(`data-testid="recovery-signal-${id}"`);
+  expect(start).toBeGreaterThan(-1);
+  const end = html.indexOf("</li>", start);
+  return html.slice(start, end);
+}
+
+describe("RecoveryReadinessCard", () => {
+  it("renders every signal with its state and sentence", () => {
+    pathname.current = "/admin";
+    const html = render(READINESS);
+    for (const s of READINESS.signals) {
+      expect(row(html, s.id)).toContain(`data-state="${s.state}"`);
+    }
+    expect(row(html, "weekly_copy")).toContain(
+      "The oldest weekly copy is 11 days old. Accounts behind: 2.",
+    );
+    expect(row(html, "database_dump")).toContain(
+      "This version does not schedule a database dump.",
+    );
+  });
+
+  it("links a row that needs work to where it is fixed, and nothing else", () => {
+    pathname.current = "/admin";
+    const html = render(READINESS);
+    expect(row(html, "weekly_copy")).toContain('href="/admin/backups"');
+    expect(row(html, "restore_drill")).toContain('href="/admin/backups"');
+    expect(row(html, "recovery_kit")).toContain('href="/admin/encryption"');
+    expect(row(html, "offhost_copies")).not.toContain("href=");
+    expect(row(html, "database_dump")).not.toContain("href=");
+  });
+
+  it("does not link to the page already open", () => {
+    pathname.current = "/admin/backups";
+    const html = render(READINESS);
+    expect(row(html, "weekly_copy")).not.toContain("href=");
+    expect(row(html, "recovery_kit")).toContain('href="/admin/encryption"');
+  });
+
+  it("shows a grace period's due date", () => {
+    pathname.current = "/admin";
+    expect(row(render(READINESS), "recovery_kit")).toContain("Due by ");
+  });
+
+  it("keeps a months-old drill failure on the card with its reason", () => {
+    pathname.current = "/admin";
+    const drill = row(render(READINESS), "restore_drill");
+    expect(drill).toContain(
+      "The last restore drill failed: 1 of 4 copies could not be read back.",
+    );
+    expect(drill).toContain("Restore drill: bad tag");
+    expect(drill).toContain("Last drill: ");
+  });
+
+  it("carries the worst state on the badge, and speaks German", () => {
+    pathname.current = "/admin";
+    const html = render(READINESS);
+    expect(html).toContain('data-testid="recovery-readiness-state"');
+    expect(html).toContain('data-state="failing"');
+    expect(render({ ...READINESS, state: "ok" })).toContain(">Ready<");
+    expect(render(READINESS, "de")).toContain("Wiederherstellbarkeit");
+  });
+});

src/components/admin/recovery-readiness-card.tsx

@@ -0,0 +1,208 @@
+"use client";
+
+/**
+ * `<RecoveryReadinessCard>` — every backup signal on one card: the database
+ * dump, the weekly account copies, the off-host copies, the last restore
+ * drill, the off-host setup and the recovery kit. Each row says its verdict,
+ * the time it is about and where to fix it.
+ *
+ * No time window anywhere: a failed drill stays on this card until a later
+ * drill passes, where the failing-jobs card forgot it after 72 hours. The
+ * verdicts are the server's (`src/lib/recovery/readiness-model.ts`), the same
+ * ones the hourly evaluation pushes to every admin, so the card and the push
+ * cannot disagree.
+ *
+ * Shown on the admin overview and on the Backups page. A row whose fix lives
+ * on the page already open carries no link to itself.
+ */
+
+import Link from "next/link";
+import { usePathname } from "next/navigation";
+import {
+  AlertTriangle,
+  CheckCircle2,
+  Clock,
+  LifeBuoy,
+  Loader2,
+  MinusCircle,
+  XCircle,
+} from "lucide-react";
+
+import { SettingsCardHeader } from "@/components/settings/_card-header";
+import { SettingsCard } from "@/components/settings/settings-card";
+import { Badge } from "@/components/ui/badge";
+import { InlineAction } from "@/components/ui/inline-action";
+import { QueryErrorRow } from "@/components/ui/query-error-row";
+import { useFormatters, useTranslations } from "@/lib/i18n/context";
+import { cn } from "@/lib/utils";
+
+import {
+  useRecoveryReadiness,
+  type ReadinessSignal,
+  type ReadinessState,
+  type RecoveryReadiness,
+} from "./use-recovery-readiness";
+
+const STATE_ICON: Record<
+  ReadinessState,
+  React.ComponentType<{ className?: string; "aria-hidden"?: boolean }>
+> = {
+  ok: CheckCircle2,
+  attention: AlertTriangle,
+  failing: XCircle,
+  inactive: MinusCircle,
+};
+
+const STATE_TONE: Record<ReadinessState, string> = {
+  ok: "text-success",
+  attention: "text-warning",
+  failing: "text-destructive",
+  inactive: "text-muted-foreground",
+};
+
+/** A grace period is ok for the verdict, but still a step for the admin. */
+const isGrace = (signal: ReadinessSignal) => signal.reason.endsWith("_grace");
+
+function OverallBadge({
+  state,
+}: {
+  state: Exclude<ReadinessState, "inactive">;
+}) {
+  const { t } = useTranslations();
+  const label = {
+    ok: t("admin.recoveryReadiness.state.ok"),
+    attention: t("admin.recoveryReadiness.state.attention"),
+    failing: t("admin.recoveryReadiness.state.failing"),
+  }[state];
+  return (
+    <Badge
+      variant="outline"
+      data-testid="recovery-readiness-state"
+      data-state={state}
+      className={cn(
+        state === "attention" && "border-warning/40 text-warning",
+        state === "failing" && "border-destructive/40 text-destructive",
+      )}
+    >
+      {label}
+    </Badge>
+  );
+}
+
+function SignalRow({
+  signal,
+  currentPath,
+}: {
+  signal: ReadinessSignal;
+  currentPath: string | null;
+}) {
+  const { t } = useTranslations();
+  const fmt = useFormatters();
+  const grace = isGrace(signal);
+  const Icon = grace ? Clock : STATE_ICON[signal.state];
+  const tone = grace ? "text-info" : STATE_TONE[signal.state];
+  const date = signal.at ? fmt.dateTime(new Date(signal.at)) : null;
+  const showFix =
+    signal.state !== "inactive" &&
+    (signal.state !== "ok" || grace) &&
+    signal.fixPath !== currentPath;
+
+  return (
+    <li
+      className="flex items-start gap-3 py-3 first:pt-0 last:pb-0"
+      data-testid={`recovery-signal-${signal.id}`}
+      data-state={signal.state}
+    >
+      <Icon className={cn("mt-0.5 size-4 shrink-0", tone)} aria-hidden />
+      <div className="min-w-0 flex-1 space-y-1">
+        <p className="text-sm font-medium">
+          {t(`admin.recoveryReadiness.signal.${signal.id}`)}
+        </p>
+        <p className="text-sm">
+          {t(
+            `admin.recoveryReadiness.reason.${signal.id}.${signal.reason}`,
+            signal.params,
+          )}
+        </p>
+        {date ? (
+          <p className="text-muted-foreground text-xs">
+            {grace
+              ? t("admin.recoveryReadiness.dueBy", { date })
+              : t(`admin.recoveryReadiness.at.${signal.id}`, { date })}
+          </p>
+        ) : null}
+        {signal.detail && signal.state !== "ok" ? (
+          <p className="text-muted-foreground text-xs break-words">
+            {signal.detail}
+          </p>
+        ) : null}
+      </div>
+      {showFix ? (
+        <InlineAction asChild size="sm" className="shrink-0 font-medium">
+          <Link href={signal.fixPath}>
+            {t(`admin.recoveryReadiness.fix.${signal.id}`)}
+          </Link>
+        </InlineAction>
+      ) : null}
+    </li>
+  );
+}
+
+/** The list itself, for a readiness that has loaded. */
+export function RecoveryReadinessList({
+  readiness,
+  currentPath,
+}: {
+  readiness: RecoveryReadiness;
+  currentPath: string | null;
+}) {
+  return (
+    <ul className="divide-border divide-y" data-testid="recovery-readiness">
+      {readiness.signals.map((signal) => (
+        <SignalRow key={signal.id} signal={signal} currentPath={currentPath} />
+      ))}
+    </ul>
+  );
+}
+
+export function RecoveryReadinessCard() {
+  const { t } = useTranslations();
+  const currentPath = usePathname();
+  const { data, isError, error, refetch } = useRecoveryReadiness();
+
+  return (
+    <SettingsCard
+      as="section"
+      aria-labelledby="recovery-readiness-heading"
+      data-testid="recovery-readiness-card"
+    >
+      <SettingsCardHeader
+        icon={LifeBuoy}
+        titleId="recovery-readiness-heading"
+        title={t("admin.recoveryReadiness.title")}
+        description={t("admin.recoveryReadiness.description")}
+        status={data ? <OverallBadge state={data.state} /> : null}
+      />
+      {data ? (
+        <RecoveryReadinessList readiness={data} currentPath={currentPath} />
+      ) : isError ? (
+        <QueryErrorRow
+          inset
+          error={error}
+          message={t("admin.recoveryReadiness.loadError")}
+          onRetry={() => void refetch()}
+        />
+      ) : (
+        <div className="flex items-center gap-2">
+          <Loader2
+            className="text-muted-foreground size-4 animate-spin motion-reduce:animate-none"
+            aria-hidden="true"
+          />
+          <span className="text-muted-foreground text-sm">
+            {t("admin.recoveryReadiness.loading")}
+          </span>
+        </div>
+      )}
+    </SettingsCard>
+  );
+}

src/components/admin/use-recovery-readiness.ts

@@ -0,0 +1,24 @@
+"use client";
+
+import { useQuery } from "@tanstack/react-query";
+import { apiGet } from "@/lib/api/api-fetch";
+import { queryKeys } from "@/lib/query-keys";
+import type { RecoveryReadiness } from "@/lib/recovery/readiness-model";
+
+export type {
+  ReadinessSignal,
+  ReadinessState,
+  RecoveryReadiness,
+} from "@/lib/recovery/readiness-model";
+
+/** `GET /api/admin/recovery-readiness`; admin-only and cookie-only. */
+export function useRecoveryReadiness() {
+  return useQuery({
+    queryKey: queryKeys.adminRecoveryReadiness(),
+    queryFn: () =>
+      apiGet<RecoveryReadiness>("/api/admin/recovery-readiness", {
+        credentials: "include",
+      }),
+    staleTime: 60_000,
+  });
+}

src/lib/jobs/__tests__/backup-schedule-status.test.ts

@@ -50,6 +50,19 @@ describe("summariseBackupSchedule", () => {
     expect(status.lastRunFailed).toBe(false);
   });
 
+  it("draws the line where the readiness model does: past nine days", () => {
+    const at = (ms: number) =>
+      summariseBackupSchedule({
+        scheduledCreatedAt: [new Date(NOW.getTime() - ms)],
+        lastRun: null,
+        now: NOW,
+      }).stale;
+    const DAY = 24 * 60 * 60 * 1000;
+    expect(BACKUP_STALE_AFTER_DAYS).toBe(9);
+    expect(at(9 * DAY)).toBe(false);
+    expect(at(9 * DAY + 1)).toBe(true);
+  });
+
   it("surfaces a failed last run even while the copy is still in window", () => {
     const status = summariseBackupSchedule({
       scheduledCreatedAt: [daysAgo(3)],

src/lib/jobs/__tests__/restore-drill.test.ts

@@ -14,7 +14,12 @@ vi.mock("../report-worker-error", () => ({
   reportWorkerError: vi.fn().mockResolvedValue(undefined),
 }));
 
+vi.mock("../restore-drill-outcome", () => ({
+  recordRestoreDrillOutcome: vi.fn().mockResolvedValue(undefined),
+}));
+
 import { reportWorkerError } from "../report-worker-error";
+import { recordRestoreDrillOutcome } from "../restore-drill-outcome";
 
 const ENC_KEY =
   "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
@@ -257,6 +262,8 @@ describe("handleRestoreDrill", () => {
     vi.stubEnv("BACKUP_S3_ENDPOINT", "");
     await handleRestoreDrill([]);
     expect(reportWorkerError).not.toHaveBeenCalled();
+    // Nothing was checked, so there is no verdict to keep.
+    expect(recordRestoreDrillOutcome).not.toHaveBeenCalled();
   });
 
   it("pages via reportWorkerError when the drill fails", async () => {
@@ -270,5 +277,22 @@ describe("handleRestoreDrill", () => {
     expect(vi.mocked(reportWorkerError).mock.calls[0][0]).toBe(
       RESTORE_DRILL_QUEUE,
     );
+    // The failure is kept for the readiness tile, past pg-boss's retention.
+    expect(recordRestoreDrillOutcome).toHaveBeenCalledTimes(1);
+    expect(vi.mocked(recordRestoreDrillOutcome).mock.calls[0][0]).toMatchObject(
+      { ok: false, accountsChecked: 0, accountsFailed: 0 },
+    );
+  });
+
+  it("keeps a failed drill's verdict even when keeping it fails", async () => {
+    stubConfigEnv();
+    vi.stubEnv("BACKUP_ENCRYPTION_KEY", "not-a-key");
+    vi.mocked(recordRestoreDrillOutcome).mockRejectedValueOnce(
+      new Error("database away"),
+    );
+    const outcome = await handleRestoreDrill([]);
+    // The drill's own failure is still what the run reports.
+    expect(outcome.ok).toBe(false);
+    expect(reportWorkerError).toHaveBeenCalledTimes(1);
   });
 });

src/lib/jobs/backup-schedule-status.ts

@@ -23,13 +23,15 @@
  * alive.
  */
 import type { LastQueueRun } from "@/lib/jobs/job-failures";
+import { WEEKLY_COPY_ATTENTION_AFTER_DAYS } from "@/lib/recovery/readiness-model";
 
 /**
  * How old the newest scheduled copy may get before the page says so. The cron
- * is weekly, so seven days is normal and eight is a rounding error; ten means
- * at least one Sunday produced nothing.
+ * is weekly, so seven days is normal and eight is a long week; past nine at
+ * least one Sunday produced nothing. The recovery readiness model draws its
+ * attention line at the same instant, so this page and the push agree.
  */
-export const BACKUP_STALE_AFTER_DAYS = 10;
+export const BACKUP_STALE_AFTER_DAYS = WEEKLY_COPY_ATTENTION_AFTER_DAYS;
 
 const DAY_MS = 24 * 60 * 60 * 1000;
 
@@ -71,7 +73,9 @@ export function summariseBackupSchedule(args: {
     // new — the page's empty state already says there is nothing there, and
     // calling a fresh install "overdue" would train the operator to ignore the
     // warning that matters.
-    stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS,
+    stale:
+      newest !== null &&
+      args.now.getTime() - newest.getTime() > BACKUP_STALE_AFTER_DAYS * DAY_MS,
     lastRun: args.lastRun,
     lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed",
   };

src/lib/jobs/job-outcome.ts

@@ -234,6 +234,13 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
   "restore_drill_plaintext_bytes",
   "restore_drill_unchecked",
   "restore_drill_stale",
+  // The hourly recovery readiness evaluation: the overall verdict, how many
+  // signals need attention or are failing, notices sent, admins reached.
+  "recovery_readiness_admins",
+  "recovery_readiness_attention",
+  "recovery_readiness_failing",
+  "recovery_readiness_notices",
+  "recovery_readiness_state",
   "restore_fail_cause",
   "restore_failed",
   "restore_measurements",

src/lib/jobs/queue-runtime.ts

@@ -368,6 +368,9 @@ export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
   "geo-backfill": short("At most 500 audit rows per pass."),
   "geolite2-fetch": short("One database download."),
   "tls-pin-monitor": short("One TLS handshake."),
+  "recovery-readiness": short(
+    "A few indexed reads, one row per account from two backup ledgers, and at most one notice per signal.",
+  ),
   "integration-watchdog-hourly": short(
     "A few indexed reads per account with a connection and no provider call; a 14-day history count only for a silent source.",
   ),

src/lib/jobs/recovery-readiness-queue.ts

@@ -0,0 +1,12 @@
+/**
+ * The queue facts of `recovery-readiness.ts`: its name and schedule. They live
+ * apart from the job so the worker registry can read them without loading it.
+ */
+export const RECOVERY_READINESS_QUEUE = "recovery-readiness";
+
+/**
+ * Hourly at :33, a minute no other schedule uses. Hourly is far inside every
+ * threshold the evaluation judges (36 hours for the dump, two nights for the
+ * off-host copies), and a notice is sent once per change whatever the cadence.
+ */
+export const RECOVERY_READINESS_CRON = "33 * * * *";

src/lib/jobs/recovery-readiness.ts

@@ -0,0 +1,42 @@
+/**
+ * The hourly recovery readiness evaluation: read every backup signal, judge
+ * it, and push each change to the admins (`readiness-notify.ts`). The tile on
+ * the admin pages reads the same verdict live; this job is what makes it a
+ * push rather than a page somebody has to open.
+ *
+ * Cheap and bounded: a handful of indexed reads, one row per account for the
+ * two backup ledgers, and at most one notice per signal per run. The next
+ * tick is the retry.
+ */
+import type { Job } from "pg-boss";
+
+import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
+import { withBackgroundEvent } from "@/lib/logging/background";
+import { readRecoveryReadiness } from "@/lib/recovery/readiness";
+import { notifyReadinessChanges } from "@/lib/recovery/readiness-notify";
+
+export async function handleRecoveryReadiness(
+  jobs: Job<object>[],
+): Promise<JobOutcome> {
+  void jobs;
+  return withBackgroundEvent("job.recovery_readiness", async (evt) => {
+    try {
+      const readiness = await readRecoveryReadiness();
+      const outcome = await notifyReadinessChanges(readiness);
+      const counts = (state: string) =>
+        readiness.signals.filter((s) => s.state === state).length;
+      const did = {
+        recovery_readiness_state: readiness.state,
+        recovery_readiness_attention: counts("attention"),
+        recovery_readiness_failing: counts("failing"),
+        recovery_readiness_notices: outcome.sent.length,
+        recovery_readiness_admins: outcome.admins,
+      };
+      for (const [key, value] of Object.entries(did)) evt.addMeta(key, value);
+      return jobDone(did);
+    } catch (err) {
+      evt.addWarning(`recovery-readiness failed: ${err}`);
+      return jobFailed("recovery readiness evaluation failed", err);
+    }
+  });
+}

src/lib/jobs/registry/maintenance.ts

@@ -169,6 +169,10 @@ import {
 } from "@/lib/jobs/pr-detection";
 import { PULSE_DAY_MEAN_QUEUE } from "@/lib/jobs/pulse-day-mean";
 import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared";
+import {
+  RECOVERY_READINESS_CRON,
+  RECOVERY_READINESS_QUEUE,
+} from "@/lib/jobs/recovery-readiness-queue";
 import {
   RESTORE_DRILL_CRON,
   RESTORE_DRILL_QUEUE,
@@ -343,6 +347,17 @@ const jobs: readonly JobDefinition[] = [
     load: async () =>
       (await import("@/lib/jobs/restore-drill")).handleRestoreDrill,
   }),
+  // Hourly: judge every backup signal and push each change to the admins.
+  // Single-flight and no retries; the next tick is the retry, and the notice
+  // claim keeps two evaluations from sending the same notice.
+  defineJob({
+    queue: RECOVERY_READINESS_QUEUE,
+    cron: RECOVERY_READINESS_CRON,
+    scheduleOptions: cronIsTheRetry,
+    work: { localConcurrency: 1 },
+    load: async () =>
+      (await import("@/lib/jobs/recovery-readiness")).handleRecoveryReadiness,
+  }),
   // The nightly backstop for the purge an account deletion kicks at once.
   prune(
     OFFHOST_PURGE_QUEUE,

src/lib/jobs/restore-drill-outcome.ts

@@ -0,0 +1,54 @@
+/**
+ * The monthly restore drill's verdict, kept until the next drill replaces it.
+ *
+ * The drill used to say what it found only through `reportWorkerError`
+ * (stderr, and GlitchTip where configured) and through the pg-boss row of its
+ * run. The row is gone after seven days and the admin failing-jobs card reads
+ * 72 hours of it, so a drill that failed on the first of the month was on
+ * screen until the fourth and invisible for the rest of it. One row, written
+ * at the end of every drill that had something to check, is what the recovery
+ * readiness tile and its push read instead, with no window at all.
+ *
+ * A failure stays the verdict until a later drill passes: `lastFailureAt`
+ * newer than `lastSuccessAt` is the rule (`evaluateRestoreDrill`).
+ */
+import { prisma } from "@/lib/db";
+import { redactSecrets } from "@/lib/logging/redact";
+
+/** The reason is a line on a status tile, not a stack trace. */
+const MAX_REASON_CHARS = 300;
+
+export interface RestoreDrillResult {
+  ok: boolean;
+  accountsChecked: number;
+  accountsFailed: number;
+  /** Why it failed; ignored when `ok`. */
+  reason?: string | null;
+}
+
+export async function recordRestoreDrillOutcome(
+  result: RestoreDrillResult,
+  at: Date = new Date(),
+): Promise<void> {
+  const reason = result.ok
+    ? null
+    : (() => {
+        const text = redactSecrets((result.reason ?? "").trim());
+        return text.length > MAX_REASON_CHARS
+          ? `${text.slice(0, MAX_REASON_CHARS)}…`
+          : text || null;
+      })();
+  const verdict = result.ok
+    ? { lastSuccessAt: at }
+    : { lastFailureAt: at, lastFailureReason: reason };
+  const counts = {
+    lastRunAt: at,
+    accountsChecked: result.accountsChecked,
+    accountsFailed: result.accountsFailed,
+  };
+  await prisma.restoreDrillOutcome.upsert({
+    where: { id: "singleton" },
+    create: { id: "singleton", ...counts, ...verdict },
+    update: { ...counts, ...verdict },
+  });
+}

src/lib/jobs/restore-drill.ts

@@ -15,8 +15,10 @@
  * application key it needs. Each account is reported on its own. It deliberately performs NO database restore — the
  * drill validates the artefact, not the import path. The result is
  * surfaced through the wide-event meta on success and through
- * `reportWorkerError` (stderr + GlitchTip) on any failure, so a broken
- * backup chain pages the operator eleven months before it matters.
+ * `reportWorkerError` (stderr + GlitchTip) on any failure, and the verdict is
+ * kept (`restore-drill-outcome.ts`) for the recovery readiness tile and its
+ * push to the admins, so a broken backup chain reaches the operator eleven
+ * months before it matters.
  *
  * Schedule: 04:11 on the 1st of each month (`11 4 1 * *`), after the
  * 02:30 nightly upload and the 03:xx cleanup window, on a minute slot
@@ -45,6 +47,10 @@ import {
   type S3Like,
 } from "@/lib/jobs/offhost-backup";
 import { reportWorkerError } from "@/lib/jobs/report-worker-error";
+import {
+  recordRestoreDrillOutcome,
+  type RestoreDrillResult,
+} from "@/lib/jobs/restore-drill-outcome";
 import { jobBudget } from "@/lib/jobs/job-budget";
 import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
 import { withBackgroundEvent } from "@/lib/logging/background";
@@ -254,6 +260,21 @@ export async function runRestoreDrill(
   };
 }
 
+/**
+ * Keep the verdict for the readiness tile. A failed write is a warning on the
+ * run, never a failed drill: the drill's own reports already went out.
+ */
+async function keepOutcome(
+  evt: { addWarning: (message: string) => void },
+  result: RestoreDrillResult,
+): Promise<void> {
+  try {
+    await recordRestoreDrillOutcome(result);
+  } catch (err) {
+    evt.addWarning(`restore-drill outcome not recorded: ${err}`);
+  }
+}
+
 /**
  * pg-boss handler. Mirrors the off-host uploader's posture: a
  * not-configured deployment skips with a wide-event warning (most
@@ -323,6 +344,17 @@ export async function handleRestoreDrill(
           { dateKey: report.dateKey, unchecked: report.unchecked },
         );
       }
+      await keepOutcome(evt, {
+        ok: report.failed.length === 0,
+        accountsChecked: report.accounts.length,
+        accountsFailed: report.failed.length,
+        reason:
+          report.failed.length === 0
+            ? null
+            : `${report.failed.length} of ${report.accounts.length} backup objects failed. ${
+                report.failed[0].error ?? report.failed[0].objectKey
+              }`,
+      });
       if (report.failed.length > 0) {
         // Already paged above, per account. The queue does not retry: a
         // retry reads the same objects and would page the same accounts again.
@@ -346,6 +378,12 @@ export async function handleRestoreDrill(
       }
       evt.addWarning(`restore-drill failed: ${err}`);
       await reportWorkerError(RESTORE_DRILL_QUEUE, err);
+      await keepOutcome(evt, {
+        ok: false,
+        accountsChecked: 0,
+        accountsFailed: 0,
+        reason: err instanceof Error ? err.message : String(err),
+      });
       return jobFailed("restore drill failed", err);
     }
   });

src/lib/jobs/subsystem-surface.ts

@@ -100,6 +100,7 @@ const subsystemSurface = {
   "step-up-elevation-cleanup": { audience: "system" },
   "data-backup-offhost": { audience: "system" },
   "data-restore-drill": { audience: "system" },
+  "recovery-readiness": { audience: "system" },
   "offhost-backup-purge": { audience: "system" },
   "backup-restore": { audience: "system" },
   "host-metric-sample": { audience: "system" },

src/lib/notifications/tap-target.ts

@@ -51,6 +51,7 @@ export const NOTIFICATION_TAP_PATHS: Record<EventType, string | null> = {
   SECURITY_ALERT: "/settings/security",
   DAILY_BRIEFING: "/",
   SHARED_RECORD_INTAKE: "/medications",
+  BACKUP_HEALTH: "/admin/backups",
 };
 
 const MAX_PATH_LENGTH = 512;

src/lib/notifications/types.ts

@@ -125,6 +125,13 @@ export const EVENT_TYPES = [
   // dispatcher addresses a userId, and under record substitution that userId
   // is the owner by design). Default ON, see EVENT_DEFAULT_ENABLED.
   "SHARED_RECORD_INTAKE",
+  // Recovery readiness, for admins only: a backup signal turned to
+  // attention or failing, is still failing a day later, or is back to normal
+  // (`src/lib/recovery/readiness-notify.ts`). Default ON: the whole point is
+  // that a stopped backup is pushed rather than found on a page weeks later.
+  // An admin routes it per channel in the /notifications matrix, where only
+  // admins see the row (`eventTypesForRole`).
+  "BACKUP_HEALTH",
 ] as const;
 export type EventType = (typeof EVENT_TYPES)[number];
 
@@ -214,8 +221,21 @@ export const EVENT_DEFAULT_ENABLED: Record<EventType, boolean> = {
   // sits in the /notifications matrix from day one, so an owner who finds it
   // noisy silences it per channel with one switch.
   SHARED_RECORD_INTAKE: true,
+  // ON by default; see the note on `EVENT_TYPES`. Only admins are ever
+  // addressed, and at most once per state change per signal.
+  BACKUP_HEALTH: true,
 };
 
+/** Event types only an admin is ever sent, and only an admin is shown. */
+export const ADMIN_ONLY_EVENT_TYPES: readonly EventType[] = ["BACKUP_HEALTH"];
+
+/** The event types a person with `role` can receive, in matrix order. */
+export function eventTypesForRole(role: string): EventType[] {
+  return role === "ADMIN"
+    ? [...EVENT_TYPES]
+    : EVENT_TYPES.filter((type) => !ADMIN_ONLY_EVENT_TYPES.includes(type));
+}
+
 export const CHANNEL_TYPE_LABELS: Record<ChannelType, string> = {
   TELEGRAM: "Telegram",
   NTFY: "ntfy",

src/lib/recovery/__tests__/readiness-model.test.ts

@@ -0,0 +1,424 @@
+/**
+ * The recovery readiness thresholds, one signal at a time, on both sides of
+ * every line the model draws.
+ *
+ * Mutation check: move any threshold by one unit (35 h for the dump, 8 or 13
+ * days for the weekly copy, 6 days of setup grace) and the matching
+ * "just inside" or "just past" case goes red.
+ */
+import { describe, expect, it } from "vitest";
+
+import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
+import {
+  DUMP_FAILING_AFTER_HOURS,
+  evaluateDatabaseDump,
+  evaluateOffhostCopies,
+  evaluateOffhostSetup,
+  evaluateRecoveryKit,
+  evaluateRecoveryReadiness,
+  evaluateRestoreDrill,
+  evaluateWeeklyCopy,
+  READINESS_FIX_PATHS,
+  READINESS_SIGNALS,
+  SETUP_GRACE_DAYS,
+  WEEKLY_COPY_ATTENTION_AFTER_DAYS,
+  WEEKLY_COPY_FAILING_AFTER_DAYS,
+  type ReadinessInputs,
+} from "@/lib/recovery/readiness-model";
+
+const NOW = new Date("2026-10-25T12:00:00Z");
+const HOUR = 3_600_000;
+const DAY = 24 * HOUR;
+const ago = (ms: number) => new Date(NOW.getTime() - ms);
+
+/** A dump report in the dump module's own shape. */
+function dump(
+  over: Partial<DatabaseBackupHealth> & {
+    lastGoodAt?: Date | null;
+    lastFailureAt?: Date | null;
+    lastFailureReason?: string;
+  } = {},
+): DatabaseBackupHealth {
+  const {
+    lastGoodAt = ago(2 * HOUR),
+    lastFailureAt = null,
+    lastFailureReason,
+    ...rest
+  } = over;
+  return {
+    enabled: true,
+    directory: "/data/db-backups",
+    localDirWritable: true,
+    offhostConfigured: false,
+    lastGood: lastGoodAt
+      ? {
+          at: lastGoodAt.toISOString(),
+          fileName: "healthlog.dump.enc",
+          sizeBytes: 1024,
+          local: true,
+          offhost: false,
+        }
+      : null,
+    lastFailure: lastFailureAt
+      ? {
+          at: lastFailureAt.toISOString(),
+          error: lastFailureReason ?? "failed",
+        }
+      : null,
+    ...rest,
+  };
+}
+
+function inputs(over: Partial<ReadinessInputs> = {}): ReadinessInputs {
+  return {
+    now: NOW,
+    firstAdminAt: ago(30 * DAY),
+    dump: null,
+    dumpWatchedSince: null,
+    weekly: { accounts: [{ createdAt: ago(60 * DAY), lastCopyAt: ago(DAY) }] },
+    offhost: {
+      configured: true,
+      accounts: [
+        { lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) },
+      ],
+    },
+    drill: null,
+    kit: { confirmedAt: ago(10 * DAY), unconfirmed: [] },
+    ...over,
+  };
+}
+
+describe("database dump", () => {
+  const evalDump = (d: ReadinessInputs["dump"], watched: Date | null = null) =>
+    evaluateDatabaseDump({ now: NOW, dump: d, dumpWatchedSince: watched });
+
+  it("does not apply while no dump is scheduled, or while it is switched off", () => {
+    expect(evalDump(null)).toMatchObject({
+      state: "inactive",
+      reason: "not_available",
+    });
+    expect(evalDump(dump({ enabled: false }))).toMatchObject({
+      state: "inactive",
+      reason: "disabled",
+    });
+  });
+
+  it("fails when its status cannot be read", () => {
+    expect(evalDump("unreadable")).toMatchObject({
+      state: "failing",
+      reason: "unreadable",
+    });
+  });
+
+  it("is ok up to 36 hours and failing past them", () => {
+    const inside = evalDump(
+      dump({ lastGoodAt: ago(DUMP_FAILING_AFTER_HOURS * HOUR - 1) }),
+    );
+    expect(inside).toMatchObject({ state: "ok", reason: "fresh" });
+    const past = evalDump(
+      dump({ lastGoodAt: ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1) }),
+    );
+    expect(past).toMatchObject({
+      state: "failing",
+      reason: "stale",
+      params: { hours: 36 },
+    });
+    expect(past.at).toBe(
+      ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1).toISOString(),
+    );
+  });
+
+  it("calls a recent failure after a recent dump attention, with the reason", () => {
+    const s = evalDump(
+      dump({ lastFailureAt: ago(HOUR), lastFailureReason: "pg_dump exited 1" }),
+    );
+    expect(s).toMatchObject({
+      state: "attention",
+      reason: "last_failed",
+      detail: "pg_dump exited 1",
+    });
+  });
+
+  it("fails when a dump can be kept nowhere, and accepts a bucket-only setup", () => {
+    expect(evalDump(dump({ localDirWritable: false }))).toMatchObject({
+      state: "failing",
+      reason: "nowhere_to_write",
+      params: { directory: "/data/db-backups" },
+    });
+    expect(
+      evalDump(dump({ localDirWritable: false, offhostConfigured: true }))
+        .state,
+    ).toBe("ok");
+  });
+
+  it("gives a newly switched-on dump 36 hours before calling it failing", () => {
+    const fresh = dump({ lastGoodAt: null });
+    expect(evalDump(fresh, ago(HOUR))).toMatchObject({
+      state: "ok",
+      reason: "first_pending",
+    });
+    expect(evalDump(fresh, null)).toMatchObject({
+      state: "ok",
+      reason: "first_pending",
+    });
+    expect(evalDump(fresh, ago(37 * HOUR))).toMatchObject({
+      state: "failing",
+      reason: "never_ran",
+    });
+    expect(
+      evalDump(dump({ lastGoodAt: null, lastFailureAt: ago(HOUR) }), ago(HOUR)),
+    ).toMatchObject({ state: "failing", reason: "never_succeeded" });
+  });
+});
+
+describe("weekly copy", () => {
+  const weekly = (
+    ...accounts: Array<{ createdAt: Date; lastCopyAt: Date | null }>
+  ) => evaluateWeeklyCopy({ now: NOW, weekly: { accounts } });
+
+  it("is attention past 9 days and failing past 14, judged by the worst account", () => {
+    const fresh = { createdAt: ago(90 * DAY), lastCopyAt: ago(DAY) };
+    expect(
+      weekly(fresh, { createdAt: ago(90 * DAY), lastCopyAt: ago(9 * DAY - 1) })
+        .state,
+    ).toBe("ok");
+    expect(
+      weekly(fresh, {
+        createdAt: ago(90 * DAY),
+        lastCopyAt: ago(WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY + 1),
+      }),
+    ).toMatchObject({
+      state: "attention",
+      reason: "stale",
+      params: { days: 9, accounts: 1 },
+    });
+    expect(
+      weekly(fresh, {
+        createdAt: ago(90 * DAY),
+        lastCopyAt: ago(WEEKLY_COPY_FAILING_AFTER_DAYS * DAY - 1),
+      }).state,
+    ).toBe("attention");
+    expect(
+      weekly(fresh, {
+        createdAt: ago(90 * DAY),
+        lastCopyAt: ago(WEEKLY_COPY_FAILING_AFTER_DAYS * DAY + 1),
+      }),
+    ).toMatchObject({ state: "failing", params: { days: 14 } });
+  });
+
+  it("ages an account without any copy from its creation", () => {
+    expect(weekly({ createdAt: ago(3 * DAY), lastCopyAt: null })).toMatchObject(
+      {
+        state: "ok",
+        reason: "first_pending",
+      },
+    );
+    expect(
+      weekly({ createdAt: ago(20 * DAY), lastCopyAt: null }),
+    ).toMatchObject({
+      state: "failing",
+      reason: "never",
+    });
+  });
+
+  it("does not apply without accounts", () => {
+    expect(weekly().state).toBe("inactive");
+  });
+});
+
+describe("off-host copies", () => {
+  const offhost = (
+    configured: boolean,
+    ...accounts: Array<{
+      lastAttemptAt: Date | null;
+      lastSuccessAt: Date | null;
+    }>
+  ) => evaluateOffhostCopies({ now: NOW, offhost: { configured, accounts } });
+
+  it("does not apply while no bucket is configured", () => {
+    expect(offhost(false).state).toBe("inactive");
+  });
+
+  it("calls any account more than two nights behind attention", () => {
+    const fresh = {
+      lastAttemptAt: ago(5 * HOUR),
+      lastSuccessAt: ago(5 * HOUR),
+    };
+    // One missed night (inside two periods plus the grace) is not yet news.
+    expect(
+      offhost(true, fresh, {
+        lastAttemptAt: ago(HOUR),
+        lastSuccessAt: ago(40 * HOUR),
+      }).state,
+    ).toBe("ok");
+    expect(
+      offhost(true, fresh, {
+        lastAttemptAt: ago(HOUR),
+        lastSuccessAt: ago(2 * DAY + 7 * HOUR),
+      }),
+    ).toMatchObject({
+      state: "attention",
+      reason: "stale",
+      params: { accounts: 1 },
+    });
+  });
+
+  it("counts an account a run reached and never wrote as behind", () => {
+    expect(
+      offhost(true, { lastAttemptAt: ago(HOUR), lastSuccessAt: null }),
+    ).toMatchObject({ state: "attention", params: { accounts: 1 } });
+  });
+
+  it("waits for the first run rather than calling the cohort behind", () => {
+    expect(
+      offhost(true, { lastAttemptAt: null, lastSuccessAt: null }),
+    ).toMatchObject({
+      state: "ok",
+      reason: "first_pending",
+    });
+  });
+});
+
+describe("restore drill", () => {
+  const drill = (d: ReadinessInputs["drill"], configured = true) =>
+    evaluateRestoreDrill({ offhost: { configured, accounts: [] }, drill: d });
+  const run = {
+    lastRunAt: ago(5 * DAY),
+    lastSuccessAt: null as Date | null,
+    lastFailureAt: null as Date | null,
+    lastFailureReason: null as string | null,
+    accountsChecked: 3,
+    accountsFailed: 0,
+  };
+
+  it("stays failing after a failure, with no time window, until a later success", () => {
+    const failed = {
+      ...run,
+      lastFailureAt: ago(60 * DAY),
+      accountsFailed: 1,
+      lastFailureReason: "bad tag",
+    };
+    expect(drill(failed)).toMatchObject({
+      state: "failing",
+      reason: "failed",
+      params: { failed: 1, checked: 3 },
+      detail: "bad tag",
+    });
+    expect(drill({ ...failed, lastSuccessAt: ago(61 * DAY) }).state).toBe(
+      "failing",
+    );
+    expect(drill({ ...failed, lastSuccessAt: ago(DAY) })).toMatchObject({
+      state: "ok",
+      reason: "passed",
+    });
+  });
+
+  it("names a run that checked nothing", () => {
+    expect(
+      drill({ ...run, accountsChecked: 0, lastFailureAt: ago(DAY) }).reason,
+    ).toBe("failed_run");
+  });
+
+  it("does not apply without a bucket, and is ok before its first run", () => {
+    expect(drill(null, false).state).toBe("inactive");
+    expect(drill(null)).toMatchObject({ state: "ok", reason: "not_run_yet" });
+  });
+});
+
+describe("off-host setup", () => {
+  const setup = (configured: boolean, firstAdminAt: Date | null) =>
+    evaluateOffhostSetup({
+      now: NOW,
+      firstAdminAt,
+      offhost: { configured, accounts: [] },
+    });
+
+  it("asks for a bucket 7 days after the first admin exists", () => {
+    expect(setup(false, ago(SETUP_GRACE_DAYS * DAY - 1))).toMatchObject({
+      state: "ok",
+      reason: "not_configured_grace",
+    });
+    expect(setup(false, ago(SETUP_GRACE_DAYS * DAY + 1))).toMatchObject({
+      state: "attention",
+      reason: "not_configured",
+    });
+    expect(setup(true, ago(90 * DAY)).state).toBe("ok");
+    expect(setup(false, null).state).toBe("inactive");
+  });
+});
+
+describe("recovery kit", () => {
+  const kit = (
+    confirmedAt: Date | null,
+    unconfirmed: Array<{
+      kind: "application" | "backup" | "token";
+      id: string;
+    }>,
+    firstAdminAt: Date | null = ago(30 * DAY),
+  ) =>
+    evaluateRecoveryKit({
+      now: NOW,
+      firstAdminAt,
+      kit: { confirmedAt, unconfirmed },
+    });
+
+  it("is ok while the confirmation covers every key", () => {
+    expect(kit(ago(DAY), [])).toMatchObject({
+      state: "ok",
+      reason: "confirmed",
+    });
+  });
+
+  it("names a needed key nobody confirmed, at once once a kit was confirmed", () => {
+    expect(kit(ago(DAY), [{ kind: "application", id: "v1" }])).toMatchObject({
+      state: "attention",
+      reason: "out_of_date",
+      params: { keys: "v1", count: 1 },
+    });
+  });
+
+  it("gives a never-confirmed kit 7 days after setup", () => {
+    const keys = [
+      { kind: "application" as const, id: "v1" },
+      { kind: "token" as const, id: "API_TOKEN_HMAC_KEY" },
+    ];
+    expect(kit(null, keys, ago(SETUP_GRACE_DAYS * DAY - 1))).toMatchObject({
+      state: "ok",
+      reason: "not_confirmed_grace",
+    });
+    expect(kit(null, keys, ago(SETUP_GRACE_DAYS * DAY + 1))).toMatchObject({
+      state: "attention",
+      reason: "not_confirmed",
+      params: { keys: "v1, API_TOKEN_HMAC_KEY", count: 2 },
+    });
+  });
+});
+
+describe("the whole verdict", () => {
+  it("is the worst applying signal, and inactive ones never count", () => {
+    expect(evaluateRecoveryReadiness(inputs()).state).toBe("ok");
+    expect(
+      evaluateRecoveryReadiness(
+        inputs({ dump: dump({ lastGoodAt: ago(40 * HOUR) }) }),
+      ).state,
+    ).toBe("failing");
+    expect(
+      evaluateRecoveryReadiness(
+        inputs({
+          kit: {
+            confirmedAt: ago(DAY),
+            unconfirmed: [{ kind: "backup", id: "abc" }],
+          },
+        }),
+      ).state,
+    ).toBe("attention");
+  });
+
+  it("reports every signal once, in order, each with a fix path", () => {
+    const r = evaluateRecoveryReadiness(inputs());
+    expect(r.signals.map((s) => s.id)).toEqual([...READINESS_SIGNALS]);
+    for (const s of r.signals)
+      expect(s.fixPath).toBe(READINESS_FIX_PATHS[s.id]);
+    expect(r.evaluatedAt).toBe(NOW.toISOString());
+  });
+});

src/lib/recovery/__tests__/readiness-notify.test.ts

@@ -0,0 +1,287 @@
+/**
+ * The notice state machine: once per state change per signal, a recovery
+ * notice once, a reminder at most once a day while failing, nothing else.
+ *
+ * Two layers. `decideNotice` is pure and pinned case by case.
+ * `notifyReadinessChanges` runs against an in-memory stand-in for its one
+ * table that honours the compare-and-set exactly as Postgres would (a row
+ * moves only from the values the caller read), so the hourly sequence and the
+ * two-workers race are proven on the real claim logic rather than assumed.
+ *
+ * Mutation check: drop the `last !== current` comparison and the "same state
+ * twice" case sends twice; drop the CAS `where` on `notifiedState` and the
+ * race sends two notices.
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const store = vi.hoisted(() => ({
+  notices: new Map<
+    string,
+    {
+      signal: string;
+      notifiedState: string | null;
+      notifiedAt: Date | null;
+      watchedSince: Date | null;
+    }
+  >(),
+  admins: [] as Array<{ id: string }>,
+}));
+
+const same = (a: Date | null, b: Date | null | undefined) =>
+  (a?.getTime() ?? null) === (b?.getTime() ?? null);
+
+vi.mock("@/lib/db", () => ({
+  prisma: {
+    user: { findMany: vi.fn(async () => store.admins) },
+    recoveryReadinessNotice: {
+      createMany: vi.fn(
+        async ({ data }: { data: Array<{ signal: string }> }) => {
+          for (const { signal } of data) {
+            if (!store.notices.has(signal)) {
+              store.notices.set(signal, {
+                signal,
+                notifiedState: null,
+                notifiedAt: null,
+                watchedSince: null,
+              });
+            }
+          }
+          return { count: data.length };
+        },
+      ),
+      findMany: vi.fn(async () =>
+        [...store.notices.values()].map((row) => ({ ...row })),
+      ),
+      updateMany: vi.fn(
+        async ({
+          where,
+          data,
+        }: {
+          where: {
+            signal: string;
+            notifiedState?: string | null;
+            notifiedAt?: Date | null;
+            watchedSince?: Date | null;
+          };
+          data: Record<string, unknown>;
+        }) => {
+          const row = store.notices.get(where.signal);
+          if (!row) return { count: 0 };
+          if (
+            "notifiedState" in where &&
+            row.notifiedState !== where.notifiedState
+          )
+            return { count: 0 };
+          if ("notifiedAt" in where && !same(row.notifiedAt, where.notifiedAt))
+            return { count: 0 };
+          if (
+            "watchedSince" in where &&
+            !same(row.watchedSince, where.watchedSince)
+          )
+            return { count: 0 };
+          Object.assign(row, data);
+          return { count: 1 };
+        },
+      ),
+    },
+  },
+}));
+
+const dispatch = vi.hoisted(() =>
+  vi.fn(async (opts: Record<string, unknown>) => {
+    void opts;
+  }),
+);
+vi.mock("@/lib/notifications/dispatch-localised", () => ({
+  dispatchLocalisedNotification: dispatch,
+}));
+
+import {
+  decideNotice,
+  FAILING_REMINDER_HOURS,
+  notifyReadinessChanges,
+} from "@/lib/recovery/readiness-notify";
+import {
+  READINESS_FIX_PATHS,
+  READINESS_SIGNALS,
+  type ReadinessSignal,
+  type ReadinessState,
+  type RecoveryReadiness,
+} from "@/lib/recovery/readiness-model";
+
+const T0 = new Date("2026-10-25T12:00:00Z");
+const HOUR = 3_600_000;
+const at = (hours: number) => new Date(T0.getTime() + hours * HOUR);
+
+describe("decideNotice", () => {
+  it("announces a change to attention or failing once", () => {
+    expect(decideNotice(null, "attention", T0)).toEqual({
+      send: "problem",
+      next: { notifiedState: "attention", notifiedAt: T0 },
+    });
+    const told = { notifiedState: "attention" as const, notifiedAt: T0 };
+    expect(decideNotice(told, "attention", at(30)).send).toBeNull();
+    expect(decideNotice(told, "failing", at(1)).send).toBe("problem");
+  });
+
+  it("reminds a failing signal once a day, not sooner", () => {
+    const told = { notifiedState: "failing" as const, notifiedAt: T0 };
+    expect(
+      decideNotice(told, "failing", at(FAILING_REMINDER_HOURS - 1)).send,
+    ).toBeNull();
+    expect(decideNotice(told, "failing", at(FAILING_REMINDER_HOURS))).toEqual({
+      send: "reminder",
+      next: {
+        notifiedState: "failing",
+        notifiedAt: at(FAILING_REMINDER_HOURS),
+      },
+    });
+  });
+
+  it("announces a recovery once, and says nothing about an ok that was never broken", () => {
+    const told = { notifiedState: "failing" as const, notifiedAt: T0 };
+    expect(decideNotice(told, "ok", at(2)).send).toBe("recovered");
+    expect(
+      decideNotice({ notifiedState: "ok", notifiedAt: at(2) }, "ok", at(3)),
+    ).toEqual({
+      send: null,
+      next: null,
+    });
+    expect(decideNotice(null, "ok", T0)).toEqual({ send: null, next: null });
+  });
+
+  it("forgets silently when a signal stops applying", () => {
+    expect(
+      decideNotice(
+        { notifiedState: "failing", notifiedAt: T0 },
+        "inactive",
+        at(1),
+      ),
+    ).toEqual({ send: null, next: { notifiedState: null, notifiedAt: null } });
+    expect(decideNotice(null, "inactive", T0)).toEqual({
+      send: null,
+      next: null,
+    });
+  });
+});
+
+function readiness(
+  states: Partial<Record<ReadinessSignal["id"], [ReadinessState, string]>>,
+  now: Date,
+): RecoveryReadiness {
+  return {
+    state: "ok",
+    evaluatedAt: now.toISOString(),
+    signals: READINESS_SIGNALS.map((id) => {
+      const [state, reason] = states[id] ?? ["ok", "fresh"];
+      return {
+        id,
+        state,
+        reason,
+        at: null,
+        params: { days: 12, accounts: 1 },
+        detail: null,
+        fixPath: READINESS_FIX_PATHS[id],
+      };
+    }),
+  };
+}
+
+const sent = () =>
+  dispatch.mock.calls.map(([opts]) => {
+    const o = opts as unknown as {
+      userId: string;
+      titleKey: string;
+      messageKey: string;
+    };
+    return `${o.userId} ${o.titleKey.split(".").at(-1)} ${o.messageKey}`;
+  });
+
+describe("notifyReadinessChanges", () => {
+  beforeEach(() => {
+    store.notices.clear();
+    store.admins = [{ id: "admin-a" }, { id: "admin-b" }];
+    dispatch.mockClear();
+  });
+
+  it("sends each notice to every admin, through the backup event", async () => {
+    await notifyReadinessChanges(
+      readiness({ weekly_copy: ["attention", "stale"] }, T0),
+    );
+    expect(sent()).toEqual([
+      "admin-a titleAttention admin.recoveryReadiness.reason.weekly_copy.stale",
+      "admin-b titleAttention admin.recoveryReadiness.reason.weekly_copy.stale",
+    ]);
+    const first = dispatch.mock.calls[0][0] as unknown as {
+      eventType: string;
+      params: Record<string, unknown>;
+      metadata: Record<string, unknown>;
+    };
+    expect(first.eventType).toBe("BACKUP_HEALTH");
+    expect(first.params).toEqual({ days: 12, accounts: 1 });
+    expect(first.metadata).toMatchObject({
+      url: "/admin/backups",
+      signal: "weekly_copy",
+    });
+  });
+
+  it("runs an hourly sequence without nagging: change, silence, escalation, daily reminder, recovery", async () => {
+    store.admins = [{ id: "admin-a" }];
+    const run = (h: number, state: ReadinessState, reason: string) =>
+      notifyReadinessChanges(
+        readiness({ restore_drill: [state, reason] }, at(h)),
+      );
+
+    await run(0, "attention", "failed");
+    for (let h = 1; h < 5; h++) await run(h, "attention", "failed");
+    expect(sent()).toHaveLength(1);
+
+    await run(5, "failing", "failed");
+    for (let h = 6; h < 5 + FAILING_REMINDER_HOURS; h++)
+      await run(h, "failing", "failed");
+    expect(sent()).toHaveLength(2);
+
+    await run(5 + FAILING_REMINDER_HOURS, "failing", "failed");
+    await run(6 + FAILING_REMINDER_HOURS, "failing", "failed");
+    expect(sent()).toHaveLength(3);
+    expect(sent()[2]).toContain("titleReminder");
+
+    await run(40, "ok", "passed");
+    await run(41, "ok", "passed");
+    expect(sent()).toHaveLength(4);
+    expect(sent()[3]).toBe(
+      "admin-a titleRecovered admin.recoveryReadiness.recovered.restore_drill",
+    );
+  });
+
+  it("sends one notice when two evaluations race on the same row", async () => {
+    const r = readiness({ recovery_kit: ["attention", "out_of_date"] }, T0);
+    await Promise.all([notifyReadinessChanges(r), notifyReadinessChanges(r)]);
+    expect(sent()).toHaveLength(2); // one notice, two admins
+  });
+
+  it("claims nothing while there is no admin to tell", async () => {
+    store.admins = [];
+    const r = readiness({ offhost_setup: ["attention", "not_configured"] }, T0);
+    await notifyReadinessChanges(r);
+    expect(store.notices.get("offhost_setup")?.notifiedState).toBeNull();
+    store.admins = [{ id: "admin-a" }];
+    await notifyReadinessChanges(r);
+    expect(sent()).toHaveLength(1);
+  });
+
+  it("remembers since when a signal applies, and forgets it when it stops", async () => {
+    await notifyReadinessChanges(
+      readiness({ database_dump: ["ok", "fresh"] }, T0),
+    );
+    expect(store.notices.get("database_dump")?.watchedSince).toEqual(T0);
+    await notifyReadinessChanges(
+      readiness({ database_dump: ["ok", "fresh"] }, at(5)),
+    );
+    expect(store.notices.get("database_dump")?.watchedSince).toEqual(T0);
+    await notifyReadinessChanges(
+      readiness({ database_dump: ["inactive", "disabled"] }, at(6)),
+    );
+    expect(store.notices.get("database_dump")?.watchedSince).toBeNull();
+  });
+});

src/lib/recovery/database-dump-health.ts

@@ -0,0 +1,50 @@
+/**
+ * The scheduled database dump, as the recovery readiness model sees it.
+ *
+ * The dump is the one artifact that rebuilds a lost host: it holds every
+ * account, every grant and every credential, and a schema change cannot leave
+ * anything out of it. Its health is a signal of its own in the readiness
+ * model, read through a provider so the model does not reach into the dump's
+ * module and a test can hand in any state it wants.
+ *
+ * A provider that answers `null` means "this build does not schedule a dump",
+ * which the model reports as not available rather than as healthy or broken.
+ */
+
+/**
+ * What the dump module reports about its own schedule. Mirrors
+ * `DatabaseBackupHealth` in `src/lib/database-backup/health.ts` field for
+ * field, so the provider below becomes that module's
+ * `getDatabaseBackupHealth` without a translation.
+ */
+export interface DatabaseBackupHealth {
+  /** `DB_BACKUP_ENABLED` is not off. */
+  enabled: boolean;
+  /** The directory dumps are kept in (`DB_BACKUP_DIR`). */
+  directory: string;
+  /** This process can write there. False means dumps only go off-host. */
+  localDirWritable: boolean;
+  /** The off-host bucket is configured, or a dump has been uploaded to it. */
+  offhostConfigured: boolean;
+  /** The newest dump that was kept on the server or reached the bucket. */
+  lastGood: {
+    at: string;
+    fileName: string;
+    sizeBytes: number | null;
+    local: boolean;
+    offhost: boolean;
+  } | null;
+  /** The newest run that failed, or that a dying process left unfinished. */
+  lastFailure: { at: string; error: string } | null;
+}
+
+export type DatabaseBackupHealthProvider = (
+  now?: Date,
+) => Promise<DatabaseBackupHealth | null>;
+
+/**
+ * The provider every caller uses unless it passes its own: the dump's health,
+ * or `null` while no dump is scheduled by this build.
+ */
+export const readDatabaseBackupHealth: DatabaseBackupHealthProvider =
+  async () => null;

src/lib/recovery/readiness-model.ts

@@ -0,0 +1,422 @@
+/**
+ * Recovery readiness: can this instance be put back after a loss, judged from
+ * every fact the server holds about its own copies, in one place.
+ *
+ * Every one of these facts used to be a pull, on a page nobody opens until the
+ * day it matters. The weekly copy stopped for 46 days and showed a perfectly
+ * ordinary row. A failed monthly drill was on the failing-jobs card three days
+ * in thirty. An off-host bucket that was never configured showed only on the
+ * backups page, and the key step confirmed one key of the several a restore
+ * needs. This module turns them into six signals with one verdict each, which
+ * the readiness tile shows and the hourly evaluation pushes to every admin.
+ *
+ * Pure: the inputs are read elsewhere (`readiness.ts`), so each threshold is a
+ * unit test away and the tile and the push cannot disagree about a verdict.
+ *
+ * States:
+ *   ok        — nothing to do. A grace period (`*_grace` reasons) is ok too:
+ *               the admin is shown the step, nobody is pushed about it yet.
+ *   attention — something will hurt on the day of a restore; worth a push.
+ *   failing   — the instance cannot be restored the way it is meant to be.
+ *   inactive  — the signal does not apply here (no dump in this build, no
+ *               off-host bucket for the drill). Never pushed, never counts
+ *               towards the overall verdict.
+ */
+import {
+  classifyOffhostBackup,
+  type OffhostBackupFreshness,
+} from "@/lib/jobs/offhost-backup-freshness";
+import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
+import type { KitEntry } from "@/lib/recovery/kit-rules";
+
+const HOUR_MS = 60 * 60 * 1000;
+const DAY_MS = 24 * HOUR_MS;
+
+/** A dump older than this, while dumps are on, means no usable dump. */
+export const DUMP_FAILING_AFTER_HOURS = 36;
+/** The weekly copy runs every Sunday; past nine days one Sunday was missed. */
+export const WEEKLY_COPY_ATTENTION_AFTER_DAYS = 9;
+/** Past fourteen days two Sundays produced nothing for some account. */
+export const WEEKLY_COPY_FAILING_AFTER_DAYS = 14;
+/**
+ * How long after the first admin exists the off-host bucket and the recovery
+ * kit may stay unset before the admins are pushed about it.
+ */
+export const SETUP_GRACE_DAYS = 7;
+
+export const READINESS_SIGNALS = [
+  "database_dump",
+  "weekly_copy",
+  "offhost_copies",
+  "restore_drill",
+  "offhost_setup",
+  "recovery_kit",
+] as const;
+export type ReadinessSignalId = (typeof READINESS_SIGNALS)[number];
+
+export const READINESS_STATES = [
+  "ok",
+  "attention",
+  "failing",
+  "inactive",
+] as const;
+export type ReadinessState = (typeof READINESS_STATES)[number];
+
+/** Where each signal is fixed. Every path is an admin page. */
+export const READINESS_FIX_PATHS: Record<ReadinessSignalId, string> = {
+  database_dump: "/admin/backups",
+  weekly_copy: "/admin/backups",
+  offhost_copies: "/admin/backups",
+  restore_drill: "/admin/backups",
+  offhost_setup: "/admin/backups",
+  recovery_kit: "/admin/encryption",
+};
+
+export interface ReadinessSignal {
+  id: ReadinessSignalId;
+  state: ReadinessState;
+  /**
+   * Why, as a closed code per signal. The tile and the push render it through
+   * `admin.recoveryReadiness.<signal>.<reason>`, so it never carries prose.
+   */
+  reason: string;
+  /** The instant the signal is about (the newest dump, the last drill). */
+  at: string | null;
+  /** Values the sentence for `reason` interpolates. Never key material. */
+  params: Record<string, string | number>;
+  /**
+   * An operator-facing line from the source (a dump's or a drill's failure
+   * text), already redacted where it was written. Shown in the tile only.
+   */
+  detail: string | null;
+  fixPath: string;
+}
+
+export interface RecoveryReadiness {
+  /** The worst state of the signals that apply; `ok` when none do. */
+  state: Exclude<ReadinessState, "inactive">;
+  evaluatedAt: string;
+  signals: ReadinessSignal[];
+}
+
+/** What the evaluation reads, gathered by `readiness.ts`. */
+export interface ReadinessInputs {
+  now: Date;
+  /** When the first admin account was created; null on an empty install. */
+  firstAdminAt: Date | null;
+  /**
+   * `null` while no dump is scheduled by this build; `"unreadable"` when the
+   * dump module threw while reporting, which is itself a failure to report.
+   */
+  dump: DatabaseBackupHealth | null | "unreadable";
+  /**
+   * Since when the evaluation has seen dumps switched on. A dump that has
+   * never been attempted is judged against this, so an upgrade that turns
+   * dumps on is not failing before the first run had a chance.
+   */
+  dumpWatchedSince: Date | null;
+  weekly: {
+    /** One entry per account: when it was created and its newest copy. */
+    accounts: ReadonlyArray<{ createdAt: Date; lastCopyAt: Date | null }>;
+  };
+  offhost: {
+    configured: boolean;
+    accounts: ReadonlyArray<{
+      lastAttemptAt: Date | null;
+      lastSuccessAt: Date | null;
+    }>;
+  };
+  drill: {
+    lastRunAt: Date;
+    lastSuccessAt: Date | null;
+    lastFailureAt: Date | null;
+    lastFailureReason: string | null;
+    accountsChecked: number;
+    accountsFailed: number;
+  } | null;
+  kit: {
+    /** A confirmation exists at all. */
+    confirmedAt: Date | null;
+    /** Keys the kit lists that the confirmation does not cover. */
+    unconfirmed: ReadonlyArray<Pick<KitEntry, "kind" | "id">>;
+  };
+}
+
+function signal(
+  id: ReadinessSignalId,
+  state: ReadinessState,
+  reason: string,
+  at: Date | null,
+  params: Record<string, string | number> = {},
+  detail: string | null = null,
+): ReadinessSignal {
+  return {
+    id,
+    state,
+    reason,
+    at: at?.toISOString() ?? null,
+    params,
+    detail,
+    fixPath: READINESS_FIX_PATHS[id],
+  };
+}
+
+const wholeHours = (ms: number) => Math.max(0, Math.floor(ms / HOUR_MS));
+const wholeDays = (ms: number) => Math.max(0, Math.floor(ms / DAY_MS));
+
+function setupGraceEndsAt(firstAdminAt: Date | null): Date | null {
+  return firstAdminAt
+    ? new Date(firstAdminAt.getTime() + SETUP_GRACE_DAYS * DAY_MS)
+    : null;
+}
+
+export function evaluateDatabaseDump(
+  input: Pick<ReadinessInputs, "now" | "dump" | "dumpWatchedSince">,
+): ReadinessSignal {
+  const { dump, now } = input;
+  const id = "database_dump";
+  if (dump === null) return signal(id, "inactive", "not_available", null);
+  if (dump === "unreadable") return signal(id, "failing", "unreadable", null);
+  if (!dump.enabled) return signal(id, "inactive", "disabled", null);
+  const lastGoodAt = dump.lastGood ? new Date(dump.lastGood.at) : null;
+  const lastFailureAt = dump.lastFailure ? new Date(dump.lastFailure.at) : null;
+  const failureReason = dump.lastFailure?.error ?? null;
+  // A directory this process cannot write is a valid setup when the dumps go
+  // to the bucket; with no bucket either, no dump can be kept anywhere.
+  if (!dump.localDirWritable && !dump.offhostConfigured) {
+    return signal(id, "failing", "nowhere_to_write", lastGoodAt, {
+      directory: dump.directory,
+    });
+  }
+  const limitMs = DUMP_FAILING_AFTER_HOURS * HOUR_MS;
+  const failedSinceGood =
+    lastFailureAt !== null &&
+    (lastGoodAt === null || lastFailureAt > lastGoodAt);
+  if (lastGoodAt === null) {
+    if (failedSinceGood) {
+      return signal(
+        id,
+        "failing",
+        "never_succeeded",
+        lastFailureAt,
+        {},
+        failureReason,
+      );
+    }
+    const watched = input.dumpWatchedSince ?? now;
+    return now.getTime() - watched.getTime() > limitMs
+      ? signal(id, "failing", "never_ran", null, {
+          hours: DUMP_FAILING_AFTER_HOURS,
+        })
+      : signal(id, "ok", "first_pending", null);
+  }
+  const ageMs = now.getTime() - lastGoodAt.getTime();
+  if (ageMs > limitMs) {
+    return signal(
+      id,
+      "failing",
+      "stale",
+      lastGoodAt,
+      { hours: wholeHours(ageMs) },
+      failedSinceGood ? failureReason : null,
+    );
+  }
+  if (failedSinceGood) {
+    return signal(
+      id,
+      "attention",
+      "last_failed",
+      lastGoodAt,
+      {},
+      failureReason,
+    );
+  }
+  return signal(id, "ok", "fresh", lastGoodAt);
+}
+
+/**
+ * The worst account decides. An account without any copy ages from its
+ * creation, so a new account is not overdue before its first Sunday, and one
+ * the pass keeps skipping becomes overdue like a stale copy would.
+ */
+export function evaluateWeeklyCopy(
+  input: Pick<ReadinessInputs, "now" | "weekly">,
+): ReadinessSignal {
+  const id = "weekly_copy";
+  const { now } = input;
+  if (input.weekly.accounts.length === 0) {
+    return signal(id, "inactive", "no_accounts", null);
+  }
+  let worstMs = -Infinity;
+  let worst: { createdAt: Date; lastCopyAt: Date | null } | null = null;
+  let overdue = 0;
+  for (const account of input.weekly.accounts) {
+    const ref = account.lastCopyAt ?? account.createdAt;
+    const ageMs = now.getTime() - ref.getTime();
+    if (ageMs > WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY_MS) overdue += 1;
+    if (ageMs > worstMs) {
+      worstMs = ageMs;
+      worst = account;
+    }
+  }
+  const days = wholeDays(worstMs);
+  const at = worst?.lastCopyAt ?? null;
+  const params = { days, accounts: overdue };
+  if (worstMs > WEEKLY_COPY_FAILING_AFTER_DAYS * DAY_MS) {
+    return signal(id, "failing", at ? "stale" : "never", at, params);
+  }
+  if (worstMs > WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY_MS) {
+    return signal(id, "attention", at ? "stale" : "never", at, params);
+  }
+  // The oldest of the newest copies: the one a restore would be furthest
+  // behind with. Null only while no account has been copied yet.
+  const oldestCopy = input.weekly.accounts
+    .map((a) => a.lastCopyAt)
+    .filter((d): d is Date => d !== null)
+    .reduce<Date | null>((min, d) => (min === null || d < min ? d : min), null);
+  return signal(id, "ok", oldestCopy ? "fresh" : "first_pending", oldestCopy);
+}
+
+/**
+ * Any account whose newest off-host object is two schedule periods old (the
+ * freshness module's `stale`, which carries the DST and long-night grace), or
+ * that a run reached and never put an object there.
+ */
+export function evaluateOffhostCopies(
+  input: Pick<ReadinessInputs, "now" | "offhost">,
+): ReadinessSignal {
+  const id = "offhost_copies";
+  if (!input.offhost.configured) {
+    return signal(id, "inactive", "not_configured", null);
+  }
+  const verdicts: Array<{
+    freshness: OffhostBackupFreshness;
+    lastSuccessAt: Date | null;
+  }> = input.offhost.accounts.map((a) => ({
+    freshness: classifyOffhostBackup({
+      lastAttemptAt: a.lastAttemptAt,
+      lastSuccessAt: a.lastSuccessAt,
+      now: input.now,
+    }).freshness,
+    lastSuccessAt: a.lastSuccessAt,
+  }));
+  const oldest = (rows: typeof verdicts) =>
+    rows
+      .map((r) => r.lastSuccessAt)
+      .filter((d): d is Date => d !== null)
+      .reduce<Date | null>(
+        (min, d) => (min === null || d < min ? d : min),
+        null,
+      );
+  const behind = verdicts.filter(
+    (v) => v.freshness === "stale" || v.freshness === "never",
+  );
+  if (behind.length > 0) {
+    return signal(id, "attention", "stale", oldest(behind), {
+      accounts: behind.length,
+    });
+  }
+  const seen = verdicts.filter((v) => v.freshness !== "unknown");
+  if (seen.length === 0) return signal(id, "ok", "first_pending", null);
+  return signal(id, "ok", "fresh", oldest(seen));
+}
+
+/** A failed drill stays the verdict until a later drill passes. */
+export function evaluateRestoreDrill(
+  input: Pick<ReadinessInputs, "offhost" | "drill">,
+): ReadinessSignal {
+  const id = "restore_drill";
+  const { drill } = input;
+  if (!input.offhost.configured) {
+    return signal(id, "inactive", "not_configured", null);
+  }
+  if (drill === null) return signal(id, "ok", "not_run_yet", null);
+  const failing =
+    drill.lastFailureAt !== null &&
+    (drill.lastSuccessAt === null || drill.lastFailureAt > drill.lastSuccessAt);
+  if (failing) {
+    return signal(
+      id,
+      "failing",
+      // A drill that checked nothing stopped before any object: the bucket
+      // could not be listed, or held nothing to check.
+      drill.accountsChecked === 0 ? "failed_run" : "failed",
+      drill.lastFailureAt,
+      { failed: drill.accountsFailed, checked: drill.accountsChecked },
+      drill.lastFailureReason,
+    );
+  }
+  return signal(id, "ok", "passed", drill.lastSuccessAt ?? drill.lastRunAt, {
+    checked: drill.accountsChecked,
+  });
+}
+
+export function evaluateOffhostSetup(
+  input: Pick<ReadinessInputs, "now" | "firstAdminAt" | "offhost">,
+): ReadinessSignal {
+  const id = "offhost_setup";
+  if (input.offhost.configured) return signal(id, "ok", "configured", null);
+  const graceEnds = setupGraceEndsAt(input.firstAdminAt);
+  if (graceEnds === null) return signal(id, "inactive", "no_admin", null);
+  return input.now > graceEnds
+    ? signal(id, "attention", "not_configured", null)
+    : signal(id, "ok", "not_configured_grace", graceEnds);
+}
+
+export function evaluateRecoveryKit(
+  input: Pick<ReadinessInputs, "now" | "firstAdminAt" | "kit">,
+): ReadinessSignal {
+  const id = "recovery_kit";
+  const { kit } = input;
+  const keys = kit.unconfirmed.map((k) => k.id).join(", ");
+  if (kit.unconfirmed.length === 0 && kit.confirmedAt !== null) {
+    return signal(id, "ok", "confirmed", kit.confirmedAt);
+  }
+  if (kit.confirmedAt !== null) {
+    // Confirmed once, and a key it needs is not in that confirmation: a
+    // rotation, a new backup key, a re-keyed install. No grace: the keys are
+    // already in use.
+    return signal(id, "attention", "out_of_date", kit.confirmedAt, {
+      keys,
+      count: kit.unconfirmed.length,
+    });
+  }
+  const graceEnds = setupGraceEndsAt(input.firstAdminAt);
+  if (graceEnds === null) return signal(id, "inactive", "no_admin", null);
+  return input.now > graceEnds
+    ? signal(id, "attention", "not_confirmed", null, {
+        keys,
+        count: kit.unconfirmed.length,
+      })
+    : signal(id, "ok", "not_confirmed_grace", graceEnds, {
+        keys,
+        count: kit.unconfirmed.length,
+      });
+}
+
+const SEVERITY: Record<Exclude<ReadinessState, "inactive">, number> = {
+  ok: 0,
+  attention: 1,
+  failing: 2,
+};
+
+export function evaluateRecoveryReadiness(
+  inputs: ReadinessInputs,
+): RecoveryReadiness {
+  const signals = [
+    evaluateDatabaseDump(inputs),
+    evaluateWeeklyCopy(inputs),
+    evaluateOffhostCopies(inputs),
+    evaluateRestoreDrill(inputs),
+    evaluateOffhostSetup(inputs),
+    evaluateRecoveryKit(inputs),
+  ];
+  const state = signals.reduce<Exclude<ReadinessState, "inactive">>(
+    (worst, s) =>
+      s.state !== "inactive" && SEVERITY[s.state] > SEVERITY[worst]
+        ? s.state
+        : worst,
+    "ok",
+  );
+  return { state, evaluatedAt: inputs.now.toISOString(), signals };
+}

src/lib/recovery/readiness-notify.ts

@@ -0,0 +1,205 @@
+/**
+ * Push recovery readiness to every admin: once when a signal turns to
+ * attention or failing, once when it is back to ok, and a reminder at most
+ * once a day while it stays failing. Nothing else.
+ *
+ * The memory is one row per signal (`RecoveryReadinessNotice`): the state the
+ * admins were last told about and when. A notice is claimed with a
+ * compare-and-set on both columns before it is sent, so two workers that
+ * evaluate in the same minute send it once, and a crash between the claim and
+ * the send loses a notice rather than repeating it every hour.
+ *
+ * Delivery goes through the ordinary localised path, as the deploy-failure
+ * alert does: every ADMIN, in their own locale, on the channels they enabled
+ * for `BACKUP_HEALTH`, with the dispatcher's cooldowns and hard-reject rules.
+ * An admin who switched the event off on a channel is not reached there.
+ */
+import { prisma } from "@/lib/db";
+import { dispatchLocalisedNotification } from "@/lib/notifications/dispatch-localised";
+import {
+  READINESS_SIGNALS,
+  type ReadinessSignal,
+  type ReadinessSignalId,
+  type ReadinessState,
+  type RecoveryReadiness,
+} from "@/lib/recovery/readiness-model";
+
+/** A failing signal is reminded at most this often. */
+export const FAILING_REMINDER_HOURS = 24;
+
+type NotifiedState = Exclude<ReadinessState, "inactive">;
+
+export interface StoredNotice {
+  notifiedState: NotifiedState | null;
+  notifiedAt: Date | null;
+}
+
+export type NoticeKind = "problem" | "reminder" | "recovered";
+
+export interface NoticeDecision {
+  /** What to send, or null for nothing. */
+  send: NoticeKind | null;
+  /** What to store, or null to leave the row as it is. */
+  next: StoredNotice | null;
+}
+
+const NOTHING: NoticeDecision = { send: null, next: null };
+
+/**
+ * Pure: what one evaluation of one signal sends and stores.
+ *
+ *   - attention / failing, different from what was last announced: send it.
+ *   - failing, as announced, and the last notice is a day old: remind.
+ *   - ok after attention or failing: announce the recovery.
+ *   - inactive: forget silently. A signal that stops applying (the bucket was
+ *     unset, so there is no drill to judge) did not recover, and saying so
+ *     would be untrue; if it applies again and is still broken, it is news.
+ */
+export function decideNotice(
+  stored: StoredNotice | null,
+  current: ReadinessState,
+  now: Date,
+): NoticeDecision {
+  const last = stored?.notifiedState ?? null;
+  if (current === "inactive") {
+    return last === null
+      ? NOTHING
+      : { send: null, next: { notifiedState: null, notifiedAt: null } };
+  }
+  if (current === "ok") {
+    return last === "attention" || last === "failing"
+      ? { send: "recovered", next: { notifiedState: "ok", notifiedAt: now } }
+      : NOTHING;
+  }
+  if (last !== current) {
+    return {
+      send: "problem",
+      next: { notifiedState: current, notifiedAt: now },
+    };
+  }
+  const lastAt = stored?.notifiedAt ?? null;
+  if (
+    current === "failing" &&
+    (lastAt === null ||
+      now.getTime() - lastAt.getTime() >= FAILING_REMINDER_HOURS * 3_600_000)
+  ) {
+    return {
+      send: "reminder",
+      next: { notifiedState: current, notifiedAt: now },
+    };
+  }
+  return NOTHING;
+}
+
+/** The title of a notice, by what it says. */
+function titleKey(kind: NoticeKind, state: ReadinessState): string {
+  if (kind === "recovered") {
+    return "notifications.recoveryReadiness.titleRecovered";
+  }
+  if (kind === "reminder") {
+    return "notifications.recoveryReadiness.titleReminder";
+  }
+  return state === "failing"
+    ? "notifications.recoveryReadiness.titleFailing"
+    : "notifications.recoveryReadiness.titleAttention";
+}
+
+/**
+ * The body: the same sentence the tile shows for the signal's reason, or the
+ * signal's own recovery line.
+ */
+function messageKey(kind: NoticeKind, signal: ReadinessSignal): string {
+  return kind === "recovered"
+    ? `admin.recoveryReadiness.recovered.${signal.id}`
+    : `admin.recoveryReadiness.reason.${signal.id}.${signal.reason}`;
+}
+
+export interface NotifyOutcome {
+  /** Notices claimed and sent, by signal. */
+  sent: Array<{ signal: ReadinessSignalId; kind: NoticeKind }>;
+  /** Admins each notice went to. */
+  admins: number;
+}
+
+function isNotifiedState(value: string | null): value is NotifiedState {
+  return value === "ok" || value === "attention" || value === "failing";
+}
+
+/**
+ * Compare a fresh evaluation against what the admins were last told, claim
+ * every notice that is due, and send it.
+ */
+export async function notifyReadinessChanges(
+  readiness: RecoveryReadiness,
+  now: Date = new Date(readiness.evaluatedAt),
+): Promise<NotifyOutcome> {
+  await prisma.recoveryReadinessNotice.createMany({
+    data: READINESS_SIGNALS.map((signal) => ({ signal })),
+    skipDuplicates: true,
+  });
+  const rows = await prisma.recoveryReadinessNotice.findMany();
+  const byId = new Map(rows.map((row) => [row.signal, row]));
+  const admins = await prisma.user.findMany({
+    where: { role: "ADMIN" },
+    select: { id: true },
+  });
+
+  const outcome: NotifyOutcome = { sent: [], admins: admins.length };
+  for (const signal of readiness.signals) {
+    const row = byId.get(signal.id);
+    const applies = signal.state !== "inactive";
+    // Since when the signal applies: set once, cleared when it stops.
+    if (row && applies !== (row.watchedSince !== null)) {
+      await prisma.recoveryReadinessNotice.updateMany({
+        where: { signal: signal.id, watchedSince: row.watchedSince },
+        data: { watchedSince: applies ? now : null },
+      });
+    }
+
+    const stored: StoredNotice | null = row
+      ? {
+          notifiedState: isNotifiedState(row.notifiedState)
+            ? row.notifiedState
+            : null,
+          notifiedAt: row.notifiedAt,
+        }
+      : null;
+    const decision = decideNotice(stored, signal.state, now);
+    // With no admin yet there is nobody to tell, so nothing is claimed: the
+    // first admin hears about a problem that is still standing.
+    if (!decision.next || admins.length === 0) continue;
+
+    // Claim: only the evaluation that moves the row from what it read sends.
+    const claimed = await prisma.recoveryReadinessNotice.updateMany({
+      where: {
+        signal: signal.id,
+        notifiedState: row?.notifiedState ?? null,
+        notifiedAt: row?.notifiedAt ?? null,
+      },
+      data: {
+        notifiedState: decision.next.notifiedState,
+        notifiedAt: decision.next.notifiedAt,
+      },
+    });
+    if (claimed.count !== 1 || decision.send === null) continue;
+
+    for (const admin of admins) {
+      await dispatchLocalisedNotification({
+        userId: admin.id,
+        eventType: "BACKUP_HEALTH",
+        titleKey: titleKey(decision.send, signal.state),
+        messageKey: messageKey(decision.send, signal),
+        params: signal.params,
+        metadata: {
+          source: "recovery-readiness",
+          url: signal.fixPath,
+          signal: signal.id,
+          state: signal.state,
+          reason: signal.reason,
+        },
+      });
+    }
+    outcome.sent.push({ signal: signal.id, kind: decision.send });
+  }
+  return outcome;
+}

src/lib/recovery/readiness.ts

@@ -0,0 +1,117 @@
+/**
+ * Read every fact the recovery readiness model judges, and judge it.
+ *
+ * One reader for both consumers: the admin tile (`GET
+ * /api/admin/recovery-readiness`) and the hourly evaluation that pushes state
+ * changes to the admins. Each fact comes from where its own surface already
+ * reads it, so the tile can never disagree with the backups page about a copy:
+ *
+ *   - the weekly copy: the `WEEKLY_AUTO` row per account, which never ages out
+ *     of any retention window (pg-boss forgets a run after seven days);
+ *   - the off-host copies: the per-account ledger the nightly run writes, and
+ *     the same "configured" rule the backups page uses (the environment, or a
+ *     ledger row as proof that an upload happened);
+ *   - the drill: its persisted last outcome, with no time window;
+ *   - the kit: the same assessment the kit card shows;
+ *   - the dump: whatever the dump module reports, through a provider.
+ */
+import { prisma } from "@/lib/db";
+import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
+import {
+  readDatabaseBackupHealth,
+  type DatabaseBackupHealthProvider,
+} from "@/lib/recovery/database-dump-health";
+import { readRecoveryKit } from "@/lib/recovery/kit";
+import {
+  evaluateRecoveryReadiness,
+  type ReadinessInputs,
+  type RecoveryReadiness,
+} from "@/lib/recovery/readiness-model";
+
+/** The type the weekly worker writes; anything else was uploaded by hand. */
+const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO";
+
+export interface ReadRecoveryReadinessOptions {
+  now?: Date;
+  dumpHealth?: DatabaseBackupHealthProvider;
+}
+
+export async function readReadinessInputs(
+  options: ReadRecoveryReadinessOptions = {},
+): Promise<ReadinessInputs> {
+  const now = options.now ?? new Date();
+  const dumpHealth = options.dumpHealth ?? readDatabaseBackupHealth;
+  const [firstAdmin, accounts, drill, dumpNotice, dump, kit] =
+    await Promise.all([
+      prisma.user.findFirst({
+        where: { role: "ADMIN" },
+        orderBy: { createdAt: "asc" },
+        select: { createdAt: true },
+      }),
+      prisma.user.findMany({
+        select: {
+          createdAt: true,
+          dataBackups: {
+            where: { type: SCHEDULED_BACKUP_TYPE },
+            select: { createdAt: true },
+          },
+          offhostBackupState: {
+            select: { lastAttemptAt: true, lastSuccessAt: true },
+          },
+        },
+      }),
+      prisma.restoreDrillOutcome.findUnique({ where: { id: "singleton" } }),
+      prisma.recoveryReadinessNotice.findUnique({
+        where: { signal: "database_dump" },
+        select: { watchedSince: true },
+      }),
+      dumpHealth(now).catch(() => "unreadable" as const),
+      readRecoveryKit(now),
+    ]);
+
+  // Evidence beats this process's environment, as on the backups page: a
+  // ledger row is proof an upload happened, whichever process reads it.
+  const configured =
+    offhostBackupConfigured() ||
+    accounts.some((account) => account.offhostBackupState !== null);
+
+  return {
+    now,
+    firstAdminAt: firstAdmin?.createdAt ?? null,
+    dump,
+    dumpWatchedSince: dumpNotice?.watchedSince ?? null,
+    weekly: {
+      accounts: accounts.map((account) => ({
+        createdAt: account.createdAt,
+        lastCopyAt: account.dataBackups[0]?.createdAt ?? null,
+      })),
+    },
+    offhost: {
+      configured,
+      accounts: accounts.map((account) => ({
+        lastAttemptAt: account.offhostBackupState?.lastAttemptAt ?? null,
+        lastSuccessAt: account.offhostBackupState?.lastSuccessAt ?? null,
+      })),
+    },
+    drill: drill
+      ? {
+          lastRunAt: drill.lastRunAt,
+          lastSuccessAt: drill.lastSuccessAt,
+          lastFailureAt: drill.lastFailureAt,
+          lastFailureReason: drill.lastFailureReason,
+          accountsChecked: drill.accountsChecked,
+          accountsFailed: drill.accountsFailed,
+        }
+      : null,
+    kit: {
+      confirmedAt: kit.confirmedAt ? new Date(kit.confirmedAt) : null,
+      unconfirmed: kit.unconfirmed,
+    },
+  };
+}
+
+export async function readRecoveryReadiness(
+  options: ReadRecoveryReadinessOptions = {},
+): Promise<RecoveryReadiness> {
+  return evaluateRecoveryReadiness(await readReadinessInputs(options));
+}