Merge the per-account off-host backup freshness and the honest error-report badge

src/app/api/admin/backups/__tests__/route.test.ts

@@ -0,0 +1,199 @@
+/**
+ * `GET /api/admin/backups` — the off-host leg.
+ *
+ * The rows the weekly pass writes into this database were already covered by
+ * the page that lists them. What was not covered is the question this response
+ * now answers: per account, when did an encrypted copy last reach the
+ * operator's bucket, and is that inside what the nightly schedule promises.
+ *
+ * The bucket is never touched here, and that is the assertion that matters —
+ * the verdicts come from the ledger the worker writes, so an operator whose
+ * worker holds no listing grant on the bucket still gets a true answer.
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("@/lib/api-handler", () => ({
+  apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
+  requireAdmin: vi.fn(async () => ({ user: { id: "admin-1" } })),
+}));
+
+vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+
+vi.mock("@/lib/jobs/job-failures", () => ({
+  readLastQueueRun: vi.fn(async () => null),
+}));
+
+const queryRawMock = vi.fn();
+const findManyMock = vi.fn();
+vi.mock("@/lib/db", () => ({
+  prisma: {
+    $queryRaw: (...a: unknown[]) => queryRawMock(...a),
+    user: { findMany: (...a: unknown[]) => findManyMock(...a) },
+  },
+}));
+
+const configuredMock = vi.fn(() => true);
+vi.mock("@/lib/jobs/offhost-backup", () => ({
+  offhostBackupConfigured: () => configuredMock(),
+}));
+
+import { GET } from "../route";
+import type { BackupsList } from "@/types/backups";
+
+const HOUR = 3_600_000;
+
+async function read(): Promise<BackupsList> {
+  const res = await (GET as unknown as () => Promise<Response>)();
+  expect(res.status).toBe(200);
+  return (await res.json()).data as BackupsList;
+}
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  configuredMock.mockReturnValue(true);
+  queryRawMock.mockResolvedValue([]);
+  findManyMock.mockResolvedValue([]);
+});
+
+describe("GET /api/admin/backups — off-host freshness", () => {
+  it("rates every account against the nightly schedule without listing the bucket", async () => {
+    const now = Date.now();
+    findManyMock.mockResolvedValue([
+      {
+        id: "u1",
+        username: "account-one",
+        offhostBackupState: {
+          lastAttemptAt: new Date(now - 3 * HOUR),
+          lastSuccessAt: new Date(now - 3 * HOUR),
+          // The column is BigInt — an object may be 80 GB — and the wire type
+          // is a plain number.
+          sizeBytes: BigInt(4096),
+        },
+      },
+      {
+        id: "u2",
+        username: "account-two",
+        offhostBackupState: {
+          lastAttemptAt: new Date(now - 31 * HOUR),
+          lastSuccessAt: new Date(now - 31 * HOUR),
+          sizeBytes: BigInt(8192),
+        },
+      },
+      {
+        id: "u3",
+        username: "account-three",
+        offhostBackupState: {
+          lastAttemptAt: new Date(now - 100 * HOUR),
+          lastSuccessAt: new Date(now - 100 * HOUR),
+          sizeBytes: BigInt(128),
+        },
+      },
+      {
+        id: "u4",
+        username: "account-four",
+        // Walked by a run, nothing landed.
+        offhostBackupState: {
+          lastAttemptAt: new Date(now - 2 * HOUR),
+          lastSuccessAt: null,
+          sizeBytes: null,
+        },
+      },
+      // No run has recorded this one at all — the state of every account on
+      // the morning the ledger ships.
+      { id: "u5", username: "account-five", offhostBackupState: null },
+    ]);
+
+    const body = await read();
+
+    expect(body.offhost.configured).toBe(true);
+    expect(body.offhost.periodHours).toBe(24);
+    // Worst first, alphabetical within a verdict. The database hands them
+    // back by username; the one account that needs attention must not be
+    // wherever the alphabet leaves it on a hundred-account cohort.
+    expect(
+      body.offhost.rows.map((row) => [row.username, row.freshness]),
+    ).toEqual([
+      ["account-three", "stale"],
+      ["account-two", "due"],
+      ["account-four", "never"],
+      ["account-five", "unknown"],
+      ["account-one", "fresh"],
+    ]);
+    const byName = new Map(body.offhost.rows.map((row) => [row.username, row]));
+    expect(byName.get("account-one")?.sizeBytes).toBe(4096);
+    expect(typeof byName.get("account-one")?.sizeBytes).toBe("number");
+    expect(byName.get("account-one")?.ageHours).toBe(3);
+    expect(byName.get("account-four")?.lastSuccessAt).toBeNull();
+    expect(byName.get("account-four")?.sizeBytes).toBeNull();
+    expect(byName.get("account-four")?.lastAttemptAt).not.toBeNull();
+    // `unknown` is the honest one: the ledger has no history for this
+    // account, which is not the same claim as "the bucket is empty".
+    expect(byName.get("account-five")?.lastAttemptAt).toBeNull();
+  });
+
+  it("says so when neither the environment nor the ledger knows of any upload", async () => {
+    configuredMock.mockReturnValue(false);
+    findManyMock.mockResolvedValue([
+      { id: "u1", username: "account-one", offhostBackupState: null },
+    ]);
+
+    const body = await read();
+
+    expect(body.offhost).toEqual({
+      configured: false,
+      periodHours: 24,
+      rows: [],
+    });
+  });
+
+  it("lets a ledger row outrank this process's own environment", async () => {
+    // A hand-rolled split deployment can give BACKUP_S3_* to the worker and
+    // not to the web process. The rows are proof uploads are happening;
+    // saying "nothing leaves this host" over them would be this process
+    // reporting on a job it cannot see.
+    configuredMock.mockReturnValue(false);
+    findManyMock.mockResolvedValue([
+      {
+        id: "u1",
+        username: "account-one",
+        offhostBackupState: {
+          lastAttemptAt: new Date(),
+          lastSuccessAt: new Date(),
+          sizeBytes: BigInt(4096),
+        },
+      },
+    ]);
+
+    const body = await read();
+
+    expect(body.offhost.configured).toBe(true);
+    expect(body.offhost.rows.map((row) => row.freshness)).toEqual(["fresh"]);
+  });
+
+  it("never ships a credential or an object key with the verdicts", async () => {
+    findManyMock.mockResolvedValue([
+      {
+        id: "u1",
+        username: "account-one",
+        offhostBackupState: {
+          lastAttemptAt: new Date(),
+          lastSuccessAt: new Date(),
+          sizeBytes: BigInt(1),
+        },
+      },
+    ]);
+
+    const body = await read();
+
+    expect(Object.keys(body.offhost.rows[0] ?? {}).sort()).toEqual([
+      "ageHours",
+      "freshness",
+      "lastAttemptAt",
+      "lastSuccessAt",
+      "sizeBytes",
+      "userId",
+      "username",
+    ]);
+    expect(JSON.stringify(body.offhost)).not.toMatch(/bucket|endpoint|secret/i);
+  });
+});

src/app/api/admin/backups/route.ts

@@ -13,6 +13,12 @@
  * Without those two the page cannot tell a working backup from one that
  * stopped six weeks ago — every row it lists has a perfectly ordinary
  * timestamp either way.
+ *
+ * And alongside both, the off-host leg: one row per account saying when this
+ * host last put that account's encrypted copy in the operator's bucket and how
+ * stale that is against the nightly schedule. It comes from the ledger the
+ * worker writes, never from a bucket listing — the answer has to hold on a host
+ * whose worker holds no listing grant on the bucket.
  */
 import { prisma } from "@/lib/db";
 import { apiHandler, requireAdmin } from "@/lib/api-handler";
@@ -21,10 +27,20 @@ import { annotate } from "@/lib/logging/context";
 import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
 import { readLastQueueRun } from "@/lib/jobs/job-failures";
 import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
+import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
+import {
+  classifyOffhostBackup,
+  OFFHOST_BACKUP_PERIOD_HOURS,
+  type OffhostBackupFreshness,
+} from "@/lib/jobs/offhost-backup-freshness";
 // v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts`
 // so callers (in particular `components/admin/backups-section.tsx`) don't
 // have to reach across the component → route-handler layer boundary.
-import type { BackupRow, BackupsList } from "@/types/backups";
+import type {
+  BackupRow,
+  BackupsList,
+  OffhostAccountRow,
+} from "@/types/backups";
 
 export const dynamic = "force-dynamic";
 
@@ -67,6 +83,79 @@ export const GET = apiHandler(async () => {
     createdAt: row.created_at.toISOString(),
   }));
 
+  // The off-host leg, per account, read from the ledger the nightly worker
+  // writes rather than from the bucket. The page must be able to answer "which
+  // account has no recent copy off-host" on a host whose worker holds the
+  // three grants the runbook documents and no listing grant at all, and a
+  // listing call from a page render would put the operator's credentials on
+  // the request path of every page view.
+  const now = new Date();
+  const offhostAccounts = await prisma.user.findMany({
+    select: {
+      id: true,
+      username: true,
+      offhostBackupState: {
+        select: {
+          lastAttemptAt: true,
+          lastSuccessAt: true,
+          sizeBytes: true,
+        },
+      },
+    },
+    orderBy: { username: "asc" },
+  });
+
+  // Evidence beats this process's environment. `offhostBackupConfigured()`
+  // reads the variables of whichever process serves this request, and a
+  // hand-rolled split that gives `BACKUP_S3_*` only to the worker would make
+  // the web process say "nothing leaves this host" over a ledger full of
+  // fresh rows. A row in that table is proof an upload happened; no
+  // environment read can outrank it.
+  const configured =
+    offhostBackupConfigured() ||
+    offhostAccounts.some((account) => account.offhostBackupState !== null);
+
+  const offhostRows: OffhostAccountRow[] = offhostAccounts.map((account) => {
+    const state = account.offhostBackupState;
+    const verdict = classifyOffhostBackup({
+      lastAttemptAt: state?.lastAttemptAt ?? null,
+      lastSuccessAt: state?.lastSuccessAt ?? null,
+      now,
+    });
+    return {
+      userId: account.id,
+      username: account.username,
+      lastAttemptAt: state?.lastAttemptAt.toISOString() ?? null,
+      lastSuccessAt: state?.lastSuccessAt?.toISOString() ?? null,
+      // BigInt column -> number on the wire. The value is bytes of one
+      // object, capped by the uploader at 80 GB, so it is nowhere near
+      // Number.MAX_SAFE_INTEGER and JSON has no BigInt.
+      sizeBytes:
+        state?.sizeBytes === null || state?.sizeBytes === undefined
+          ? null
+          : Number(state.sizeBytes),
+      ageHours: verdict.ageHours,
+      freshness: verdict.freshness,
+    };
+  });
+
+  // Worst first. Sorting by username puts the one account that needs
+  // attention wherever the alphabet happens to leave it, which on a
+  // hundred-account cohort is the same "one bad row hides in the crowd"
+  // problem this card was built to end, moved out of a count and into a list.
+  const FRESHNESS_ORDER: Record<OffhostBackupFreshness, number> = {
+    stale: 0,
+    due: 1,
+    never: 2,
+    unknown: 3,
+    fresh: 4,
+  };
+  offhostRows.sort(
+    (a, b) =>
+      FRESHNESS_ORDER[a.freshness] - FRESHNESS_ORDER[b.freshness] ||
+      a.username.localeCompare(b.username),
+  );
+
   const payload: BackupsList = {
     rows: list,
     // Matches the retention window the backup-prune job enforces so the
@@ -77,8 +166,15 @@ export const GET = apiHandler(async () => {
         .filter((row) => row.type === SCHEDULED_BACKUP_TYPE)
         .map((row) => row.created_at),
       lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE),
-      now: new Date(),
+      now,
     }),
+    offhost: {
+      configured,
+      periodHours: OFFHOST_BACKUP_PERIOD_HOURS,
+      // An unconfigured host has nothing to list, and the card says so rather
+      // than painting an empty table that reads as "no problems".
+      rows: configured ? offhostRows : [],
+    },
   };
 
   return apiSuccess(payload);

src/app/api/admin/settings/route.ts

@@ -16,9 +16,32 @@ import {
   invalidateServerDefaultTimezone,
   isValidTimezone,
 } from "@/lib/tz/resolver";
+import { summariseGlitchtipDelivery } from "@/lib/monitoring/glitchtip-delivery";
 
 export const dynamic = "force-dynamic";
 
+/** The delivery verdict, flattened onto the settings payload. */
+function summariseGlitchtipDeliveryFields(state: {
+  lastOkAt?: Date | null;
+  lastFailureAt?: Date | null;
+  lastFailureReason?: string | null;
+}) {
+  const summary = summariseGlitchtipDelivery(
+    {
+      lastOkAt: state.lastOkAt ?? null,
+      lastFailureAt: state.lastFailureAt ?? null,
+      lastFailureReason: state.lastFailureReason ?? null,
+    },
+    new Date(),
+  );
+  return {
+    glitchtipReportsDelivering: summary.reportsDelivering,
+    glitchtipEverDelivered: summary.everDelivered,
+    glitchtipLastFailureReason: summary.lastFailureReason,
+    glitchtipDeliveryWindowHours: summary.windowHours,
+  };
+}
+
 export const GET = apiHandler(async () => {
   await requireAdmin();
   annotate({ action: { name: "admin.settings.get" } });
@@ -48,6 +71,15 @@ export const GET = apiHandler(async () => {
     glitchtipEnabled: settings?.glitchtipEnabled ?? false,
     glitchtipDsn: settings?.glitchtipDsn ?? null,
     glitchtipEnvironment: settings?.glitchtipEnvironment ?? "production",
+    // The delivery outcome, not the intent, and resolved here rather than on
+    // the client: a DSN that is set and parses says a target was typed, only
+    // this says something left the host, and whether it left recently enough
+    // is a question about a clock that belongs with the ledger.
+    ...summariseGlitchtipDeliveryFields({
+      lastOkAt: settings?.glitchtipLastOkAt ?? null,
+      lastFailureAt: settings?.glitchtipLastFailureAt ?? null,
+      lastFailureReason: settings?.glitchtipLastFailureReason ?? null,
+    }),
     reminderLateMinutes: settings?.reminderLateMinutes ?? 120,
     reminderMissedMinutes: settings?.reminderMissedMinutes ?? 240,
     // Document vault limits (per-file cap + per-user quota default). BigInt
@@ -274,6 +306,11 @@ export const PUT = apiHandler(async (request: NextRequest) => {
     glitchtipEnabled: settings.glitchtipEnabled,
     glitchtipDsn: settings.glitchtipDsn,
     glitchtipEnvironment: settings.glitchtipEnvironment ?? "production",
+    ...summariseGlitchtipDeliveryFields({
+      lastOkAt: settings.glitchtipLastOkAt,
+      lastFailureAt: settings.glitchtipLastFailureAt,
+      lastFailureReason: settings.glitchtipLastFailureReason,
+    }),
     reminderLateMinutes: settings.reminderLateMinutes,
     reminderMissedMinutes: settings.reminderMissedMinutes,
     documentMaxFileBytes: settings.documentMaxFileBytes,