mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-12 14:16:12 +00:00
feat(admin): say per account how fresh the off-host copy is
The nightly off-host job reports counts — uploaded, failed, total — and a count cannot name the account it is failing for. A cohort where one object has been refused every night for a month reads as "99 uploaded, 1 failed" each time, which looks like weather rather than like an account with no disaster-recovery copy at all. The worker now writes a ledger row when an object lands (account, instant, size), and Admin → Backups gains a second card that rates every account against the nightly schedule: fresh inside one run, due when one run produced nothing, stale when two did not, never when no object has ever landed. On a host without BACKUP_S3_* the card says off-host backup is not configured and stops there. The verdicts are computed from the ledger, never from a bucket listing: the worker's grant is deliberately PutObject + GetObject, and an admin page must not put the operator's credentials on its render path to state a number the database already holds.
This commit is contained in:
1 parent
348d9e0739
commit
1c10bbbde8
23 files changed
+797
-10
No files matched your search
@@ -109,6 +109,10 @@ const PINNED_AFFORDANCES: Record<
|
||||
"src/components/admin/module-availability-section.tsx": {
|
||||
"toast.success": 1,
|
||||
},
|
||||
// Reads a ledger, writes nothing: the wash marks how fresh one account's
|
||||
// off-host copy is, and the state it paints green came out of the database
|
||||
// rather than out of a request this card just made.
|
||||
"src/components/admin/offhost-backups-section.tsx": { "text-success": 1 },
|
||||
"src/components/admin/recent-audit-preview.tsx": {
|
||||
CheckCircle2: 2,
|
||||
"text-success": 1,
|
||||
|
||||
@@ -25,6 +25,7 @@ import { CoachFeedbackSection } from "@/components/admin/coach-feedback-section"
|
||||
import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-section";
|
||||
import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section";
|
||||
import { BackupsSection } from "@/components/admin/backups-section";
|
||||
import { OffhostBackupsSection } from "@/components/admin/offhost-backups-section";
|
||||
import { DangerZoneSection } from "@/components/admin/danger-zone-section";
|
||||
import { EncryptionSection } from "@/components/admin/encryption-section";
|
||||
import { GeneralSettingsSection } from "@/components/admin/general-settings-section";
|
||||
@@ -144,9 +145,12 @@ export function AdminSectionRenderer({
|
||||
</SectionFrame>
|
||||
);
|
||||
case "backups":
|
||||
// Two cards, two independent failure modes: the copies held in this
|
||||
// database, then the copies that reached the operator's bucket.
|
||||
return (
|
||||
<SectionFrame>
|
||||
<BackupsSection />
|
||||
<OffhostBackupsSection />
|
||||
</SectionFrame>
|
||||
);
|
||||
case "encryption":
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* `GET /api/admin/backups` — the off-host leg.
|
||||
*
|
||||
* The rows the weekly pass writes into this database were already covered by
|
||||
* the page that lists them. What was not covered is the question this response
|
||||
* now answers: per account, when did an encrypted copy last reach the
|
||||
* operator's bucket, and is that inside what the nightly schedule promises.
|
||||
*
|
||||
* The bucket is never touched here, and that is the assertion that matters —
|
||||
* the verdicts come from the ledger the worker writes, so an operator whose
|
||||
* worker holds PutObject and nothing else still gets a true answer.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/api-handler", () => ({
|
||||
apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
|
||||
requireAdmin: vi.fn(async () => ({ user: { id: "admin-1" } })),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
|
||||
|
||||
vi.mock("@/lib/jobs/job-failures", () => ({
|
||||
readLastQueueRun: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
const queryRawMock = vi.fn();
|
||||
const findManyMock = vi.fn();
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
$queryRaw: (...a: unknown[]) => queryRawMock(...a),
|
||||
user: { findMany: (...a: unknown[]) => findManyMock(...a) },
|
||||
},
|
||||
}));
|
||||
|
||||
const configuredMock = vi.fn(() => true);
|
||||
vi.mock("@/lib/jobs/offhost-backup", () => ({
|
||||
offhostBackupConfigured: () => configuredMock(),
|
||||
}));
|
||||
|
||||
import { GET } from "../route";
|
||||
import type { BackupsList } from "@/types/backups";
|
||||
|
||||
const HOUR = 3_600_000;
|
||||
|
||||
async function read(): Promise<BackupsList> {
|
||||
const res = await (GET as unknown as () => Promise<Response>)();
|
||||
expect(res.status).toBe(200);
|
||||
return (await res.json()).data as BackupsList;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
configuredMock.mockReturnValue(true);
|
||||
queryRawMock.mockResolvedValue([]);
|
||||
findManyMock.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
it("rates every account against the nightly schedule without listing the bucket", async () => {
|
||||
const now = Date.now();
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "u1",
|
||||
username: "account-one",
|
||||
offhostBackupState: {
|
||||
lastSuccessAt: new Date(now - 3 * HOUR),
|
||||
sizeBytes: 4096,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "u2",
|
||||
username: "account-two",
|
||||
offhostBackupState: {
|
||||
lastSuccessAt: new Date(now - 30 * HOUR),
|
||||
sizeBytes: 8192,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "u3",
|
||||
username: "account-three",
|
||||
offhostBackupState: {
|
||||
lastSuccessAt: new Date(now - 100 * HOUR),
|
||||
sizeBytes: 128,
|
||||
},
|
||||
},
|
||||
{ id: "u4", username: "account-four", offhostBackupState: null },
|
||||
]);
|
||||
|
||||
const body = await read();
|
||||
|
||||
expect(body.offhost.configured).toBe(true);
|
||||
expect(body.offhost.periodHours).toBe(24);
|
||||
expect(
|
||||
body.offhost.rows.map((row) => [row.username, row.freshness]),
|
||||
).toEqual([
|
||||
["account-one", "fresh"],
|
||||
["account-two", "due"],
|
||||
["account-three", "stale"],
|
||||
["account-four", "never"],
|
||||
]);
|
||||
expect(body.offhost.rows[0]?.sizeBytes).toBe(4096);
|
||||
expect(body.offhost.rows[0]?.ageHours).toBe(3);
|
||||
expect(body.offhost.rows[3]?.lastSuccessAt).toBeNull();
|
||||
expect(body.offhost.rows[3]?.sizeBytes).toBeNull();
|
||||
});
|
||||
|
||||
it("says so and reads no account when off-host backup is not configured", async () => {
|
||||
configuredMock.mockReturnValue(false);
|
||||
|
||||
const body = await read();
|
||||
|
||||
expect(body.offhost).toEqual({
|
||||
configured: false,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
});
|
||||
// Not merely empty output — the query never runs, so an unconfigured host
|
||||
// does not pay for a table scan to be told nothing is set up.
|
||||
expect(findManyMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("never ships a credential or an object key with the verdicts", async () => {
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "u1",
|
||||
username: "account-one",
|
||||
offhostBackupState: {
|
||||
lastSuccessAt: new Date(),
|
||||
sizeBytes: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const body = await read();
|
||||
|
||||
expect(Object.keys(body.offhost.rows[0] ?? {}).sort()).toEqual([
|
||||
"ageHours",
|
||||
"freshness",
|
||||
"lastSuccessAt",
|
||||
"sizeBytes",
|
||||
"userId",
|
||||
"username",
|
||||
]);
|
||||
expect(JSON.stringify(body.offhost)).not.toMatch(/bucket|endpoint|secret/i);
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,12 @@
|
||||
* Without those two the page cannot tell a working backup from one that
|
||||
* stopped six weeks ago — every row it lists has a perfectly ordinary
|
||||
* timestamp either way.
|
||||
*
|
||||
* And alongside both, the off-host leg: one row per account saying when this
|
||||
* host last put that account's encrypted copy in the operator's bucket and how
|
||||
* stale that is against the nightly schedule. It comes from the ledger the
|
||||
* worker writes, never from a bucket listing — the answer has to hold on a host
|
||||
* whose worker grant is PutObject and nothing else.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
@@ -21,10 +27,19 @@ import { annotate } from "@/lib/logging/context";
|
||||
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
||||
import { readLastQueueRun } from "@/lib/jobs/job-failures";
|
||||
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
} from "@/lib/jobs/offhost-backup-freshness";
|
||||
// v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts`
|
||||
// so callers (in particular `components/admin/backups-section.tsx`) don't
|
||||
// have to reach across the component → route-handler layer boundary.
|
||||
import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
import type {
|
||||
BackupRow,
|
||||
BackupsList,
|
||||
OffhostAccountRow,
|
||||
} from "@/types/backups";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -67,6 +82,42 @@ export const GET = apiHandler(async () => {
|
||||
createdAt: row.created_at.toISOString(),
|
||||
}));
|
||||
|
||||
// The off-host leg, per account, read from the ledger the nightly worker
|
||||
// writes rather than from the bucket. The page must be able to answer "which
|
||||
// account has no recent copy off-host" on a host whose worker grant is
|
||||
// PutObject and nothing else, and a listing call from a page render would
|
||||
// put the operator's credentials on the request path of every page view.
|
||||
const now = new Date();
|
||||
const configured = offhostBackupConfigured();
|
||||
const offhostAccounts = configured
|
||||
? await prisma.user.findMany({
|
||||
select: {
|
||||
id: true,
|
||||
username: true,
|
||||
offhostBackupState: {
|
||||
select: { lastSuccessAt: true, sizeBytes: true },
|
||||
},
|
||||
},
|
||||
orderBy: { username: "asc" },
|
||||
})
|
||||
: [];
|
||||
|
||||
const offhostRows: OffhostAccountRow[] = offhostAccounts.map((account) => {
|
||||
const state = account.offhostBackupState;
|
||||
const verdict = classifyOffhostBackup({
|
||||
lastSuccessAt: state?.lastSuccessAt ?? null,
|
||||
now,
|
||||
});
|
||||
return {
|
||||
userId: account.id,
|
||||
username: account.username,
|
||||
lastSuccessAt: state?.lastSuccessAt.toISOString() ?? null,
|
||||
sizeBytes: state?.sizeBytes ?? null,
|
||||
ageHours: verdict.ageHours,
|
||||
freshness: verdict.freshness,
|
||||
};
|
||||
});
|
||||
|
||||
const payload: BackupsList = {
|
||||
rows: list,
|
||||
// Matches the retention window the backup-prune job enforces so the
|
||||
@@ -77,8 +128,13 @@ export const GET = apiHandler(async () => {
|
||||
.filter((row) => row.type === SCHEDULED_BACKUP_TYPE)
|
||||
.map((row) => row.created_at),
|
||||
lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE),
|
||||
now: new Date(),
|
||||
now,
|
||||
}),
|
||||
offhost: {
|
||||
configured,
|
||||
periodHours: OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
rows: offhostRows,
|
||||
},
|
||||
};
|
||||
|
||||
return apiSuccess(payload);
|
||||
|
||||
@@ -322,7 +322,14 @@ function RestoreRowDialog({
|
||||
);
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number, fmt: ReturnType<typeof useFormatters>) {
|
||||
/**
|
||||
* Exported for the off-host card, which reads the same sizes out of the same
|
||||
* response and must not spell "1.25 MB" a second way two cards apart.
|
||||
*/
|
||||
export function formatBytes(
|
||||
bytes: number,
|
||||
fmt: ReturnType<typeof useFormatters>,
|
||||
) {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) {
|
||||
return `${fmt.integer(Math.round(bytes / 1024))} KB`;
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* `<OffhostBackupsSection>` — per-account freshness of the off-host copies.
|
||||
*
|
||||
* The card above it lists the copies that live in THIS database. This one
|
||||
* answers the different question: for each account, how long ago did this host
|
||||
* put an encrypted copy in the operator's bucket, and is that inside what the
|
||||
* nightly schedule promises. The two fail independently — a database whose
|
||||
* weekly rows are current tells an operator nothing about a bucket that has
|
||||
* been refusing the worker's signature since March.
|
||||
*
|
||||
* Every verdict is computed server-side from the ledger the worker writes. The
|
||||
* page never lists the bucket: the worker's grant is deliberately PutObject +
|
||||
* GetObject, and putting a credentialed listing call on the render path of an
|
||||
* admin page would be a poor trade for a number the database already holds.
|
||||
*/
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { CloudUpload } from "lucide-react";
|
||||
import { SettingsCard } from "@/components/settings/settings-card";
|
||||
import { SettingsCardHeader } from "@/components/settings/_card-header";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { EmptyState } from "@/components/ui/empty-state";
|
||||
import { ListRow } from "@/components/ui/list-row";
|
||||
import { QueryErrorRow } from "@/components/ui/query-error-row";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { apiGet } from "@/lib/api/api-fetch";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness";
|
||||
import type { BackupsList, OffhostAccountRow } from "@/types/backups";
|
||||
import { formatBytes } from "./backups-section";
|
||||
|
||||
/**
|
||||
* Status wash per verdict. `never` is deliberately neutral rather than
|
||||
* alarming: a host that turned the off-host job on this afternoon has not
|
||||
* failed at anything yet, and painting every row red on day one is how an
|
||||
* operator learns to stop reading the colour.
|
||||
*/
|
||||
const FRESHNESS_STYLE: Record<OffhostBackupFreshness, string> = {
|
||||
fresh: "border-success/30 bg-success/10 text-success",
|
||||
due: "border-warning/30 bg-warning/10 text-warning",
|
||||
stale: "border-destructive/30 bg-destructive/10 text-destructive",
|
||||
never: "border-border bg-muted text-muted-foreground",
|
||||
};
|
||||
|
||||
function freshnessLabel(
|
||||
freshness: OffhostBackupFreshness,
|
||||
t: ReturnType<typeof useTranslations>["t"],
|
||||
): string {
|
||||
// Literal keys, one per arm: a template-built key resolves at runtime and is
|
||||
// invisible to the call-site coverage guard.
|
||||
switch (freshness) {
|
||||
case "fresh":
|
||||
return t("admin.section.backups.offhost.stateFresh");
|
||||
case "due":
|
||||
return t("admin.section.backups.offhost.stateDue");
|
||||
case "stale":
|
||||
return t("admin.section.backups.offhost.stateStale");
|
||||
case "never":
|
||||
return t("admin.section.backups.offhost.stateNever");
|
||||
}
|
||||
}
|
||||
|
||||
function AccountRow({ row }: { row: OffhostAccountRow }) {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
|
||||
return (
|
||||
<ListRow asChild className="bg-muted/30 border-border">
|
||||
<li
|
||||
data-slot="offhost-backup-row"
|
||||
data-offhost-username={row.username}
|
||||
data-offhost-freshness={row.freshness}
|
||||
>
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<span className="min-w-0 flex-1 truncate font-medium">
|
||||
{row.username}
|
||||
</span>
|
||||
<Badge
|
||||
variant="outline"
|
||||
className={`shrink-0 ${FRESHNESS_STYLE[row.freshness]}`}
|
||||
>
|
||||
{freshnessLabel(row.freshness, t)}
|
||||
</Badge>
|
||||
</div>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{row.lastSuccessAt === null || row.sizeBytes === null
|
||||
? t("admin.section.backups.offhost.neverDetail")
|
||||
: `${fmt.dateTime(row.lastSuccessAt)} · ${formatBytes(
|
||||
row.sizeBytes,
|
||||
fmt,
|
||||
)} · ${t("admin.section.backups.offhost.age", {
|
||||
hours: row.ageHours ?? 0,
|
||||
})}`}
|
||||
</p>
|
||||
</li>
|
||||
</ListRow>
|
||||
);
|
||||
}
|
||||
|
||||
export function OffhostBackupsSection() {
|
||||
const { t } = useTranslations();
|
||||
|
||||
const { data, isLoading, isError, refetch } = useQuery({
|
||||
queryKey: queryKeys.adminBackups(),
|
||||
queryFn: async () => apiGet<BackupsList>("/api/admin/backups"),
|
||||
});
|
||||
|
||||
const offhost = data?.offhost;
|
||||
|
||||
return (
|
||||
<SettingsCard>
|
||||
<SettingsCardHeader
|
||||
icon={CloudUpload}
|
||||
title={t("admin.section.backups.offhost.title")}
|
||||
description={t("admin.section.backups.offhost.description")}
|
||||
status={
|
||||
offhost && offhost.configured ? (
|
||||
<Badge variant="secondary" className="text-xs">
|
||||
{offhost.rows.length}
|
||||
</Badge>
|
||||
) : null
|
||||
}
|
||||
/>
|
||||
|
||||
{isLoading ? (
|
||||
<div className="space-y-2" data-slot="offhost-backup-loading">
|
||||
<Skeleton className="h-14 w-full" />
|
||||
<Skeleton className="h-14 w-full" />
|
||||
</div>
|
||||
) : isError ? (
|
||||
<QueryErrorRow
|
||||
message={t("admin.section.backups.offhost.loadError")}
|
||||
onRetry={() => void refetch()}
|
||||
/>
|
||||
) : offhost && !offhost.configured ? (
|
||||
<EmptyState
|
||||
icon={<CloudUpload className="size-6" />}
|
||||
title={t("admin.section.backups.offhost.notConfiguredTitle")}
|
||||
description={t(
|
||||
"admin.section.backups.offhost.notConfiguredDescription",
|
||||
)}
|
||||
/>
|
||||
) : offhost ? (
|
||||
<div className="space-y-2">
|
||||
<ul className="space-y-2" data-slot="offhost-backup-rows">
|
||||
{offhost.rows.map((row) => (
|
||||
<AccountRow key={row.userId} row={row} />
|
||||
))}
|
||||
</ul>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{t("admin.section.backups.offhost.scheduleNote", {
|
||||
hours: offhost.periodHours,
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
) : null}
|
||||
</SettingsCard>
|
||||
);
|
||||
}
|
||||
@@ -210,6 +210,11 @@ export const WIPE_MODELS = [
|
||||
// just returned.
|
||||
"IdempotencyKey",
|
||||
"DataBackup",
|
||||
// What this host last put in the operator's bucket for this account. The
|
||||
// objects themselves stay — the worker holds no DeleteObject grant and the
|
||||
// bucket's lifecycle rule owns their retirement — but the row saying when
|
||||
// they were written is about this account and goes with it.
|
||||
"OffhostBackupState",
|
||||
"AuditLog",
|
||||
"ConsentReceipt",
|
||||
"UserKnownDevice",
|
||||
|
||||
@@ -642,6 +642,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly<Record<string, string>> = {
|
||||
"Replay protection for requests this host already answered. Restoring it would suppress a legitimate new request.",
|
||||
DataBackup:
|
||||
"The backup catalogue itself. A backup that contains the list of backups is a recursion with no reader.",
|
||||
OffhostBackupState:
|
||||
"When this host last put this account's copy in this operator's bucket, and how big it was. It describes one deployment's relationship with one bucket, so restoring it elsewhere would assert an off-host copy that host has never written.",
|
||||
ImportJob:
|
||||
"A job record pointing at an uploaded file that the backup does not carry, so restoring it would resurrect a task with nothing to work on.",
|
||||
InviteToken:
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
} from "@/lib/jobs/offhost-backup-freshness";
|
||||
|
||||
const NOW = new Date("2026-09-10T09:00:00.000Z");
|
||||
const HOUR = 3_600_000;
|
||||
const PERIOD = OFFHOST_BACKUP_PERIOD_HOURS * HOUR;
|
||||
|
||||
/** `age` hours before NOW. */
|
||||
function ago(ms: number): Date {
|
||||
return new Date(NOW.getTime() - ms);
|
||||
}
|
||||
|
||||
describe("classifyOffhostBackup", () => {
|
||||
it("says never when the worker has put nothing there for this account", () => {
|
||||
expect(classifyOffhostBackup({ lastSuccessAt: null, now: NOW })).toEqual({
|
||||
freshness: "never",
|
||||
ageHours: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("is fresh right up to the one-period boundary and due one millisecond past it", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD), now: NOW }).freshness,
|
||||
).toBe("fresh");
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD + 1), now: NOW })
|
||||
.freshness,
|
||||
).toBe("due");
|
||||
});
|
||||
|
||||
it("is due right up to the two-period boundary and stale one millisecond past it", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD * 2), now: NOW })
|
||||
.freshness,
|
||||
).toBe("due");
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD * 2 + 1), now: NOW })
|
||||
.freshness,
|
||||
).toBe("stale");
|
||||
});
|
||||
|
||||
it("reports whole hours of age", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(90 * 60_000), now: NOW }),
|
||||
).toEqual({ freshness: "fresh", ageHours: 1 });
|
||||
});
|
||||
|
||||
it("honours a caller's own period rather than the nightly default", () => {
|
||||
// A six-hour schedule: the same 12-hour-old object that is fresh under the
|
||||
// nightly cron is two periods behind under this one.
|
||||
const twelveHours = ago(12 * HOUR);
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: twelveHours, now: NOW }).freshness,
|
||||
).toBe("fresh");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
periodHours: 6,
|
||||
}).freshness,
|
||||
).toBe("due");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
periodHours: 5,
|
||||
}).freshness,
|
||||
).toBe("stale");
|
||||
});
|
||||
|
||||
it("does not call a copy stale because the host clock ran backwards", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastSuccessAt: new Date(NOW.getTime() + 2 * HOUR),
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ freshness: "fresh", ageHours: 0 });
|
||||
});
|
||||
});
|
||||
@@ -184,6 +184,7 @@ describe("runOffhostBackup", () => {
|
||||
s3.store.set("2020-01-01/user-old.json.enc", Buffer.from([0]));
|
||||
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: {
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
@@ -229,11 +230,26 @@ describe("runOffhostBackup", () => {
|
||||
// Nothing went through the whole-buffer arm.
|
||||
expect(s3.putObject).not.toHaveBeenCalled();
|
||||
expect(s3.putStream).toHaveBeenCalledTimes(2);
|
||||
|
||||
// Every account that got an object gets a ledger row, because the run's
|
||||
// own counts cannot say WHICH account has a copy off-host and which one
|
||||
// has been failing quietly since March.
|
||||
expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(2);
|
||||
const ledgered = prisma.offhostBackupState.upsert.mock.calls.map(
|
||||
(call) => (call[0] as { where: { userId: string } }).where.userId,
|
||||
);
|
||||
expect(ledgered).toEqual(["u1", "u2"]);
|
||||
const first = prisma.offhostBackupState.upsert.mock.calls[0]?.[0] as {
|
||||
create: { sizeBytes: number; lastSuccessAt: Date };
|
||||
};
|
||||
expect(first.create.sizeBytes).toBe(ct.byteLength);
|
||||
expect(first.create.lastSuccessAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it("uploads the canonical builder output without reshaping it", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
const canonicalPayload = {
|
||||
@@ -291,6 +307,7 @@ describe("runOffhostBackup", () => {
|
||||
it("counts per-user failures without aborting the whole run", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: {
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
@@ -319,6 +336,7 @@ describe("runOffhostBackup", () => {
|
||||
it("refuses an account whose object outgrows one multipart upload", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
mocks.buildFullBackupPayload.mockResolvedValue({
|
||||
@@ -355,6 +373,7 @@ describe("runOffhostBackup", () => {
|
||||
it("reports the largest object it wrote", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
const report = await runOffhostBackup(
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* How fresh one account's newest off-host copy is.
|
||||
*
|
||||
* The nightly off-host job reports counts — uploaded, failed, total — and a
|
||||
* count is exactly the shape that hides a single account. A cohort where one
|
||||
* account's object has failed every night for a month reports "99 uploaded, 1
|
||||
* failed" every night, which reads like weather rather than like an account
|
||||
* that has no disaster-recovery copy at all. The per-account ledger the worker
|
||||
* writes (`OffhostBackupState`) turns that into a fact the console can state
|
||||
* per row, and this function is the only place that fact is judged.
|
||||
*
|
||||
* Deliberately pure and deliberately not talking to the bucket: the admin page
|
||||
* computes staleness from the timestamp the worker wrote, so the answer is the
|
||||
* same whether or not the app process holds a read grant on the bucket, and a
|
||||
* listing of somebody's backups never crosses the network to show a page.
|
||||
*
|
||||
* Four states, because the run is nightly and one missed night is not the same
|
||||
* event as four:
|
||||
*
|
||||
* never — the worker has never put an object there for this account.
|
||||
* fresh — inside one schedule period. This is what every account looks like
|
||||
* on a host where the cron is doing its job.
|
||||
* due — past one period, inside two. One night produced nothing: a retry, a
|
||||
* long run, a restart during the window. Worth showing, not worth
|
||||
* alarming about.
|
||||
* stale — past two periods. Two consecutive nights produced nothing for this
|
||||
* account, which no ordinary hiccup explains.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Hours between two scheduled off-host runs. Tracks `OFFHOST_BACKUP_CRON`
|
||||
* (`30 2 * * *`) in `src/lib/jobs/reminder/register-maintenance.ts` — nightly.
|
||||
*/
|
||||
export const OFFHOST_BACKUP_PERIOD_HOURS = 24;
|
||||
|
||||
export type OffhostBackupFreshness = "never" | "fresh" | "due" | "stale";
|
||||
|
||||
export interface OffhostBackupVerdict {
|
||||
freshness: OffhostBackupFreshness;
|
||||
/** Whole hours since the newest object landed; null when there is none. */
|
||||
ageHours: number | null;
|
||||
}
|
||||
|
||||
export function classifyOffhostBackup(args: {
|
||||
lastSuccessAt: Date | null;
|
||||
now: Date;
|
||||
/** Defaults to the nightly schedule. Tests pass their own. */
|
||||
periodHours?: number;
|
||||
}): OffhostBackupVerdict {
|
||||
const { lastSuccessAt, now } = args;
|
||||
if (lastSuccessAt === null) return { freshness: "never", ageHours: null };
|
||||
|
||||
const periodMs =
|
||||
(args.periodHours ?? OFFHOST_BACKUP_PERIOD_HOURS) * 3_600_000;
|
||||
const ageMs = now.getTime() - lastSuccessAt.getTime();
|
||||
// A clock that ran backwards between the upload and this read reports a
|
||||
// negative age. That is a host-clock problem, not a backup problem, and
|
||||
// calling the copy stale over it would point the operator at the wrong
|
||||
// thing.
|
||||
const ageHours = Math.max(0, Math.floor(ageMs / 3_600_000));
|
||||
|
||||
if (ageMs <= periodMs) return { freshness: "fresh", ageHours };
|
||||
if (ageMs <= periodMs * 2) return { freshness: "due", ageHours };
|
||||
return { freshness: "stale", ageHours };
|
||||
}
|
||||
@@ -61,6 +61,27 @@ function decodeBackupKey(raw: string): Buffer {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this host has the five variables the nightly job needs.
|
||||
*
|
||||
* Separate from `loadOffhostConfig()` because a surface that only wants to say
|
||||
* "off-host backup is not set up here" must not decode the encryption key to
|
||||
* find out — `decodeBackupKey` throws on a malformed one, and an operator
|
||||
* whose key has a typo should see the admin page, not a 500.
|
||||
*/
|
||||
export function offhostBackupConfigured(): boolean {
|
||||
return OFFHOST_ENV_VARS.every((name) => Boolean(process.env[name]));
|
||||
}
|
||||
|
||||
/** The five variables the nightly job needs, in one place. */
|
||||
const OFFHOST_ENV_VARS = [
|
||||
"BACKUP_S3_ENDPOINT",
|
||||
"BACKUP_S3_BUCKET",
|
||||
"BACKUP_S3_ACCESS_KEY",
|
||||
"BACKUP_S3_SECRET_KEY",
|
||||
"BACKUP_ENCRYPTION_KEY",
|
||||
] as const;
|
||||
|
||||
export function loadOffhostConfig(): OffhostBackupConfig | null {
|
||||
const endpoint = process.env.BACKUP_S3_ENDPOINT;
|
||||
const bucket = process.env.BACKUP_S3_BUCKET;
|
||||
@@ -573,6 +594,21 @@ export async function runOffhostBackup(
|
||||
}),
|
||||
options,
|
||||
);
|
||||
// The per-account ledger, written the moment the object exists. The run
|
||||
// itself only ever reported counts, and a count cannot say WHICH account
|
||||
// has no copy — which is the one question an operator asks about a
|
||||
// backup. `new Date()` rather than the run's `now`, because on a large
|
||||
// cohort the two are hours apart and the row is meant to say when this
|
||||
// account's object landed.
|
||||
await prisma.offhostBackupState.upsert({
|
||||
where: { userId: user.id },
|
||||
update: { lastSuccessAt: new Date(), sizeBytes: objectBytes },
|
||||
create: {
|
||||
userId: user.id,
|
||||
lastSuccessAt: new Date(),
|
||||
sizeBytes: objectBytes,
|
||||
},
|
||||
});
|
||||
largestObjectBytes = Math.max(largestObjectBytes, objectBytes);
|
||||
uploaded++;
|
||||
} catch (err) {
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
* keeps owning the HTTP contract, the component keeps owning the UI.
|
||||
*/
|
||||
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
|
||||
import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness";
|
||||
|
||||
export interface BackupRow {
|
||||
id: string;
|
||||
@@ -24,6 +25,35 @@ export interface BackupRow {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** One account's newest object in the operator's off-host bucket. */
|
||||
export interface OffhostAccountRow {
|
||||
userId: string;
|
||||
username: string;
|
||||
/** ISO instant the newest object landed, or null when there is none. */
|
||||
lastSuccessAt: string | null;
|
||||
/** Size of that object in bytes, or null when there is none. */
|
||||
sizeBytes: number | null;
|
||||
/** Whole hours since it landed, or null when there is none. */
|
||||
ageHours: number | null;
|
||||
freshness: OffhostBackupFreshness;
|
||||
}
|
||||
|
||||
/**
|
||||
* The off-host picture, per account.
|
||||
*
|
||||
* Computed from the ledger the nightly worker writes, never from the bucket:
|
||||
* the page must be able to answer "does this account have a recent copy
|
||||
* off-host" on a host whose worker holds PutObject and nothing else.
|
||||
*/
|
||||
export interface OffhostBackupOverview {
|
||||
/** Whether this host has the variables the nightly job needs. */
|
||||
configured: boolean;
|
||||
/** Hours between two scheduled runs — the number behind the verdicts. */
|
||||
periodHours: number;
|
||||
/** One row per account. Empty when off-host backup is not configured. */
|
||||
rows: OffhostAccountRow[];
|
||||
}
|
||||
|
||||
export interface BackupsList {
|
||||
rows: BackupRow[];
|
||||
/**
|
||||
@@ -32,6 +62,12 @@ export interface BackupsList {
|
||||
* made six weeks ago and one made on Sunday look identical in a table.
|
||||
*/
|
||||
schedule: BackupScheduleStatus;
|
||||
/**
|
||||
* The off-host leg. A weekly row in this table says the copy that lives in
|
||||
* this database is current; it says nothing about whether anything reached
|
||||
* the operator's bucket, and those two fail independently.
|
||||
*/
|
||||
offhost: OffhostBackupOverview;
|
||||
/**
|
||||
* Soft retention hint — the worker is configured for weekly backups
|
||||
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and
|
||||
|
||||
Reference in new issue
Block a user