mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 13:01:00 +00:00
Merge the per-account off-host backup freshness and the honest error-report badge
This commit is contained in:
Marc-André Bombeck committed
commit
e8a2cbe82e
36 files changed
+2227
-27
No files matched your search
@@ -109,6 +109,10 @@ const PINNED_AFFORDANCES: Record<
|
||||
"src/components/admin/module-availability-section.tsx": {
|
||||
"toast.success": 1,
|
||||
},
|
||||
// Reads a ledger, writes nothing: the wash marks how fresh one account's
|
||||
// off-host copy is, and the state it paints green came out of the database
|
||||
// rather than out of a request this card just made.
|
||||
"src/components/admin/offhost-backups-section.tsx": { "text-success": 1 },
|
||||
"src/components/admin/recent-audit-preview.tsx": {
|
||||
CheckCircle2: 2,
|
||||
"text-success": 1,
|
||||
|
||||
@@ -25,6 +25,7 @@ import { CoachFeedbackSection } from "@/components/admin/coach-feedback-section"
|
||||
import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-section";
|
||||
import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section";
|
||||
import { BackupsSection } from "@/components/admin/backups-section";
|
||||
import { OffhostBackupsSection } from "@/components/admin/offhost-backups-section";
|
||||
import { DangerZoneSection } from "@/components/admin/danger-zone-section";
|
||||
import { EncryptionSection } from "@/components/admin/encryption-section";
|
||||
import { GeneralSettingsSection } from "@/components/admin/general-settings-section";
|
||||
@@ -144,9 +145,12 @@ export function AdminSectionRenderer({
|
||||
</SectionFrame>
|
||||
);
|
||||
case "backups":
|
||||
// Two cards, two independent failure modes: the copies held in this
|
||||
// database, then the copies that reached the operator's bucket.
|
||||
return (
|
||||
<SectionFrame>
|
||||
<BackupsSection />
|
||||
<OffhostBackupsSection />
|
||||
</SectionFrame>
|
||||
);
|
||||
case "encryption":
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
/**
|
||||
* `GET /api/admin/backups` — the off-host leg.
|
||||
*
|
||||
* The rows the weekly pass writes into this database were already covered by
|
||||
* the page that lists them. What was not covered is the question this response
|
||||
* now answers: per account, when did an encrypted copy last reach the
|
||||
* operator's bucket, and is that inside what the nightly schedule promises.
|
||||
*
|
||||
* The bucket is never touched here, and that is the assertion that matters —
|
||||
* the verdicts come from the ledger the worker writes, so an operator whose
|
||||
* worker holds no listing grant on the bucket still gets a true answer.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/api-handler", () => ({
|
||||
apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
|
||||
requireAdmin: vi.fn(async () => ({ user: { id: "admin-1" } })),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
|
||||
|
||||
vi.mock("@/lib/jobs/job-failures", () => ({
|
||||
readLastQueueRun: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
const queryRawMock = vi.fn();
|
||||
const findManyMock = vi.fn();
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
$queryRaw: (...a: unknown[]) => queryRawMock(...a),
|
||||
user: { findMany: (...a: unknown[]) => findManyMock(...a) },
|
||||
},
|
||||
}));
|
||||
|
||||
const configuredMock = vi.fn(() => true);
|
||||
vi.mock("@/lib/jobs/offhost-backup", () => ({
|
||||
offhostBackupConfigured: () => configuredMock(),
|
||||
}));
|
||||
|
||||
import { GET } from "../route";
|
||||
import type { BackupsList } from "@/types/backups";
|
||||
|
||||
const HOUR = 3_600_000;
|
||||
|
||||
async function read(): Promise<BackupsList> {
|
||||
const res = await (GET as unknown as () => Promise<Response>)();
|
||||
expect(res.status).toBe(200);
|
||||
return (await res.json()).data as BackupsList;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
configuredMock.mockReturnValue(true);
|
||||
queryRawMock.mockResolvedValue([]);
|
||||
findManyMock.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
it("rates every account against the nightly schedule without listing the bucket", async () => {
|
||||
const now = Date.now();
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "u1",
|
||||
username: "account-one",
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(now - 3 * HOUR),
|
||||
lastSuccessAt: new Date(now - 3 * HOUR),
|
||||
// The column is BigInt — an object may be 80 GB — and the wire type
|
||||
// is a plain number.
|
||||
sizeBytes: BigInt(4096),
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "u2",
|
||||
username: "account-two",
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(now - 31 * HOUR),
|
||||
lastSuccessAt: new Date(now - 31 * HOUR),
|
||||
sizeBytes: BigInt(8192),
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "u3",
|
||||
username: "account-three",
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(now - 100 * HOUR),
|
||||
lastSuccessAt: new Date(now - 100 * HOUR),
|
||||
sizeBytes: BigInt(128),
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "u4",
|
||||
username: "account-four",
|
||||
// Walked by a run, nothing landed.
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(now - 2 * HOUR),
|
||||
lastSuccessAt: null,
|
||||
sizeBytes: null,
|
||||
},
|
||||
},
|
||||
// No run has recorded this one at all — the state of every account on
|
||||
// the morning the ledger ships.
|
||||
{ id: "u5", username: "account-five", offhostBackupState: null },
|
||||
]);
|
||||
|
||||
const body = await read();
|
||||
|
||||
expect(body.offhost.configured).toBe(true);
|
||||
expect(body.offhost.periodHours).toBe(24);
|
||||
// Worst first, alphabetical within a verdict. The database hands them
|
||||
// back by username; the one account that needs attention must not be
|
||||
// wherever the alphabet leaves it on a hundred-account cohort.
|
||||
expect(
|
||||
body.offhost.rows.map((row) => [row.username, row.freshness]),
|
||||
).toEqual([
|
||||
["account-three", "stale"],
|
||||
["account-two", "due"],
|
||||
["account-four", "never"],
|
||||
["account-five", "unknown"],
|
||||
["account-one", "fresh"],
|
||||
]);
|
||||
const byName = new Map(body.offhost.rows.map((row) => [row.username, row]));
|
||||
expect(byName.get("account-one")?.sizeBytes).toBe(4096);
|
||||
expect(typeof byName.get("account-one")?.sizeBytes).toBe("number");
|
||||
expect(byName.get("account-one")?.ageHours).toBe(3);
|
||||
expect(byName.get("account-four")?.lastSuccessAt).toBeNull();
|
||||
expect(byName.get("account-four")?.sizeBytes).toBeNull();
|
||||
expect(byName.get("account-four")?.lastAttemptAt).not.toBeNull();
|
||||
// `unknown` is the honest one: the ledger has no history for this
|
||||
// account, which is not the same claim as "the bucket is empty".
|
||||
expect(byName.get("account-five")?.lastAttemptAt).toBeNull();
|
||||
});
|
||||
|
||||
it("says so when neither the environment nor the ledger knows of any upload", async () => {
|
||||
configuredMock.mockReturnValue(false);
|
||||
findManyMock.mockResolvedValue([
|
||||
{ id: "u1", username: "account-one", offhostBackupState: null },
|
||||
]);
|
||||
|
||||
const body = await read();
|
||||
|
||||
expect(body.offhost).toEqual({
|
||||
configured: false,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("lets a ledger row outrank this process's own environment", async () => {
|
||||
// A hand-rolled split deployment can give BACKUP_S3_* to the worker and
|
||||
// not to the web process. The rows are proof uploads are happening;
|
||||
// saying "nothing leaves this host" over them would be this process
|
||||
// reporting on a job it cannot see.
|
||||
configuredMock.mockReturnValue(false);
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "u1",
|
||||
username: "account-one",
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(),
|
||||
lastSuccessAt: new Date(),
|
||||
sizeBytes: BigInt(4096),
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const body = await read();
|
||||
|
||||
expect(body.offhost.configured).toBe(true);
|
||||
expect(body.offhost.rows.map((row) => row.freshness)).toEqual(["fresh"]);
|
||||
});
|
||||
|
||||
it("never ships a credential or an object key with the verdicts", async () => {
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "u1",
|
||||
username: "account-one",
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(),
|
||||
lastSuccessAt: new Date(),
|
||||
sizeBytes: BigInt(1),
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const body = await read();
|
||||
|
||||
expect(Object.keys(body.offhost.rows[0] ?? {}).sort()).toEqual([
|
||||
"ageHours",
|
||||
"freshness",
|
||||
"lastAttemptAt",
|
||||
"lastSuccessAt",
|
||||
"sizeBytes",
|
||||
"userId",
|
||||
"username",
|
||||
]);
|
||||
expect(JSON.stringify(body.offhost)).not.toMatch(/bucket|endpoint|secret/i);
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,12 @@
|
||||
* Without those two the page cannot tell a working backup from one that
|
||||
* stopped six weeks ago — every row it lists has a perfectly ordinary
|
||||
* timestamp either way.
|
||||
*
|
||||
* And alongside both, the off-host leg: one row per account saying when this
|
||||
* host last put that account's encrypted copy in the operator's bucket and how
|
||||
* stale that is against the nightly schedule. It comes from the ledger the
|
||||
* worker writes, never from a bucket listing — the answer has to hold on a host
|
||||
* whose worker holds no listing grant on the bucket.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
@@ -21,10 +27,20 @@ import { annotate } from "@/lib/logging/context";
|
||||
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
||||
import { readLastQueueRun } from "@/lib/jobs/job-failures";
|
||||
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
type OffhostBackupFreshness,
|
||||
} from "@/lib/jobs/offhost-backup-freshness";
|
||||
// v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts`
|
||||
// so callers (in particular `components/admin/backups-section.tsx`) don't
|
||||
// have to reach across the component → route-handler layer boundary.
|
||||
import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
import type {
|
||||
BackupRow,
|
||||
BackupsList,
|
||||
OffhostAccountRow,
|
||||
} from "@/types/backups";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -67,6 +83,79 @@ export const GET = apiHandler(async () => {
|
||||
createdAt: row.created_at.toISOString(),
|
||||
}));
|
||||
|
||||
// The off-host leg, per account, read from the ledger the nightly worker
|
||||
// writes rather than from the bucket. The page must be able to answer "which
|
||||
// account has no recent copy off-host" on a host whose worker holds the
|
||||
// three grants the runbook documents and no listing grant at all, and a
|
||||
// listing call from a page render would put the operator's credentials on
|
||||
// the request path of every page view.
|
||||
const now = new Date();
|
||||
const offhostAccounts = await prisma.user.findMany({
|
||||
select: {
|
||||
id: true,
|
||||
username: true,
|
||||
offhostBackupState: {
|
||||
select: {
|
||||
lastAttemptAt: true,
|
||||
lastSuccessAt: true,
|
||||
sizeBytes: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { username: "asc" },
|
||||
});
|
||||
|
||||
// Evidence beats this process's environment. `offhostBackupConfigured()`
|
||||
// reads the variables of whichever process serves this request, and a
|
||||
// hand-rolled split that gives `BACKUP_S3_*` only to the worker would make
|
||||
// the web process say "nothing leaves this host" over a ledger full of
|
||||
// fresh rows. A row in that table is proof an upload happened; no
|
||||
// environment read can outrank it.
|
||||
const configured =
|
||||
offhostBackupConfigured() ||
|
||||
offhostAccounts.some((account) => account.offhostBackupState !== null);
|
||||
|
||||
const offhostRows: OffhostAccountRow[] = offhostAccounts.map((account) => {
|
||||
const state = account.offhostBackupState;
|
||||
const verdict = classifyOffhostBackup({
|
||||
lastAttemptAt: state?.lastAttemptAt ?? null,
|
||||
lastSuccessAt: state?.lastSuccessAt ?? null,
|
||||
now,
|
||||
});
|
||||
return {
|
||||
userId: account.id,
|
||||
username: account.username,
|
||||
lastAttemptAt: state?.lastAttemptAt.toISOString() ?? null,
|
||||
lastSuccessAt: state?.lastSuccessAt?.toISOString() ?? null,
|
||||
// BigInt column -> number on the wire. The value is bytes of one
|
||||
// object, capped by the uploader at 80 GB, so it is nowhere near
|
||||
// Number.MAX_SAFE_INTEGER and JSON has no BigInt.
|
||||
sizeBytes:
|
||||
state?.sizeBytes === null || state?.sizeBytes === undefined
|
||||
? null
|
||||
: Number(state.sizeBytes),
|
||||
ageHours: verdict.ageHours,
|
||||
freshness: verdict.freshness,
|
||||
};
|
||||
});
|
||||
|
||||
// Worst first. Sorting by username puts the one account that needs
|
||||
// attention wherever the alphabet happens to leave it, which on a
|
||||
// hundred-account cohort is the same "one bad row hides in the crowd"
|
||||
// problem this card was built to end, moved out of a count and into a list.
|
||||
const FRESHNESS_ORDER: Record<OffhostBackupFreshness, number> = {
|
||||
stale: 0,
|
||||
due: 1,
|
||||
never: 2,
|
||||
unknown: 3,
|
||||
fresh: 4,
|
||||
};
|
||||
offhostRows.sort(
|
||||
(a, b) =>
|
||||
FRESHNESS_ORDER[a.freshness] - FRESHNESS_ORDER[b.freshness] ||
|
||||
a.username.localeCompare(b.username),
|
||||
);
|
||||
|
||||
const payload: BackupsList = {
|
||||
rows: list,
|
||||
// Matches the retention window the backup-prune job enforces so the
|
||||
@@ -77,8 +166,15 @@ export const GET = apiHandler(async () => {
|
||||
.filter((row) => row.type === SCHEDULED_BACKUP_TYPE)
|
||||
.map((row) => row.created_at),
|
||||
lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE),
|
||||
now: new Date(),
|
||||
now,
|
||||
}),
|
||||
offhost: {
|
||||
configured,
|
||||
periodHours: OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
// An unconfigured host has nothing to list, and the card says so rather
|
||||
// than painting an empty table that reads as "no problems".
|
||||
rows: configured ? offhostRows : [],
|
||||
},
|
||||
};
|
||||
|
||||
return apiSuccess(payload);
|
||||
|
||||
@@ -16,9 +16,32 @@ import {
|
||||
invalidateServerDefaultTimezone,
|
||||
isValidTimezone,
|
||||
} from "@/lib/tz/resolver";
|
||||
import { summariseGlitchtipDelivery } from "@/lib/monitoring/glitchtip-delivery";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/** The delivery verdict, flattened onto the settings payload. */
|
||||
function summariseGlitchtipDeliveryFields(state: {
|
||||
lastOkAt?: Date | null;
|
||||
lastFailureAt?: Date | null;
|
||||
lastFailureReason?: string | null;
|
||||
}) {
|
||||
const summary = summariseGlitchtipDelivery(
|
||||
{
|
||||
lastOkAt: state.lastOkAt ?? null,
|
||||
lastFailureAt: state.lastFailureAt ?? null,
|
||||
lastFailureReason: state.lastFailureReason ?? null,
|
||||
},
|
||||
new Date(),
|
||||
);
|
||||
return {
|
||||
glitchtipReportsDelivering: summary.reportsDelivering,
|
||||
glitchtipEverDelivered: summary.everDelivered,
|
||||
glitchtipLastFailureReason: summary.lastFailureReason,
|
||||
glitchtipDeliveryWindowHours: summary.windowHours,
|
||||
};
|
||||
}
|
||||
|
||||
export const GET = apiHandler(async () => {
|
||||
await requireAdmin();
|
||||
annotate({ action: { name: "admin.settings.get" } });
|
||||
@@ -48,6 +71,15 @@ export const GET = apiHandler(async () => {
|
||||
glitchtipEnabled: settings?.glitchtipEnabled ?? false,
|
||||
glitchtipDsn: settings?.glitchtipDsn ?? null,
|
||||
glitchtipEnvironment: settings?.glitchtipEnvironment ?? "production",
|
||||
// The delivery outcome, not the intent, and resolved here rather than on
|
||||
// the client: a DSN that is set and parses says a target was typed, only
|
||||
// this says something left the host, and whether it left recently enough
|
||||
// is a question about a clock that belongs with the ledger.
|
||||
...summariseGlitchtipDeliveryFields({
|
||||
lastOkAt: settings?.glitchtipLastOkAt ?? null,
|
||||
lastFailureAt: settings?.glitchtipLastFailureAt ?? null,
|
||||
lastFailureReason: settings?.glitchtipLastFailureReason ?? null,
|
||||
}),
|
||||
reminderLateMinutes: settings?.reminderLateMinutes ?? 120,
|
||||
reminderMissedMinutes: settings?.reminderMissedMinutes ?? 240,
|
||||
// Document vault limits (per-file cap + per-user quota default). BigInt
|
||||
@@ -274,6 +306,11 @@ export const PUT = apiHandler(async (request: NextRequest) => {
|
||||
glitchtipEnabled: settings.glitchtipEnabled,
|
||||
glitchtipDsn: settings.glitchtipDsn,
|
||||
glitchtipEnvironment: settings.glitchtipEnvironment ?? "production",
|
||||
...summariseGlitchtipDeliveryFields({
|
||||
lastOkAt: settings.glitchtipLastOkAt,
|
||||
lastFailureAt: settings.glitchtipLastFailureAt,
|
||||
lastFailureReason: settings.glitchtipLastFailureReason,
|
||||
}),
|
||||
reminderLateMinutes: settings.reminderLateMinutes,
|
||||
reminderMissedMinutes: settings.reminderMissedMinutes,
|
||||
documentMaxFileBytes: settings.documentMaxFileBytes,
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* The GlitchTip badge answers "are reports arriving", not "did somebody type
|
||||
* a DSN".
|
||||
*
|
||||
* The two questions used to have one answer: enabled plus a URL that parses.
|
||||
* A wrong public key, a wrong project id and a host that stopped answering all
|
||||
* parse perfectly, so a green badge over any of them is the false reassurance
|
||||
* an operator only discovers when they go looking for a crash that was never
|
||||
* reported. The card now reads the delivery outcome this host recorded.
|
||||
*/
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import type { AdminSettings } from "../_shared";
|
||||
|
||||
const settingsResult = vi.hoisted(() => ({
|
||||
current: null as Partial<AdminSettings> | null,
|
||||
}));
|
||||
|
||||
vi.mock("@tanstack/react-query", () => ({
|
||||
useQuery: () => ({ data: settingsResult.current }),
|
||||
useMutation: () => ({ mutate: vi.fn(), isPending: false }),
|
||||
useQueryClient: () => ({ invalidateQueries: vi.fn() }),
|
||||
}));
|
||||
|
||||
vi.mock("../_shared", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../_shared")>();
|
||||
return {
|
||||
...actual,
|
||||
useAdminSettings: () => ({ data: settingsResult.current }),
|
||||
useUpdateSettings: () => ({ mutate: vi.fn(), isPending: false }),
|
||||
};
|
||||
});
|
||||
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
import { GlitchtipSection } from "../glitchtip-section";
|
||||
|
||||
const BASE = {
|
||||
glitchtipEnabled: true,
|
||||
glitchtipDsn: "https://public-key@glitchtip.example.com/7",
|
||||
glitchtipEnvironment: "production",
|
||||
glitchtipReportsDelivering: false,
|
||||
glitchtipEverDelivered: false,
|
||||
glitchtipLastFailureReason: null,
|
||||
glitchtipDeliveryWindowHours: 336,
|
||||
} satisfies Partial<AdminSettings>;
|
||||
|
||||
function render(settings: Partial<AdminSettings> | null): string {
|
||||
settingsResult.current = settings;
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<GlitchtipSection />
|
||||
</I18nProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
describe("<GlitchtipSection> delivery badge", () => {
|
||||
it("does not call a parseable DSN configured until something has left the host", () => {
|
||||
const html = render(BASE);
|
||||
expect(html).toContain("Unproven");
|
||||
expect(html).not.toContain(">Configured<");
|
||||
expect(html).toContain("no report has left this host yet");
|
||||
});
|
||||
|
||||
it("names the failure when the target refused the last report", () => {
|
||||
const html = render({ ...BASE, glitchtipLastFailureReason: "http_403" });
|
||||
expect(html).toContain("Unproven");
|
||||
expect(html).toContain("did not get through");
|
||||
expect(html).toContain("http_403");
|
||||
});
|
||||
|
||||
it("goes green only for a report that left inside the window", () => {
|
||||
const html = render({
|
||||
...BASE,
|
||||
glitchtipReportsDelivering: true,
|
||||
glitchtipEverDelivered: true,
|
||||
});
|
||||
expect(html).toContain(">Configured<");
|
||||
expect(html).toContain("glitchtip.example.com");
|
||||
expect(html).not.toContain('data-slot="glitchtip-delivery"');
|
||||
});
|
||||
|
||||
it("stops claiming delivery once the last success ages out of the window", () => {
|
||||
const html = render({
|
||||
...BASE,
|
||||
glitchtipDeliveryWindowHours: 24,
|
||||
glitchtipEverDelivered: true,
|
||||
});
|
||||
expect(html).toContain("Unproven");
|
||||
expect(html).toContain("No report has left this host in the last");
|
||||
});
|
||||
|
||||
it("says nothing is sent when the switch is off", () => {
|
||||
const html = render({ ...BASE, glitchtipEnabled: false });
|
||||
expect(html).toContain("Error reports stay on this host");
|
||||
expect(html).not.toContain("Unproven");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* The off-host card paints what the response says, including the case an
|
||||
* operator most needs stated out loud: this host is not sending anything
|
||||
* anywhere. A card that renders an empty list on an unconfigured host reads
|
||||
* as "no problems".
|
||||
*/
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import type { BackupsList } from "@/types/backups";
|
||||
|
||||
const queryResult = vi.hoisted(() => ({
|
||||
current: {
|
||||
data: null as BackupsList | null,
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@tanstack/react-query", () => ({
|
||||
useQuery: () => ({ ...queryResult.current, refetch: vi.fn() }),
|
||||
}));
|
||||
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
import { OffhostBackupsSection } from "../offhost-backups-section";
|
||||
|
||||
function render(data: BackupsList["offhost"] | null): string {
|
||||
queryResult.current = {
|
||||
data:
|
||||
data === null
|
||||
? null
|
||||
: ({
|
||||
rows: [],
|
||||
retentionDays: 90,
|
||||
schedule: null,
|
||||
offhost: data,
|
||||
} as unknown as BackupsList),
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
};
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<OffhostBackupsSection />
|
||||
</I18nProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
describe("<OffhostBackupsSection>", () => {
|
||||
it("says off-host backup is not configured rather than showing an empty list", () => {
|
||||
const html = render({ configured: false, periodHours: 24, rows: [] });
|
||||
expect(html).toContain("Off-host backup is not configured");
|
||||
expect(html).not.toContain('data-slot="offhost-backup-rows"');
|
||||
});
|
||||
|
||||
it("paints one row per account with its verdict", () => {
|
||||
const html = render({
|
||||
configured: true,
|
||||
periodHours: 24,
|
||||
rows: [
|
||||
{
|
||||
userId: "u1",
|
||||
username: "account-one",
|
||||
lastAttemptAt: "2026-09-10T06:00:00.000Z",
|
||||
lastSuccessAt: "2026-09-10T06:00:00.000Z",
|
||||
sizeBytes: 4096,
|
||||
ageHours: 3,
|
||||
freshness: "fresh",
|
||||
},
|
||||
{
|
||||
userId: "u2",
|
||||
username: "account-two",
|
||||
lastAttemptAt: "2026-09-06T06:00:00.000Z",
|
||||
lastSuccessAt: "2026-09-06T06:00:00.000Z",
|
||||
sizeBytes: 2048,
|
||||
ageHours: 99,
|
||||
freshness: "stale",
|
||||
},
|
||||
{
|
||||
userId: "u3",
|
||||
username: "account-three",
|
||||
lastAttemptAt: "2026-09-10T02:30:00.000Z",
|
||||
lastSuccessAt: null,
|
||||
sizeBytes: null,
|
||||
ageHours: null,
|
||||
freshness: "never",
|
||||
},
|
||||
{
|
||||
userId: "u4",
|
||||
username: "account-four",
|
||||
lastAttemptAt: null,
|
||||
lastSuccessAt: null,
|
||||
sizeBytes: null,
|
||||
ageHours: null,
|
||||
freshness: "unknown",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(html).toContain('data-offhost-freshness="fresh"');
|
||||
expect(html).toContain('data-offhost-freshness="stale"');
|
||||
expect(html).toContain('data-offhost-freshness="never"');
|
||||
expect(html).toContain('data-offhost-freshness="unknown"');
|
||||
expect(html).toContain("account-two");
|
||||
// The account a run walked and found nothing for says that, with the
|
||||
// instant it was checked.
|
||||
expect(html).toContain("Nothing has landed for this account");
|
||||
// The account no run has recorded says THAT — not "never", which would
|
||||
// claim the bucket is empty on the strength of an empty ledger.
|
||||
expect(html).toContain("No nightly run has recorded this account yet");
|
||||
// Never the bucket's own coordinates.
|
||||
expect(html).not.toMatch(/secret|access[- ]key/i);
|
||||
});
|
||||
});
|
||||
@@ -101,6 +101,17 @@ export interface AdminSettings {
|
||||
glitchtipEnabled: boolean;
|
||||
glitchtipDsn: string | null;
|
||||
glitchtipEnvironment: string | null;
|
||||
/** A report left this host successfully inside the delivery window. */
|
||||
glitchtipReportsDelivering: boolean;
|
||||
/** A report has left this host successfully at some point. */
|
||||
glitchtipEverDelivered: boolean;
|
||||
/**
|
||||
* Why the last attempt failed, when that failure is newer than the last
|
||||
* success. A short classification, never a remote response body.
|
||||
*/
|
||||
glitchtipLastFailureReason: string | null;
|
||||
/** How long a successful send stands for before the badge stops claiming it. */
|
||||
glitchtipDeliveryWindowHours: number;
|
||||
reminderLateMinutes: number;
|
||||
reminderMissedMinutes: number;
|
||||
// v1.4.25 W7 — null means "fall back to Europe/Berlin in the resolver".
|
||||
|
||||
@@ -47,6 +47,7 @@ import { SettingsCard } from "@/components/settings/settings-card";
|
||||
import { SettingsCardHeader } from "@/components/settings/_card-header";
|
||||
import { ListRow } from "@/components/ui/list-row";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import { formatBytes } from "@/lib/format/bytes";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import { randomId } from "@/lib/random-id";
|
||||
import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
@@ -322,14 +323,6 @@ function RestoreRowDialog({
|
||||
);
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number, fmt: ReturnType<typeof useFormatters>) {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) {
|
||||
return `${fmt.integer(Math.round(bytes / 1024))} KB`;
|
||||
}
|
||||
return `${fmt.number(bytes / 1024 / 1024, 2)} MB`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the `DataBackup.type` enum (`WEEKLY_AUTO` / `MANUAL`) as a
|
||||
* human label. Unknown values fall through to the raw enum so a future
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { AlertTriangle, Loader2 } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { SettingsCard } from "@/components/settings/settings-card";
|
||||
import { SettingsCardActions } from "@/components/settings/_card-actions";
|
||||
import { SettingsCardHeader } from "@/components/settings/_card-header";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
@@ -18,9 +19,11 @@ import {
|
||||
ConfiguredBadge,
|
||||
} from "./_shared";
|
||||
import { apiPost } from "@/lib/api/api-fetch";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
|
||||
export function GlitchtipSection() {
|
||||
const { t } = useTranslations();
|
||||
const queryClient = useQueryClient();
|
||||
const { data: settings } = useAdminSettings();
|
||||
const updateSettings = useUpdateSettings();
|
||||
const [glitchtipDsnDraft, setGlitchtipDsnDraft] = useState<string | null>(
|
||||
@@ -34,7 +37,63 @@ export function GlitchtipSection() {
|
||||
const glitchtipEnvironmentValue =
|
||||
glitchtipEnvironmentDraft ?? settings?.glitchtipEnvironment ?? "production";
|
||||
|
||||
const configured = Boolean(settings?.glitchtipDsn);
|
||||
/**
|
||||
* The host the DSN addresses, and nothing else from it. An operator asking
|
||||
* "where do my errors go" is asking for a hostname; the public key in the
|
||||
* DSN answers a different question and does not belong in a status line.
|
||||
*
|
||||
* There is deliberately no default here and none anywhere else in the tree:
|
||||
* a self-hoster's crash reports must not leave their host because nobody
|
||||
* told them not to. Reporting starts when this operator types a target.
|
||||
*/
|
||||
const targetHost = ((): string | null => {
|
||||
const dsn = settings?.glitchtipDsn;
|
||||
if (!dsn) return null;
|
||||
try {
|
||||
return new URL(dsn).host || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
|
||||
/**
|
||||
* Three states, because "a target was typed" and "reports are arriving" are
|
||||
* different facts and only the second earns a green badge.
|
||||
*
|
||||
* off — the switch is off, or no target parses. Nothing is sent.
|
||||
* unproven — a target is set and nothing has come back from it inside the
|
||||
* window. A wrong public key, a wrong project id and a host
|
||||
* that stopped answering all look exactly like this, which is
|
||||
* why it is not green.
|
||||
* delivering — a report left successfully inside the window. The test
|
||||
* button is the cheap way to prove it on demand.
|
||||
*/
|
||||
const deliveryState = ((): "off" | "unproven" | "delivering" => {
|
||||
if (!settings?.glitchtipEnabled || !targetHost) return "off";
|
||||
// Whether the last success is recent enough is decided where the ledger
|
||||
// lives, not here: a render is not the place to read a clock.
|
||||
return settings.glitchtipReportsDelivering ? "delivering" : "unproven";
|
||||
})();
|
||||
|
||||
/**
|
||||
* Why the badge is not green, in words. A reason the host recorded beats a
|
||||
* generic "unproven": an operator who typed a DSN with the wrong project id
|
||||
* gets told the target refused it rather than left to guess.
|
||||
*/
|
||||
const unprovenReason = ((): string | null => {
|
||||
if (deliveryState !== "unproven") return null;
|
||||
if (settings?.glitchtipLastFailureReason) {
|
||||
return t("admin.glitchtipDeliveryFailed", {
|
||||
reason: settings.glitchtipLastFailureReason,
|
||||
});
|
||||
}
|
||||
if (settings?.glitchtipEverDelivered) {
|
||||
return t("admin.glitchtipDeliveryStale", {
|
||||
hours: settings.glitchtipDeliveryWindowHours,
|
||||
});
|
||||
}
|
||||
return t("admin.glitchtipDeliveryUnproven");
|
||||
})();
|
||||
|
||||
const testGlitchtip = useMutation({
|
||||
mutationFn: async () => {
|
||||
@@ -45,6 +104,11 @@ export function GlitchtipSection() {
|
||||
},
|
||||
onSuccess: (message) => {
|
||||
toast.success(message);
|
||||
// The send just recorded an outcome. Re-read it, so the badge answers
|
||||
// the question the operator pressed the button to ask.
|
||||
void queryClient.invalidateQueries({
|
||||
queryKey: queryKeys.adminSettings(),
|
||||
});
|
||||
},
|
||||
onError: (error) => {
|
||||
toast.error(
|
||||
@@ -52,6 +116,9 @@ export function GlitchtipSection() {
|
||||
? error.message
|
||||
: t("admin.monitoringTestFailed"),
|
||||
);
|
||||
void queryClient.invalidateQueries({
|
||||
queryKey: queryKeys.adminSettings(),
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
@@ -76,7 +143,15 @@ export function GlitchtipSection() {
|
||||
icon={AlertTriangle}
|
||||
title={t("admin.glitchtipTitle")}
|
||||
description={t("admin.glitchtipDescription")}
|
||||
status={configured ? <ConfiguredBadge /> : null}
|
||||
status={
|
||||
deliveryState === "delivering" ? (
|
||||
<ConfiguredBadge />
|
||||
) : deliveryState === "unproven" ? (
|
||||
<Badge className="border-warning/30 bg-warning/15 text-warning">
|
||||
{t("admin.glitchtipUnproven")}
|
||||
</Badge>
|
||||
) : null
|
||||
}
|
||||
/>
|
||||
|
||||
<div className="space-y-3">
|
||||
@@ -89,6 +164,28 @@ export function GlitchtipSection() {
|
||||
}
|
||||
disabled={updateSettings.isPending}
|
||||
/>
|
||||
{/* Only once the settings are in hand: a line that says "nothing is
|
||||
sent" while the answer is still loading is a claim, not a
|
||||
placeholder. */}
|
||||
{settings ? (
|
||||
<div className="space-y-1">
|
||||
<p className="text-sm" data-slot="glitchtip-target">
|
||||
{deliveryState !== "off" && targetHost
|
||||
? t("admin.glitchtipTargetOn", { host: targetHost })
|
||||
: settings.glitchtipEnabled
|
||||
? t("admin.glitchtipTargetNoTarget")
|
||||
: t("admin.glitchtipTargetOff")}
|
||||
</p>
|
||||
{unprovenReason ? (
|
||||
<p
|
||||
className="text-muted-foreground text-xs"
|
||||
data-slot="glitchtip-delivery"
|
||||
>
|
||||
{unprovenReason}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
<div className="space-y-1.5 sm:col-span-2">
|
||||
<Label htmlFor="admin-glitchtip-dsn" className="text-xs">
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* `<OffhostBackupsSection>` — per-account freshness of the off-host copies.
|
||||
*
|
||||
* The card above it lists the copies that live in THIS database. This one
|
||||
* answers the different question: for each account, how long ago did this host
|
||||
* put an encrypted copy in the operator's bucket, and is that inside what the
|
||||
* nightly schedule promises. The two fail independently — a database whose
|
||||
* weekly rows are current tells an operator nothing about a bucket that has
|
||||
* been refusing the worker's signature since March.
|
||||
*
|
||||
* Every verdict is computed server-side from the ledger the worker writes. The
|
||||
* page never lists the bucket: the worker's grant is deliberately PutObject,
|
||||
* GetObject and AbortMultipartUpload — the three the runbook documents, the
|
||||
* abort being what the multipart path needs — and putting a credentialed
|
||||
* listing call on the render path of an admin page would be a poor trade for
|
||||
* a number the database already holds.
|
||||
*/
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { CloudUpload } from "lucide-react";
|
||||
import { SettingsCard } from "@/components/settings/settings-card";
|
||||
import { SettingsCardHeader } from "@/components/settings/_card-header";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { EmptyState } from "@/components/ui/empty-state";
|
||||
import { ListRow } from "@/components/ui/list-row";
|
||||
import { QueryErrorRow } from "@/components/ui/query-error-row";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { apiGet } from "@/lib/api/api-fetch";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness";
|
||||
import type { BackupsList, OffhostAccountRow } from "@/types/backups";
|
||||
import { formatBytes } from "@/lib/format/bytes";
|
||||
|
||||
/**
|
||||
* Status wash per verdict. `never` and `unknown` are deliberately neutral
|
||||
* rather than alarming: a host that turned the off-host job on this afternoon
|
||||
* has not failed at anything yet, and painting every row red on day one is how
|
||||
* an operator learns to stop reading the colour.
|
||||
*/
|
||||
const FRESHNESS_STYLE: Record<OffhostBackupFreshness, string> = {
|
||||
fresh: "border-success/30 bg-success/10 text-success",
|
||||
due: "border-warning/30 bg-warning/10 text-warning",
|
||||
stale: "border-destructive/30 bg-destructive/10 text-destructive",
|
||||
never: "border-border bg-muted text-muted-foreground",
|
||||
unknown: "border-border bg-muted text-muted-foreground",
|
||||
};
|
||||
|
||||
function freshnessLabel(
|
||||
freshness: OffhostBackupFreshness,
|
||||
t: ReturnType<typeof useTranslations>["t"],
|
||||
): string {
|
||||
// Literal keys, one per arm: a template-built key resolves at runtime and is
|
||||
// invisible to the call-site coverage guard.
|
||||
switch (freshness) {
|
||||
case "fresh":
|
||||
return t("admin.section.backups.offhost.stateFresh");
|
||||
case "due":
|
||||
return t("admin.section.backups.offhost.stateDue");
|
||||
case "stale":
|
||||
return t("admin.section.backups.offhost.stateStale");
|
||||
case "never":
|
||||
return t("admin.section.backups.offhost.stateNever");
|
||||
case "unknown":
|
||||
return t("admin.section.backups.offhost.stateUnknown");
|
||||
}
|
||||
}
|
||||
|
||||
function AccountRow({ row }: { row: OffhostAccountRow }) {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
|
||||
return (
|
||||
<ListRow asChild className="bg-muted/30 border-border">
|
||||
<li
|
||||
data-slot="offhost-backup-row"
|
||||
data-offhost-username={row.username}
|
||||
data-offhost-freshness={row.freshness}
|
||||
>
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<span className="min-w-0 flex-1 truncate font-medium">
|
||||
{row.username}
|
||||
</span>
|
||||
<Badge
|
||||
variant="outline"
|
||||
className={cn("shrink-0", FRESHNESS_STYLE[row.freshness])}
|
||||
>
|
||||
{freshnessLabel(row.freshness, t)}
|
||||
</Badge>
|
||||
</div>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{row.lastAttemptAt === null
|
||||
? // No run has recorded this account. Saying "never" here would
|
||||
// claim the bucket is empty when what is empty is the ledger,
|
||||
// which is exactly what every account looks like between the
|
||||
// upgrade that adds it and the first nightly run after that.
|
||||
t("admin.section.backups.offhost.unknownDetail")
|
||||
: row.lastSuccessAt === null || row.sizeBytes === null
|
||||
? t("admin.section.backups.offhost.neverDetail", {
|
||||
when: fmt.dateTime(row.lastAttemptAt),
|
||||
})
|
||||
: `${fmt.dateTime(row.lastSuccessAt)} · ${formatBytes(
|
||||
row.sizeBytes,
|
||||
fmt,
|
||||
)} · ${t("admin.section.backups.offhost.age", {
|
||||
hours: row.ageHours ?? 0,
|
||||
})}`}
|
||||
</p>
|
||||
</li>
|
||||
</ListRow>
|
||||
);
|
||||
}
|
||||
|
||||
export function OffhostBackupsSection() {
|
||||
const { t } = useTranslations();
|
||||
|
||||
const { data, isLoading, isError, refetch } = useQuery({
|
||||
queryKey: queryKeys.adminBackups(),
|
||||
queryFn: async () => apiGet<BackupsList>("/api/admin/backups"),
|
||||
});
|
||||
|
||||
const offhost = data?.offhost;
|
||||
|
||||
return (
|
||||
<SettingsCard>
|
||||
<SettingsCardHeader
|
||||
icon={CloudUpload}
|
||||
title={t("admin.section.backups.offhost.title")}
|
||||
description={t("admin.section.backups.offhost.description")}
|
||||
status={
|
||||
offhost && offhost.configured ? (
|
||||
<Badge variant="secondary" className="text-xs">
|
||||
{offhost.rows.length}
|
||||
</Badge>
|
||||
) : null
|
||||
}
|
||||
/>
|
||||
|
||||
{isLoading ? (
|
||||
<div className="space-y-2" data-slot="offhost-backup-loading">
|
||||
<Skeleton className="h-14 w-full" />
|
||||
<Skeleton className="h-14 w-full" />
|
||||
</div>
|
||||
) : isError ? (
|
||||
<QueryErrorRow
|
||||
message={t("admin.section.backups.offhost.loadError")}
|
||||
onRetry={() => void refetch()}
|
||||
/>
|
||||
) : offhost && !offhost.configured ? (
|
||||
<EmptyState
|
||||
icon={<CloudUpload className="size-6" />}
|
||||
title={t("admin.section.backups.offhost.notConfiguredTitle")}
|
||||
description={t(
|
||||
"admin.section.backups.offhost.notConfiguredDescription",
|
||||
)}
|
||||
/>
|
||||
) : offhost ? (
|
||||
<div className="space-y-2">
|
||||
<ul className="space-y-2" data-slot="offhost-backup-rows">
|
||||
{offhost.rows.map((row) => (
|
||||
<AccountRow key={row.userId} row={row} />
|
||||
))}
|
||||
</ul>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{t("admin.section.backups.offhost.scheduleNote", {
|
||||
hours: offhost.periodHours,
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
) : null}
|
||||
</SettingsCard>
|
||||
);
|
||||
}
|
||||
@@ -210,6 +210,11 @@ export const WIPE_MODELS = [
|
||||
// just returned.
|
||||
"IdempotencyKey",
|
||||
"DataBackup",
|
||||
// What this host last put in the operator's bucket for this account. The
|
||||
// objects themselves stay — the worker holds no DeleteObject grant and the
|
||||
// bucket's lifecycle rule owns their retirement — but the row saying when
|
||||
// they were written is about this account and goes with it.
|
||||
"OffhostBackupState",
|
||||
"AuditLog",
|
||||
"ConsentReceipt",
|
||||
"UserKnownDevice",
|
||||
|
||||
@@ -642,6 +642,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly<Record<string, string>> = {
|
||||
"Replay protection for requests this host already answered. Restoring it would suppress a legitimate new request.",
|
||||
DataBackup:
|
||||
"The backup catalogue itself. A backup that contains the list of backups is a recursion with no reader.",
|
||||
OffhostBackupState:
|
||||
"When this host last put this account's copy in this operator's bucket, and how big it was. It describes one deployment's relationship with one bucket, so restoring it elsewhere would assert an off-host copy that host has never written.",
|
||||
ImportJob:
|
||||
"A job record pointing at an uploaded file that the backup does not carry, so restoring it would resurrect a task with nothing to work on.",
|
||||
InviteToken:
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { useFormatters } from "@/lib/i18n/context";
|
||||
|
||||
/**
|
||||
* One spelling of "1.25 MB" for every surface that shows a byte count from
|
||||
* the admin backups response.
|
||||
*
|
||||
* It lives here rather than beside the card that happened to need it first:
|
||||
* the second reader was importing a formatter out of a thousand-line client
|
||||
* component, which is a dependency on a page's worth of code for four lines of
|
||||
* arithmetic. Both cards render on the same route today, so it costs nothing
|
||||
* today — and it becomes a cost the moment either one moves.
|
||||
*/
|
||||
export function formatBytes(
|
||||
bytes: number,
|
||||
fmt: ReturnType<typeof useFormatters>,
|
||||
): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) {
|
||||
return `${fmt.integer(Math.round(bytes / 1024))} KB`;
|
||||
}
|
||||
return `${fmt.number(bytes / 1024 / 1024, 2)} MB`;
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
OFFHOST_BACKUP_GRACE_HOURS,
|
||||
OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
} from "@/lib/jobs/offhost-backup-freshness";
|
||||
|
||||
const NOW = new Date("2026-09-10T09:00:00.000Z");
|
||||
const HOUR = 3_600_000;
|
||||
const PERIOD = OFFHOST_BACKUP_PERIOD_HOURS * HOUR;
|
||||
const GRACE = OFFHOST_BACKUP_GRACE_HOURS * HOUR;
|
||||
|
||||
/** `age` hours before NOW. */
|
||||
function ago(ms: number): Date {
|
||||
return new Date(NOW.getTime() - ms);
|
||||
}
|
||||
|
||||
describe("classifyOffhostBackup", () => {
|
||||
it("says unknown, not never, when no run has recorded this account", () => {
|
||||
// The ledger starts empty. On the first morning after the upgrade every
|
||||
// account on a perfectly healthy host looks like this, and `never` would
|
||||
// assert something about the bucket the host has not looked at.
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: null,
|
||||
lastSuccessAt: null,
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ freshness: "unknown", ageHours: null });
|
||||
});
|
||||
|
||||
it("says never once a run has walked the account and put nothing there", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(HOUR),
|
||||
lastSuccessAt: null,
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ freshness: "never", ageHours: null });
|
||||
});
|
||||
|
||||
it("is fresh right up to one period plus the grace and due one millisecond past it", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(PERIOD + GRACE),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("fresh");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(PERIOD + GRACE + 1),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("due");
|
||||
});
|
||||
|
||||
it("is due right up to two periods plus the grace and stale one millisecond past it", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(PERIOD * 2 + GRACE),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("due");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(PERIOD * 2 + GRACE + 1),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("stale");
|
||||
});
|
||||
|
||||
it("keeps the whole cohort fresh across the Berlin DST fall-back night", () => {
|
||||
// 2026-10-25 is the fall-back Sunday in Europe/Berlin: the clocks go back
|
||||
// at 03:00 CEST, so 02:30 local happens 25 hours after the previous
|
||||
// 02:30 local. Both instants are pinned in UTC because that is what the
|
||||
// ledger stores, and the test must not depend on the runner's own zone.
|
||||
const beforeTheChange = new Date("2026-10-24T00:30:00.000Z"); // 02:30 CEST
|
||||
const afterTheChange = new Date("2026-10-25T01:30:00.000Z"); // 02:30 CET
|
||||
const gapHours =
|
||||
(afterTheChange.getTime() - beforeTheChange.getTime()) / HOUR;
|
||||
expect(gapHours).toBe(25);
|
||||
|
||||
// Without slack the whole cohort would read `due` on that one morning,
|
||||
// on every host, with nothing wrong anywhere.
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: beforeTheChange,
|
||||
now: afterTheChange,
|
||||
}).freshness,
|
||||
).toBe("fresh");
|
||||
});
|
||||
|
||||
it("still calls a genuinely missed night due", () => {
|
||||
// 31 hours: past one period even with six hours of slack, so a run that
|
||||
// produced nothing for this account still shows.
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(31 * HOUR),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("due");
|
||||
});
|
||||
|
||||
it("reports whole hours of age", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(90 * 60_000),
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ freshness: "fresh", ageHours: 1 });
|
||||
});
|
||||
|
||||
it("honours a caller's own period rather than the nightly default", () => {
|
||||
// A six-hour schedule: the same 12-hour-old object that is fresh under the
|
||||
// nightly cron is two periods behind under this one. The grace is passed
|
||||
// as zero so the thresholds under test are the caller's period alone.
|
||||
const twelveHours = ago(12 * HOUR);
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("fresh");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
periodHours: 6,
|
||||
graceHours: 0,
|
||||
}).freshness,
|
||||
).toBe("due");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
periodHours: 5,
|
||||
graceHours: 0,
|
||||
}).freshness,
|
||||
).toBe("stale");
|
||||
});
|
||||
|
||||
it("does not call a copy stale because the host clock ran backwards", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: new Date(NOW.getTime() + 2 * HOUR),
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ freshness: "fresh", ageHours: 0 });
|
||||
});
|
||||
});
|
||||
@@ -184,6 +184,7 @@ describe("runOffhostBackup", () => {
|
||||
s3.store.set("2020-01-01/user-old.json.enc", Buffer.from([0]));
|
||||
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: {
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
@@ -229,11 +230,28 @@ describe("runOffhostBackup", () => {
|
||||
// Nothing went through the whole-buffer arm.
|
||||
expect(s3.putObject).not.toHaveBeenCalled();
|
||||
expect(s3.putStream).toHaveBeenCalledTimes(2);
|
||||
|
||||
// Every account that got an object gets a ledger row, because the run's
|
||||
// own counts cannot say WHICH account has a copy off-host and which one
|
||||
// has been failing quietly since March.
|
||||
expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(2);
|
||||
const ledgered = prisma.offhostBackupState.upsert.mock.calls.map(
|
||||
(call) => (call[0] as { where: { userId: string } }).where.userId,
|
||||
);
|
||||
expect(ledgered).toEqual(["u1", "u2"]);
|
||||
const first = prisma.offhostBackupState.upsert.mock.calls[0]?.[0] as {
|
||||
create: { sizeBytes: bigint; lastSuccessAt: Date };
|
||||
};
|
||||
// BigInt, because the uploader permits an 80 GB object and a 32-bit
|
||||
// column would refuse the biggest account on the host.
|
||||
expect(first.create.sizeBytes).toBe(BigInt(ct.byteLength));
|
||||
expect(first.create.lastSuccessAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it("uploads the canonical builder output without reshaping it", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
const canonicalPayload = {
|
||||
@@ -290,7 +308,9 @@ describe("runOffhostBackup", () => {
|
||||
|
||||
it("counts per-user failures without aborting the whole run", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const upsert = vi.fn();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert },
|
||||
user: {
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
@@ -314,11 +334,55 @@ describe("runOffhostBackup", () => {
|
||||
|
||||
expect(report.uploaded).toBe(1);
|
||||
expect(report.failed).toBe(1);
|
||||
// Both accounts are recorded as walked — that is what separates "a run
|
||||
// looked and found nothing" from "no run has looked at this account" —
|
||||
// but only the one whose object landed carries a success.
|
||||
type Upsert = {
|
||||
where: { userId: string };
|
||||
update: Record<string, unknown>;
|
||||
create: Record<string, unknown>;
|
||||
};
|
||||
const calls = upsert.mock.calls.map((call) => call[0] as Upsert);
|
||||
expect(calls.map((call) => call.where.userId)).toEqual(["u1", "u2"]);
|
||||
expect(calls[0]?.update).toHaveProperty("lastSuccessAt");
|
||||
// The guarantee an operator leans on: an account whose upload failed keeps
|
||||
// whatever `lastSuccessAt` it already had. The failed arm names only the
|
||||
// walk instant, so yesterday's good copy is still what the console
|
||||
// reports — the run does not overwrite it with today's failure.
|
||||
expect(calls[1]?.update).not.toHaveProperty("lastSuccessAt");
|
||||
expect(calls[1]?.update).not.toHaveProperty("sizeBytes");
|
||||
expect(calls[1]?.create).not.toHaveProperty("lastSuccessAt");
|
||||
});
|
||||
|
||||
it("still reports a landed object as uploaded when the ledger write fails", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: {
|
||||
upsert: vi.fn().mockRejectedValue(new Error("pool timeout")),
|
||||
},
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
|
||||
const report = await runOffhostBackup(
|
||||
prisma as never,
|
||||
s3,
|
||||
new Date("2026-05-08T00:00:00Z"),
|
||||
);
|
||||
|
||||
// The object is in the bucket. Only the record of it is missing, and that
|
||||
// is a different fact: counting it as a failure would tell the operator
|
||||
// their disaster-recovery copy does not exist when it does.
|
||||
expect(report.uploaded).toBe(1);
|
||||
expect(report.failed).toBe(0);
|
||||
expect(report.failures).toEqual([]);
|
||||
expect(s3.store.has("2026-05-08/user-u1.json.enc")).toBe(true);
|
||||
expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("refuses an account whose object outgrows one multipart upload", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
mocks.buildFullBackupPayload.mockResolvedValue({
|
||||
@@ -355,6 +419,7 @@ describe("runOffhostBackup", () => {
|
||||
it("reports the largest object it wrote", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
const report = await runOffhostBackup(
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* How fresh one account's newest off-host copy is.
|
||||
*
|
||||
* The nightly off-host job reports counts — uploaded, failed, total — and a
|
||||
* count is exactly the shape that hides a single account. A cohort where one
|
||||
* account's object has failed every night for a month reports "99 uploaded, 1
|
||||
* failed" every night, which reads like weather rather than like an account
|
||||
* that has no disaster-recovery copy at all. The per-account ledger the worker
|
||||
* writes (`OffhostBackupState`) turns that into a fact the console can state
|
||||
* per row, and this function is the only place that fact is judged.
|
||||
*
|
||||
* Deliberately pure and deliberately not talking to the bucket: the admin page
|
||||
* computes staleness from the timestamp the worker wrote, so the answer is the
|
||||
* same whether or not the app process holds a read grant on the bucket, and a
|
||||
* listing of somebody's backups never crosses the network to show a page.
|
||||
*
|
||||
* Five states, because the run is nightly and one missed night is not the same
|
||||
* event as four — and because "we have not looked yet" is not a verdict:
|
||||
*
|
||||
* unknown — no nightly run has recorded this account. The ledger starts
|
||||
* empty, so this is what every account on a perfectly healthy host
|
||||
* reads between the upgrade that adds the table and the first run
|
||||
* after it. Saying "never" there would assert something the host
|
||||
* cannot know.
|
||||
* never — a run walked this account and put no object there, and no earlier
|
||||
* run ever did either.
|
||||
* fresh — inside one schedule period plus the grace below. This is what
|
||||
* every account looks like on a host where the cron is doing its
|
||||
* job.
|
||||
* due — past one period, inside two. One night produced nothing: a retry, a
|
||||
* long run, a restart during the window. Worth showing, not worth
|
||||
* alarming about.
|
||||
* stale — past two periods. Two consecutive nights produced nothing for this
|
||||
* account, which no ordinary hiccup explains.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Hours between two scheduled off-host runs. Tracks `OFFHOST_BACKUP_CRON`
|
||||
* (`30 2 * * *`) in `src/lib/jobs/reminder/register-maintenance.ts`, which
|
||||
* every schedule runs in `Europe/Berlin` (`tz` in `registrar-shared.ts`).
|
||||
* This is the number the card quotes — the schedule's own period, not the
|
||||
* threshold a verdict is measured against.
|
||||
*/
|
||||
export const OFFHOST_BACKUP_PERIOD_HOURS = 24;
|
||||
|
||||
/**
|
||||
* Slack on top of one period, before a verdict moves.
|
||||
*
|
||||
* Two things make a healthy host miss a bare 24 hours. The schedule is
|
||||
* Berlin-local, so on the DST fall-back night the gap between two 02:30 runs
|
||||
* is 25 hours and every account on every host would cross one period on the
|
||||
* same morning. And the ledger instant is when THAT account's object landed,
|
||||
* deliberately: on a cohort walked one account at a time, an account reached
|
||||
* at 05:30 one night and 06:30 the next is 25 hours old with two entirely
|
||||
* successful runs behind it.
|
||||
*
|
||||
* Six hours is the budget for both. A card whose argument is that a count
|
||||
* trains an operator to stop reading it cannot afford an annual cohort-wide
|
||||
* false `due`.
|
||||
*/
|
||||
export const OFFHOST_BACKUP_GRACE_HOURS = 6;
|
||||
|
||||
export type OffhostBackupFreshness =
|
||||
"unknown" | "never" | "fresh" | "due" | "stale";
|
||||
|
||||
export interface OffhostBackupVerdict {
|
||||
freshness: OffhostBackupFreshness;
|
||||
/** Whole hours since the newest object landed; null when there is none. */
|
||||
ageHours: number | null;
|
||||
}
|
||||
|
||||
export function classifyOffhostBackup(args: {
|
||||
/**
|
||||
* When a run last walked this account, or null when none has. Null is the
|
||||
* only thing that separates `unknown` from `never`.
|
||||
*/
|
||||
lastAttemptAt: Date | null;
|
||||
lastSuccessAt: Date | null;
|
||||
now: Date;
|
||||
/** Defaults to the nightly schedule. Tests pass their own. */
|
||||
periodHours?: number;
|
||||
/** Defaults to `OFFHOST_BACKUP_GRACE_HOURS`. Tests pass their own. */
|
||||
graceHours?: number;
|
||||
}): OffhostBackupVerdict {
|
||||
const { lastAttemptAt, lastSuccessAt, now } = args;
|
||||
if (lastAttemptAt === null) return { freshness: "unknown", ageHours: null };
|
||||
if (lastSuccessAt === null) return { freshness: "never", ageHours: null };
|
||||
|
||||
const periodMs =
|
||||
(args.periodHours ?? OFFHOST_BACKUP_PERIOD_HOURS) * 3_600_000;
|
||||
const graceMs = (args.graceHours ?? OFFHOST_BACKUP_GRACE_HOURS) * 3_600_000;
|
||||
const ageMs = now.getTime() - lastSuccessAt.getTime();
|
||||
// A clock that ran backwards between the upload and this read reports a
|
||||
// negative age. That is a host-clock problem, not a backup problem, and
|
||||
// calling the copy stale over it would point the operator at the wrong
|
||||
// thing.
|
||||
const ageHours = Math.max(0, Math.floor(ageMs / 3_600_000));
|
||||
|
||||
// The grace rides on each threshold rather than on the period, so `due`
|
||||
// still means "one scheduled run produced nothing" and `stale` still means
|
||||
// two — the slack only keeps a long night from being read as a missed one.
|
||||
if (ageMs <= periodMs + graceMs) return { freshness: "fresh", ageHours };
|
||||
if (ageMs <= periodMs * 2 + graceMs) return { freshness: "due", ageHours };
|
||||
return { freshness: "stale", ageHours };
|
||||
}
|
||||
@@ -28,7 +28,7 @@ import { createGzip, gunzipSync, gzipSync } from "node:zlib";
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { createRawStreamEncryptor, decryptRawStream } from "@/lib/crypto";
|
||||
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
|
||||
const ALGORITHM = "aes-256-gcm";
|
||||
const IV_LENGTH = 12;
|
||||
@@ -61,6 +61,27 @@ function decodeBackupKey(raw: string): Buffer {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this host has the five variables the nightly job needs.
|
||||
*
|
||||
* Separate from `loadOffhostConfig()` because a surface that only wants to say
|
||||
* "off-host backup is not set up here" must not decode the encryption key to
|
||||
* find out — `decodeBackupKey` throws on a malformed one, and an operator
|
||||
* whose key has a typo should see the admin page, not a 500.
|
||||
*/
|
||||
export function offhostBackupConfigured(): boolean {
|
||||
return OFFHOST_ENV_VARS.every((name) => Boolean(process.env[name]));
|
||||
}
|
||||
|
||||
/** The five variables the nightly job needs, in one place. */
|
||||
const OFFHOST_ENV_VARS = [
|
||||
"BACKUP_S3_ENDPOINT",
|
||||
"BACKUP_S3_BUCKET",
|
||||
"BACKUP_S3_ACCESS_KEY",
|
||||
"BACKUP_S3_SECRET_KEY",
|
||||
"BACKUP_ENCRYPTION_KEY",
|
||||
] as const;
|
||||
|
||||
export function loadOffhostConfig(): OffhostBackupConfig | null {
|
||||
const endpoint = process.env.BACKUP_S3_ENDPOINT;
|
||||
const bucket = process.env.BACKUP_S3_BUCKET;
|
||||
@@ -555,10 +576,12 @@ export async function runOffhostBackup(
|
||||
let oversized = 0;
|
||||
let largestObjectBytes = 0;
|
||||
const failures: Array<{ userId: string; message: string }> = [];
|
||||
let ledgerWriteFailures = 0;
|
||||
const evt = getEvent();
|
||||
for (const user of users) {
|
||||
let objectBytes: number | null = null;
|
||||
try {
|
||||
const objectBytes = await uploadEncryptedBackup(
|
||||
objectBytes = await uploadEncryptedBackup(
|
||||
s3,
|
||||
`${dateKey}/user-${user.id}.json.enc`,
|
||||
cfg.encryptionKey,
|
||||
@@ -586,6 +609,62 @@ export async function runOffhostBackup(
|
||||
`offhost-backup user ${user.id} failed: ${message.slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// The per-account ledger, and deliberately NOT inside the upload's `try`.
|
||||
// `uploadEncryptedBackup` resolves only after the multipart completes, so
|
||||
// by here the object is durably in the bucket and this account's backup
|
||||
// has succeeded whatever the database does next. A pool timeout on the
|
||||
// row would otherwise un-count a copy that exists — the run would report
|
||||
// one failure too many and the console would read stale for an account
|
||||
// whose disaster-recovery object is fine, which is exactly the inversion
|
||||
// the ledger exists to remove.
|
||||
//
|
||||
// The row is written whether or not an object came of the walk, because a
|
||||
// ledger keyed only on success cannot tell "no run has reached this
|
||||
// account" from "a run reached it and nothing landed" — and the first of
|
||||
// those is what every account on a running host looks like the day this
|
||||
// table ships. A failed walk touches `lastAttemptAt` and leaves
|
||||
// `lastSuccessAt` exactly as it was, so yesterday's good copy still reads
|
||||
// as the copy this account has.
|
||||
//
|
||||
// `new Date()` rather than the run's `now`, because on a large cohort the
|
||||
// two are hours apart and the row is meant to say when this account was
|
||||
// reached.
|
||||
const at = new Date();
|
||||
try {
|
||||
await prisma.offhostBackupState.upsert({
|
||||
where: { userId: user.id },
|
||||
update:
|
||||
objectBytes === null
|
||||
? { lastAttemptAt: at }
|
||||
: {
|
||||
lastAttemptAt: at,
|
||||
lastSuccessAt: at,
|
||||
sizeBytes: BigInt(objectBytes),
|
||||
},
|
||||
create:
|
||||
objectBytes === null
|
||||
? { userId: user.id, lastAttemptAt: at }
|
||||
: {
|
||||
userId: user.id,
|
||||
lastAttemptAt: at,
|
||||
lastSuccessAt: at,
|
||||
sizeBytes: BigInt(objectBytes),
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
ledgerWriteFailures++;
|
||||
const message = (err as Error).message ?? "unknown";
|
||||
evt?.addWarning(
|
||||
`offhost-backup ledger write failed for ${user.id}: ${message.slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (ledgerWriteFailures > 0) {
|
||||
// A distinct fact from a failed upload, and one an operator has to be able
|
||||
// to tell apart: the copies are in the bucket, the console's per-account
|
||||
// view of them is behind.
|
||||
annotate({ meta: { offhost_ledger_write_failures: ledgerWriteFailures } });
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* The write end of the delivery ledger.
|
||||
*
|
||||
* The badge is only as honest as what gets recorded, and two things have to
|
||||
* hold: a failure must never carry a remote host's response body into this
|
||||
* database, and a send that could not be recorded must not become a second
|
||||
* error on a path that is already handling one.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const updateMock = vi.fn();
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: { appSettings: { update: (...a: unknown[]) => updateMock(...a) } },
|
||||
}));
|
||||
|
||||
import {
|
||||
classifyGlitchtipFailure,
|
||||
GLITCHTIP_DELIVERY_WINDOW_HOURS,
|
||||
recordGlitchtipDelivery,
|
||||
summariseGlitchtipDelivery,
|
||||
} from "../glitchtip-delivery";
|
||||
|
||||
const NOW = new Date("2026-09-10T09:00:00.000Z");
|
||||
const HOUR = 3_600_000;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
updateMock.mockResolvedValue({});
|
||||
});
|
||||
|
||||
describe("classifyGlitchtipFailure", () => {
|
||||
it("keeps the classification short and free of the remote body", () => {
|
||||
expect(
|
||||
classifyGlitchtipFailure({
|
||||
ok: false,
|
||||
status: 403,
|
||||
details: "<html>forbidden, and here is our whole error page</html>",
|
||||
}),
|
||||
).toBe("http_403");
|
||||
expect(
|
||||
classifyGlitchtipFailure({ ok: false, details: "invalid_dsn" }),
|
||||
).toBe("invalid_dsn");
|
||||
expect(classifyGlitchtipFailure({ ok: false })).toBe("network");
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordGlitchtipDelivery", () => {
|
||||
it("stamps a success and leaves the failure fields alone", async () => {
|
||||
await recordGlitchtipDelivery({ ok: true, status: 200 });
|
||||
|
||||
const data = updateMock.mock.calls[0]?.[0]?.data as Record<string, unknown>;
|
||||
expect(data.glitchtipLastOkAt).toBeInstanceOf(Date);
|
||||
expect(data).not.toHaveProperty("glitchtipLastFailureAt");
|
||||
});
|
||||
|
||||
it("records a failure by classification, never by response body", async () => {
|
||||
await recordGlitchtipDelivery({
|
||||
ok: false,
|
||||
status: 401,
|
||||
details: "invalid public key: abcdef0123456789",
|
||||
});
|
||||
|
||||
const data = updateMock.mock.calls[0]?.[0]?.data as Record<string, unknown>;
|
||||
expect(data.glitchtipLastFailureReason).toBe("http_401");
|
||||
expect(JSON.stringify(data)).not.toContain("abcdef0123456789");
|
||||
expect(data).not.toHaveProperty("glitchtipLastOkAt");
|
||||
});
|
||||
|
||||
it("swallows a write failure rather than replacing the original error", async () => {
|
||||
updateMock.mockRejectedValue(new Error("no singleton row yet"));
|
||||
await expect(
|
||||
recordGlitchtipDelivery({ ok: false, status: 500 }),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("summariseGlitchtipDelivery", () => {
|
||||
it("claims delivery only for a success inside the window", () => {
|
||||
const inside = new Date(
|
||||
NOW.getTime() - (GLITCHTIP_DELIVERY_WINDOW_HOURS - 1) * HOUR,
|
||||
);
|
||||
const outside = new Date(
|
||||
NOW.getTime() - (GLITCHTIP_DELIVERY_WINDOW_HOURS + 1) * HOUR,
|
||||
);
|
||||
|
||||
expect(
|
||||
summariseGlitchtipDelivery(
|
||||
{ lastOkAt: inside, lastFailureAt: null, lastFailureReason: null },
|
||||
NOW,
|
||||
).reportsDelivering,
|
||||
).toBe(true);
|
||||
|
||||
const aged = summariseGlitchtipDelivery(
|
||||
{ lastOkAt: outside, lastFailureAt: null, lastFailureReason: null },
|
||||
NOW,
|
||||
);
|
||||
// Still true that something once got through, no longer true that the
|
||||
// target is working — two different claims, and only one is a badge.
|
||||
expect(aged.reportsDelivering).toBe(false);
|
||||
expect(aged.everDelivered).toBe(true);
|
||||
});
|
||||
|
||||
it("says nothing has been proven when the ledger is empty", () => {
|
||||
expect(
|
||||
summariseGlitchtipDelivery(
|
||||
{ lastOkAt: null, lastFailureAt: null, lastFailureReason: null },
|
||||
NOW,
|
||||
),
|
||||
).toEqual({
|
||||
reportsDelivering: false,
|
||||
everDelivered: false,
|
||||
lastFailureReason: null,
|
||||
windowHours: GLITCHTIP_DELIVERY_WINDOW_HOURS,
|
||||
});
|
||||
});
|
||||
|
||||
it("drops a failure that a later success has superseded", () => {
|
||||
const summary = summariseGlitchtipDelivery(
|
||||
{
|
||||
lastOkAt: new Date(NOW.getTime() - HOUR),
|
||||
lastFailureAt: new Date(NOW.getTime() - 5 * HOUR),
|
||||
lastFailureReason: "http_403",
|
||||
},
|
||||
NOW,
|
||||
);
|
||||
// The target refused a report and then accepted one. Showing the refusal
|
||||
// would report a problem that has since been fixed.
|
||||
expect(summary.lastFailureReason).toBeNull();
|
||||
expect(summary.reportsDelivering).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps a failure that is newer than the last success", () => {
|
||||
expect(
|
||||
summariseGlitchtipDelivery(
|
||||
{
|
||||
lastOkAt: new Date(NOW.getTime() - 5 * HOUR),
|
||||
lastFailureAt: new Date(NOW.getTime() - HOUR),
|
||||
lastFailureReason: "http_403",
|
||||
},
|
||||
NOW,
|
||||
).lastFailureReason,
|
||||
).toBe("http_403");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* What actually happened to the last error report this host tried to send.
|
||||
*
|
||||
* The admin card used to derive "configured" from the enabled flag plus a DSN
|
||||
* that parses. Those two say a target was typed; they say nothing about the
|
||||
* target accepting anything. A wrong public key, a wrong project id, a host
|
||||
* that stopped answering — every one of them parses, and every one of them
|
||||
* would have shown a green badge and a line naming the host, which is the
|
||||
* false reassurance an operator only discovers when they go looking for a
|
||||
* crash that was never reported.
|
||||
*
|
||||
* So the outcome is recorded where the DSN already lives, next to the flag it
|
||||
* qualifies, and the badge reads it. `sendGlitchtipEvent` is the one writer —
|
||||
* every caller in the tree goes through it, so no send path can skip the
|
||||
* ledger by construction.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import type { GlitchtipDeliveryResult } from "@/lib/monitoring/glitchtip";
|
||||
|
||||
/**
|
||||
* How long a successful send stands for. A self-hoster's instance can go days
|
||||
* without an unhandled error, and a badge that expired overnight would train
|
||||
* the operator to ignore it; a fortnight is long enough that only a target
|
||||
* that has genuinely stopped working falls out of it, and the test button
|
||||
* refreshes it on demand.
|
||||
*/
|
||||
export const GLITCHTIP_DELIVERY_WINDOW_HOURS = 14 * 24;
|
||||
|
||||
export interface GlitchtipDeliveryState {
|
||||
lastOkAt: Date | null;
|
||||
lastFailureAt: Date | null;
|
||||
lastFailureReason: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A short, safe classification of a failure. Never the response body: that is
|
||||
* text from a remote host, and an incident target's error page has no business
|
||||
* being rendered on the admin page or stored in this database.
|
||||
*/
|
||||
export function classifyGlitchtipFailure(
|
||||
result: GlitchtipDeliveryResult,
|
||||
): string {
|
||||
if (result.details === "invalid_dsn") return "invalid_dsn";
|
||||
if (typeof result.status === "number") return `http_${result.status}`;
|
||||
return "network";
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the outcome of one send. Best-effort by design — a report that could
|
||||
* not be filed must not become a second error, and the singleton row may not
|
||||
* exist yet on a host that has never opened the admin page.
|
||||
*/
|
||||
export async function recordGlitchtipDelivery(
|
||||
result: GlitchtipDeliveryResult,
|
||||
): Promise<void> {
|
||||
const now = new Date();
|
||||
try {
|
||||
await prisma.appSettings.update({
|
||||
where: { id: "singleton" },
|
||||
data: result.ok
|
||||
? { glitchtipLastOkAt: now }
|
||||
: {
|
||||
glitchtipLastFailureAt: now,
|
||||
glitchtipLastFailureReason: classifyGlitchtipFailure(result),
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
// Nothing to do and nothing to report to: the caller is already on an
|
||||
// error path, and throwing here would replace the original failure with
|
||||
// this one.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The judgement, resolved server-side.
|
||||
*
|
||||
* The client gets the verdict, not the timestamps: whether a report is inside
|
||||
* the window is a question about a clock, and the process that holds the
|
||||
* ledger is the one that should answer it. A failure older than the last
|
||||
* success is not surfaced at all — it has been superseded by a report that
|
||||
* got through, and showing it would be reporting a fixed problem.
|
||||
*/
|
||||
export interface GlitchtipDeliverySummary {
|
||||
/** A report left this host successfully inside the window. */
|
||||
reportsDelivering: boolean;
|
||||
/** A report has left this host successfully at some point. */
|
||||
everDelivered: boolean;
|
||||
/** Why the last attempt failed, when that failure is the newest fact. */
|
||||
lastFailureReason: string | null;
|
||||
/** How long a success stands for, so the card can say it. */
|
||||
windowHours: number;
|
||||
}
|
||||
|
||||
export function summariseGlitchtipDelivery(
|
||||
state: GlitchtipDeliveryState,
|
||||
now: Date,
|
||||
): GlitchtipDeliverySummary {
|
||||
const windowMs = GLITCHTIP_DELIVERY_WINDOW_HOURS * 3_600_000;
|
||||
const okAge =
|
||||
state.lastOkAt === null ? null : now.getTime() - state.lastOkAt.getTime();
|
||||
const failureIsNewest =
|
||||
state.lastFailureAt !== null &&
|
||||
(state.lastOkAt === null ||
|
||||
state.lastFailureAt.getTime() > state.lastOkAt.getTime());
|
||||
|
||||
return {
|
||||
reportsDelivering: okAge !== null && okAge >= 0 && okAge <= windowMs,
|
||||
everDelivered: state.lastOkAt !== null,
|
||||
lastFailureReason: failureIsNewest ? state.lastFailureReason : null,
|
||||
windowHours: GLITCHTIP_DELIVERY_WINDOW_HOURS,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getGlitchtipDeliveryState(): Promise<GlitchtipDeliveryState> {
|
||||
try {
|
||||
const settings = await prisma.appSettings.findUnique({
|
||||
where: { id: "singleton" },
|
||||
select: {
|
||||
glitchtipLastOkAt: true,
|
||||
glitchtipLastFailureAt: true,
|
||||
glitchtipLastFailureReason: true,
|
||||
},
|
||||
});
|
||||
return {
|
||||
lastOkAt: settings?.glitchtipLastOkAt ?? null,
|
||||
lastFailureAt: settings?.glitchtipLastFailureAt ?? null,
|
||||
lastFailureReason: settings?.glitchtipLastFailureReason ?? null,
|
||||
};
|
||||
} catch {
|
||||
return { lastOkAt: null, lastFailureAt: null, lastFailureReason: null };
|
||||
}
|
||||
}
|
||||
@@ -214,13 +214,34 @@ async function tryStoreHeader(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Record what became of one send, in the one place every caller passes
|
||||
* through. Dynamically imported so this module stays free of a database
|
||||
* dependency at import time, and awaited so a test can observe it; the
|
||||
* recorder itself never throws.
|
||||
*/
|
||||
async function record(result: GlitchtipDeliveryResult): Promise<void> {
|
||||
try {
|
||||
const { recordGlitchtipDelivery } =
|
||||
await import("@/lib/monitoring/glitchtip-delivery");
|
||||
await recordGlitchtipDelivery(result);
|
||||
} catch {
|
||||
// The report is the point; its bookkeeping is not worth an exception.
|
||||
}
|
||||
}
|
||||
|
||||
export async function sendGlitchtipEvent(params: {
|
||||
dsn: string;
|
||||
input: GlitchtipEventPayloadInput;
|
||||
}): Promise<GlitchtipDeliveryResult> {
|
||||
const config = parseGlitchtipDsn(params.dsn);
|
||||
if (!config) {
|
||||
return { ok: false, details: "invalid_dsn" };
|
||||
const invalid: GlitchtipDeliveryResult = {
|
||||
ok: false,
|
||||
details: "invalid_dsn",
|
||||
};
|
||||
await record(invalid);
|
||||
return invalid;
|
||||
}
|
||||
|
||||
const eventId = createEventId();
|
||||
@@ -228,15 +249,24 @@ export async function sendGlitchtipEvent(params: {
|
||||
const envelopeBody = buildEnvelopeBody(config, eventId, payload);
|
||||
|
||||
const envelopeResult = await tryEnvelope(config, envelopeBody);
|
||||
if (envelopeResult.ok) return envelopeResult;
|
||||
if (envelopeResult.ok) {
|
||||
await record(envelopeResult);
|
||||
return envelopeResult;
|
||||
}
|
||||
|
||||
const storeQueryResult = await tryStoreQuery(config, payload);
|
||||
if (storeQueryResult.ok) return storeQueryResult;
|
||||
if (storeQueryResult.ok) {
|
||||
await record(storeQueryResult);
|
||||
return storeQueryResult;
|
||||
}
|
||||
|
||||
const storeHeaderResult = await tryStoreHeader(config, payload);
|
||||
if (storeHeaderResult.ok) return storeHeaderResult;
|
||||
if (storeHeaderResult.ok) {
|
||||
await record(storeHeaderResult);
|
||||
return storeHeaderResult;
|
||||
}
|
||||
|
||||
return {
|
||||
const failure: GlitchtipDeliveryResult = {
|
||||
ok: false,
|
||||
method: storeHeaderResult.method,
|
||||
status:
|
||||
@@ -248,4 +278,6 @@ export async function sendGlitchtipEvent(params: {
|
||||
storeQueryResult.details ??
|
||||
envelopeResult.details,
|
||||
};
|
||||
await record(failure);
|
||||
return failure;
|
||||
}
|
||||
@@ -10,6 +10,7 @@
|
||||
* keeps owning the HTTP contract, the component keeps owning the UI.
|
||||
*/
|
||||
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
|
||||
import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness";
|
||||
|
||||
export interface BackupRow {
|
||||
id: string;
|
||||
@@ -24,6 +25,41 @@ export interface BackupRow {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** One account's newest object in the operator's off-host bucket. */
|
||||
export interface OffhostAccountRow {
|
||||
userId: string;
|
||||
username: string;
|
||||
/**
|
||||
* ISO instant a run last walked this account, or null when none has. Null
|
||||
* is what the whole cohort reads until the first run after the upgrade that
|
||||
* adds the ledger, and the card says so rather than claiming `never`.
|
||||
*/
|
||||
lastAttemptAt: string | null;
|
||||
/** ISO instant the newest object landed, or null when there is none. */
|
||||
lastSuccessAt: string | null;
|
||||
/** Size of that object in bytes, or null when there is none. */
|
||||
sizeBytes: number | null;
|
||||
/** Whole hours since it landed, or null when there is none. */
|
||||
ageHours: number | null;
|
||||
freshness: OffhostBackupFreshness;
|
||||
}
|
||||
|
||||
/**
|
||||
* The off-host picture, per account.
|
||||
*
|
||||
* Computed from the ledger the nightly worker writes, never from the bucket:
|
||||
* the page must be able to answer "does this account have a recent copy
|
||||
* off-host" on a host whose worker holds no listing grant on the bucket.
|
||||
*/
|
||||
export interface OffhostBackupOverview {
|
||||
/** Whether this host has the variables the nightly job needs. */
|
||||
configured: boolean;
|
||||
/** Hours between two scheduled runs — the number behind the verdicts. */
|
||||
periodHours: number;
|
||||
/** One row per account. Empty when off-host backup is not configured. */
|
||||
rows: OffhostAccountRow[];
|
||||
}
|
||||
|
||||
export interface BackupsList {
|
||||
rows: BackupRow[];
|
||||
/**
|
||||
@@ -32,6 +68,12 @@ export interface BackupsList {
|
||||
* made six weeks ago and one made on Sunday look identical in a table.
|
||||
*/
|
||||
schedule: BackupScheduleStatus;
|
||||
/**
|
||||
* The off-host leg. A weekly row in this table says the copy that lives in
|
||||
* this database is current; it says nothing about whether anything reached
|
||||
* the operator's bucket, and those two fail independently.
|
||||
*/
|
||||
offhost: OffhostBackupOverview;
|
||||
/**
|
||||
* Soft retention hint — the worker is configured for weekly backups
|
||||
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and
|
||||
|
||||
Reference in new issue
Block a user