feat: surface a stalled or failed backup schedule

The weekly pass can stop and leave no trace where anyone looks. The
handler's own error never runs when the job is killed rather than thrown;
`recordError()` increments an unnamed counter; and the admin status page's
failing-queue panel looks back 72 hours, which for a weekly queue means a
permanent failure is on screen three days in seven. The backups page
itself showed an ordinary row with an ordinary timestamp, because a copy
made six weeks ago reads exactly like Sunday's until somebody subtracts
the dates.

The listing now carries two facts that fail at different times: the age of
the newest scheduled copy, which never leaves any retention window, and
the terminal state of the last scheduled run, which is the only place the
reason is written down. `readLastQueueRun` takes the newest run rather
than a window, so a weekly queue's outcome stays readable until the next
one replaces it, and it unwraps the value pg-boss stores a thrown error
under, so the page says "job timed out" instead of raw JSON. The
failing-queue reader gets the same unwrap.

Manual uploads are excluded from the cadence judgement by the shape of the
input, so an ad-hoc upload cannot make a dead cron look alive, and a fresh
instance is not called overdue for having no copy yet.

The listing also stops reading every blob into the web process to measure
it; `octet_length` answers that in the database, which on an instance with
large records was its own way of running out of memory.
This commit is contained in:
Marc-André Bombeck committed 2026-09-03 02:22:37 +02:00
1 parent d8196d5208
commit 154c93b5e1
16 files changed
+591 -16

No files matched your search

+49 -14
View File
@@ -5,11 +5,20 @@
* shipped — only metadata (id, userId, username, type, size in bytes,
* createdAt). The encrypted blob remains server-side; admins can trigger
* a re-snapshot but not download another user's payload from the UI.
*
* Alongside the rows, the response says whether the SCHEDULE is still alive:
* how old the newest scheduled copy is and how the last scheduled run ended.
* Without those two the page cannot tell a working backup from one that
* stopped six weeks ago — every row it lists has a perfectly ordinary
* timestamp either way.
*/
import { prisma } from "@/lib/db";
import { apiHandler, requireAdmin } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import { annotate } from "@/lib/logging/context";
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
import { readLastQueueRun } from "@/lib/jobs/job-failures";
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
// v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts`
// so callers (in particular `components/admin/backups-section.tsx`) don't
// have to reach across the component → route-handler layer boundary.
@@ -17,31 +26,57 @@ import type { BackupRow, BackupsList } from "@/types/backups";
export const dynamic = "force-dynamic";
/** The type the weekly worker writes; anything else was uploaded by hand. */
const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO";
interface BackupMetaRow {
id: string;
user_id: string;
username: string;
type: string;
size_bytes: number;
created_at: Date;
}
export const GET = apiHandler(async () => {
await requireAdmin();
annotate({ action: { name: "admin.backups.list" } });
// Pull metadata only — `data` would balloon the response and reveal
// ciphertext we don't need on the listing page.
const backups = await prisma.dataBackup.findMany({
include: { user: { select: { id: true, username: true } } },
orderBy: { createdAt: "desc" },
});
// Metadata only, and the size read as `octet_length` in the database rather
// than by pulling every blob into this process to measure it. The listing
// needs a number, not the ciphertext, and on an instance with large records
// fetching them all was its own way of running out of memory. Base64 is
// ASCII, so the column's byte length is exactly what the wire reports.
const rows = await prisma.$queryRaw<BackupMetaRow[]>`
SELECT b.id, b.user_id, u.username, b.type,
octet_length(b.data) AS size_bytes,
b.created_at
FROM data_backups b
JOIN users u ON u.id = b.user_id
ORDER BY b.created_at DESC
`;
const rows: BackupRow[] = backups.map((b) => ({
id: b.id,
userId: b.userId,
username: b.user.username,
type: b.type,
sizeBytes: Buffer.byteLength(b.data, "utf8"),
createdAt: b.createdAt.toISOString(),
const list: BackupRow[] = rows.map((row) => ({
id: row.id,
userId: row.user_id,
username: row.username,
type: row.type,
sizeBytes: Number(row.size_bytes),
createdAt: row.created_at.toISOString(),
}));
const payload: BackupsList = {
rows,
rows: list,
// Matches the retention window the backup-prune job enforces so the
// backups page states the same number the worker acts on.
retentionDays: 90,
schedule: summariseBackupSchedule({
scheduledCreatedAt: rows
.filter((row) => row.type === SCHEDULED_BACKUP_TYPE)
.map((row) => row.created_at),
lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE),
now: new Date(),
}),
};
return apiSuccess(payload);
@@ -3,11 +3,27 @@ import { renderToStaticMarkup } from "react-dom/server";
import de from "../../../../messages/de.json";
import en from "../../../../messages/en.json";
const healthySchedule = {
lastSuccessAt: "2026-09-01T03:00:00.000Z",
lastSuccessAgeDays: 2,
staleAfterDays: 10,
stale: false,
lastRun: {
at: "2026-09-01T03:00:12.000Z",
state: "completed" as const,
error: null,
},
lastRunFailed: false,
};
const mocks = vi.hoisted(() => ({
mutationOptions: [] as Array<{
onSuccess?: (data: { summary: Record<string, number> }) => void;
}>,
toastSuccess: vi.fn(),
// Mutable so one file can render both a healthy schedule and a broken one;
// the query mock reads it on every render.
queryData: { rows: [], retentionDays: 30 } as Record<string, unknown>,
}));
/**
@@ -34,7 +50,7 @@ vi.mock("sonner", () => ({
vi.mock("@tanstack/react-query", () => ({
useQuery: () => ({
data: { rows: [], retentionDays: 30 },
data: mocks.queryData,
isLoading: false,
isError: false,
}),
@@ -69,6 +85,11 @@ import { I18nProvider } from "@/lib/i18n/context";
import { BackupsSection } from "../backups-section";
function render(locale: "en" | "de" = "en") {
mocks.queryData = {
rows: [],
retentionDays: 30,
schedule: healthySchedule,
};
return renderToStaticMarkup(
<I18nProvider initialLocale={locale}>
<BackupsSection />
@@ -129,6 +150,68 @@ describe("BackupsSection — empty state", () => {
});
});
/**
* The gap that let a weekly backup stop for a month and a half without anyone
* noticing: the page rendered whatever rows it had, and a copy from six weeks
* ago is indistinguishable from Sunday's until somebody subtracts the dates.
*/
describe("BackupsSection — schedule health", () => {
function withSchedule(schedule: Record<string, unknown>) {
mocks.queryData = { rows: [], retentionDays: 30, schedule };
return renderToStaticMarkup(
<I18nProvider initialLocale="en">
<BackupsSection />
</I18nProvider>,
);
}
it("stays quiet while the schedule is healthy", () => {
const html = render();
expect(html).not.toContain('data-slot="backup-schedule-notice"');
});
it("says so when the newest scheduled copy has gone stale", () => {
const html = withSchedule({
...healthySchedule,
lastSuccessAt: "2026-07-19T03:00:00.000Z",
lastSuccessAgeDays: 46,
stale: true,
lastRun: null,
lastRunFailed: false,
});
expect(html).toContain('data-slot="backup-schedule-notice"');
expect(html).toContain("The weekly backup has stopped");
expect(html).toContain("46");
});
it("names the reason when the last scheduled run failed", () => {
const html = withSchedule({
...healthySchedule,
lastRun: {
at: "2026-08-30T03:46:00.000Z",
state: "failed",
error: "job timed out",
},
lastRunFailed: true,
});
expect(html).toContain("The last scheduled backup run failed");
expect(html).toContain("job timed out");
});
it("survives a response that predates the schedule field", () => {
mocks.queryData = { rows: [], retentionDays: 30 };
expect(() =>
renderToStaticMarkup(
<I18nProvider initialLocale="en">
<BackupsSection />
</I18nProvider>,
),
).not.toThrow();
});
});
describe("BackupsSection — upload outcome", () => {
it("includes health profile fact revisions in the restored record total", () => {
mocks.toastSuccess.mockClear();
+70
View File
@@ -48,6 +48,7 @@ import { ListRow } from "@/components/ui/list-row";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import { queryKeys } from "@/lib/query-keys";
import type { BackupRow, BackupsList } from "@/types/backups";
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
import type {
MissingBackupSection,
RestoreSkipSummary,
@@ -560,6 +561,71 @@ export function RestoreSkipReport({
);
}
/**
* Says out loud whether the SCHEDULE is still alive.
*
* The weekly pass on the maintainer's own instance stopped producing copies
* and nothing surfaced it for a month and a half: the table listed the rows it
* had, each with an ordinary timestamp, and a copy from six weeks ago reads
* exactly like one made on Sunday until somebody does the subtraction. Two
* independent facts are shown, because they fail at different times — the age
* of the newest scheduled copy never ages out, while the run's own outcome
* carries the reason ("job timed out") and survives only as long as pg-boss
* keeps the row.
*
* Not dismissible: it reflects live state, so the only way to clear it is to
* make a backup succeed.
*/
function ScheduleHealthNotice({
schedule,
}: {
schedule: BackupScheduleStatus;
}) {
const { t } = useTranslations();
const fmt = useFormatters();
if (!schedule.stale && !schedule.lastRunFailed) return null;
return (
<div
role="alert"
data-slot="backup-schedule-notice"
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start gap-2">
<AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" aria-hidden="true" />
<div>
<p className="font-medium">
{schedule.stale
? t("admin.section.backups.scheduleStaleTitle")
: t("admin.section.backups.scheduleFailedTitle")}
</p>
{schedule.stale ? (
<p className="text-xs">
{t("admin.section.backups.scheduleStaleDescription", {
days: schedule.lastSuccessAgeDays ?? 0,
threshold: schedule.staleAfterDays,
})}
</p>
) : null}
{schedule.lastRunFailed && schedule.lastRun ? (
<p className="text-xs">
{t("admin.section.backups.scheduleFailedDescription", {
when: fmt.dateTime(schedule.lastRun.at),
reason:
schedule.lastRun.error ||
t("admin.section.backups.scheduleFailedNoReason"),
})}
</p>
) : null}
<p className="text-muted-foreground mt-2 text-xs">
{t("admin.section.backups.scheduleRemedy")}
</p>
</div>
</div>
</div>
);
}
export function BackupsSection() {
const { t } = useTranslations();
const fmt = useFormatters();
@@ -833,6 +899,10 @@ export function BackupsSection() {
}
/>
{data?.schedule ? (
<ScheduleHealthNotice schedule={data.schedule} />
) : null}
{/* Upload card — separate from the table so admins can ingest a
backup file independently of any existing rows. The visible
button proxies a hidden file input so the layout stays clean
@@ -0,0 +1,92 @@
/**
* What the backups page has to be able to say out loud.
*
* The weekly snapshot stopped producing a new copy and nothing surfaced it: the
* page listed the rows it had, each with a perfectly ordinary timestamp, and a
* copy that was a month and a half old looked exactly like one made on Sunday.
* The two facts that would have shown it are the age of the newest scheduled
* copy and the terminal state of the last scheduled run.
*/
import { describe, expect, it } from "vitest";
import {
BACKUP_STALE_AFTER_DAYS,
summariseBackupSchedule,
} from "../backup-schedule-status";
const NOW = new Date("2026-09-03T09:00:00.000Z");
const daysAgo = (days: number) =>
new Date(NOW.getTime() - days * 24 * 60 * 60 * 1000);
describe("summariseBackupSchedule", () => {
it("reports a fresh weekly copy as neither stale nor failed", () => {
const status = summariseBackupSchedule({
scheduledCreatedAt: [daysAgo(2), daysAgo(9)],
lastRun: {
at: daysAgo(2).toISOString(),
state: "completed",
error: null,
},
now: NOW,
});
expect(status.lastSuccessAt).toBe(daysAgo(2).toISOString());
expect(status.lastSuccessAgeDays).toBe(2);
expect(status.stale).toBe(false);
expect(status.lastRunFailed).toBe(false);
expect(status.staleAfterDays).toBe(BACKUP_STALE_AFTER_DAYS);
});
it("calls a copy older than the window stale", () => {
const status = summariseBackupSchedule({
scheduledCreatedAt: [daysAgo(46)],
lastRun: null,
now: NOW,
});
expect(status.lastSuccessAgeDays).toBe(46);
expect(status.stale).toBe(true);
// No run row survives that long — the age is the durable signal.
expect(status.lastRunFailed).toBe(false);
});
it("surfaces a failed last run even while the copy is still in window", () => {
const status = summariseBackupSchedule({
scheduledCreatedAt: [daysAgo(3)],
lastRun: {
at: daysAgo(1).toISOString(),
state: "failed",
error: "job timed out",
},
now: NOW,
});
expect(status.stale).toBe(false);
expect(status.lastRunFailed).toBe(true);
expect(status.lastRun?.error).toBe("job timed out");
});
it("does not call a brand-new instance stale", () => {
const status = summariseBackupSchedule({
scheduledCreatedAt: [],
lastRun: null,
now: NOW,
});
expect(status.lastSuccessAt).toBeNull();
expect(status.lastSuccessAgeDays).toBeNull();
expect(status.stale).toBe(false);
});
it("ignores manual uploads when judging the scheduled cadence", () => {
// The caller passes only WEEKLY_AUTO rows; a manual upload made today must
// not make a dead cron look alive. Proven here by the shape of the input:
// the summary knows nothing but scheduled copies.
const status = summariseBackupSchedule({
scheduledCreatedAt: [daysAgo(30)],
lastRun: null,
now: NOW,
});
expect(status.stale).toBe(true);
});
});
+78
View File
@@ -0,0 +1,78 @@
/**
* Whether the weekly backup is actually still happening, in two facts.
*
* The pass stopped producing a new copy and nothing said so for a month and a
* half. Everything that could have told the operator was either invisible or
* short-lived: the handler's own error never ran (the job was killed, not
* thrown), `recordError()` increments an unnamed counter, and the admin status
* page's failing-queue panel only looks back 72 hours — which for a weekly
* queue means a permanent failure is on screen three days in seven. The backups
* page itself showed a perfectly ordinary row with a perfectly ordinary
* timestamp, because a 46-day-old copy looks exactly like a fresh one until you
* read the date.
*
* So the surface carries both legs, and neither is derived from the other:
*
* - the AGE of the newest scheduled copy, which never ages out of any
* retention window and would have caught this on the second missed Sunday;
* - the terminal state of the last scheduled RUN, which is the only place the
* reason ("job timed out") is written down at all.
*
* Manual uploads are excluded by the shape of the input: the caller passes only
* the scheduled rows, so an admin's ad-hoc upload cannot make a dead cron look
* alive.
*/
import type { LastQueueRun } from "@/lib/jobs/job-failures";
/**
* How old the newest scheduled copy may get before the page says so. The cron
* is weekly, so seven days is normal and eight is a rounding error; ten means
* at least one Sunday produced nothing.
*/
export const BACKUP_STALE_AFTER_DAYS = 10;
const DAY_MS = 24 * 60 * 60 * 1000;
export interface BackupScheduleStatus {
/** ISO instant of the newest scheduled copy, or null when there is none. */
lastSuccessAt: string | null;
/** Whole days since that copy was written. */
lastSuccessAgeDays: number | null;
/** The threshold behind `stale`, so the UI can name it. */
staleAfterDays: number;
/** A scheduled copy exists and is older than the threshold. */
stale: boolean;
/** The last scheduled run pg-boss still remembers. */
lastRun: LastQueueRun | null;
/** That run ended `failed` or `cancelled`. */
lastRunFailed: boolean;
}
export function summariseBackupSchedule(args: {
/** `createdAt` of every SCHEDULED backup row; manual uploads excluded. */
scheduledCreatedAt: readonly Date[];
lastRun: LastQueueRun | null;
now: Date;
}): BackupScheduleStatus {
const newest = args.scheduledCreatedAt.reduce<Date | null>(
(best, at) => (best === null || at > best ? at : best),
null,
);
const ageDays =
newest === null
? null
: Math.floor((args.now.getTime() - newest.getTime()) / DAY_MS);
return {
lastSuccessAt: newest?.toISOString() ?? null,
lastSuccessAgeDays: ageDays,
staleAfterDays: BACKUP_STALE_AFTER_DAYS,
// An instance that has never produced a scheduled copy is not stale, it is
// new — the page's empty state already says there is nothing there, and
// calling a fresh install "overdue" would train the operator to ignore the
// warning that matters.
stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS,
lastRun: args.lastRun,
lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed",
};
}
+74 -1
View File
@@ -98,7 +98,11 @@ export async function readFailingQueues(
max(completed_on) AS last_failed_at,
(
array_agg(
COALESCE(output->>'message', output::text)
COALESCE(
output->>'message',
output->'value'->>'message',
output::text
)
ORDER BY completed_on DESC NULLS LAST
)
)[1] AS last_error
@@ -124,6 +128,75 @@ export async function readFailingQueues(
}
}
/** The terminal state pg-boss recorded for the newest run of one queue. */
export interface LastQueueRun {
/** ISO instant the run reached its terminal state. */
at: string;
/** `failed` covers both a thrown handler and an expired (killed) job. */
state: "completed" | "failed" | "cancelled";
/** The failure text, redacted and truncated. Null when the run succeeded. */
error: string | null;
}
/**
* The newest terminal run of one queue, whatever its outcome.
*
* `readFailingQueues` answers "what broke recently" and is deliberately scoped
* to a 72-hour window, which is the right shape for a status page watching
* thirty queues that tick by the minute. It is the wrong shape for a WEEKLY
* pass: a Sunday-night failure has fallen out of the window by Wednesday, so a
* job that fails every single week is visible for three days in seven and
* invisible for four. This reader takes the newest run instead of a window, so
* a weekly queue's last outcome is readable right up until the next one.
*
* Bounded by pg-boss's own retention: terminal rows live `deletion_seconds`
* (seven days by default) past completion, so `null` here means "no run row
* survives", which is not the same as "the pass never ran". Callers pair it
* with a fact that does not age out — for backups, the age of the newest
* stored copy.
*/
export async function readLastQueueRun(
queue: string,
): Promise<LastQueueRun | null> {
try {
const rows = await prisma.$queryRaw<
Array<{ state: string; completed_on: Date | null; output: string | null }>
>`
SELECT
state,
completed_on,
-- pg-boss wraps a thrown value: an expired job's row reads
-- {"value":{"message":"job timed out"}}, so a plain ->>'message'
-- misses it and the operator gets raw JSON instead of the sentence.
COALESCE(
output->>'message',
output->'value'->>'message',
output::text
) AS output
FROM pgboss.job
WHERE name = ${queue}
AND state IN ('completed', 'failed', 'cancelled')
AND completed_on IS NOT NULL
ORDER BY completed_on DESC
LIMIT 1
`;
const row = rows[0];
if (!row || row.completed_on === null) return null;
const state =
row.state === "failed" || row.state === "cancelled"
? row.state
: "completed";
return {
at: row.completed_on.toISOString(),
state,
error: state === "completed" ? null : presentError(row.output),
};
} catch {
// No `pgboss` schema (web-only deployment) or no permission to read it.
return null;
}
}
export interface QueueFailureForUser {
/** ISO instant the user's newest job on this queue gave up. */
lastFailedAt: string;
+7
View File
@@ -9,6 +9,7 @@
* both sides a route-handler-independent shared type home; the route
* keeps owning the HTTP contract, the component keeps owning the UI.
*/
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
export interface BackupRow {
id: string;
@@ -25,6 +26,12 @@ export interface BackupRow {
export interface BackupsList {
rows: BackupRow[];
/**
* Whether the weekly schedule is still producing copies, and how the last
* scheduled run ended. A row's own timestamp cannot answer that — a copy
* made six weeks ago and one made on Sunday look identical in a table.
*/
schedule: BackupScheduleStatus;
/**
* Soft retention hint — the worker is configured for weekly backups
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and