feat: surface a stalled or failed backup schedule

The weekly pass can stop and leave no trace where anyone looks. The
handler's own error never runs when the job is killed rather than thrown;
`recordError()` increments an unnamed counter; and the admin status page's
failing-queue panel looks back 72 hours, which for a weekly queue means a
permanent failure is on screen three days in seven. The backups page
itself showed an ordinary row with an ordinary timestamp, because a copy
made six weeks ago reads exactly like Sunday's until somebody subtracts
the dates.

The listing now carries two facts that fail at different times: the age of
the newest scheduled copy, which never leaves any retention window, and
the terminal state of the last scheduled run, which is the only place the
reason is written down. `readLastQueueRun` takes the newest run rather
than a window, so a weekly queue's outcome stays readable until the next
one replaces it, and it unwraps the value pg-boss stores a thrown error
under, so the page says "job timed out" instead of raw JSON. The
failing-queue reader gets the same unwrap.

Manual uploads are excluded from the cadence judgement by the shape of the
input, so an ad-hoc upload cannot make a dead cron look alive, and a fresh
instance is not called overdue for having no copy yet.

The listing also stops reading every blob into the web process to measure
it; `octet_length` answers that in the database, which on an instance with
large records was its own way of running out of memory.
This commit is contained in:
Marc-André Bombeck committed 2026-09-03 02:22:37 +02:00
1 parent d8196d5208
commit 154c93b5e1
16 files changed
+591 -16

No files matched your search

@@ -3,11 +3,27 @@ import { renderToStaticMarkup } from "react-dom/server";
import de from "../../../../messages/de.json";
import en from "../../../../messages/en.json";
const healthySchedule = {
lastSuccessAt: "2026-09-01T03:00:00.000Z",
lastSuccessAgeDays: 2,
staleAfterDays: 10,
stale: false,
lastRun: {
at: "2026-09-01T03:00:12.000Z",
state: "completed" as const,
error: null,
},
lastRunFailed: false,
};
const mocks = vi.hoisted(() => ({
mutationOptions: [] as Array<{
onSuccess?: (data: { summary: Record<string, number> }) => void;
}>,
toastSuccess: vi.fn(),
// Mutable so one file can render both a healthy schedule and a broken one;
// the query mock reads it on every render.
queryData: { rows: [], retentionDays: 30 } as Record<string, unknown>,
}));
/**
@@ -34,7 +50,7 @@ vi.mock("sonner", () => ({
vi.mock("@tanstack/react-query", () => ({
useQuery: () => ({
data: { rows: [], retentionDays: 30 },
data: mocks.queryData,
isLoading: false,
isError: false,
}),
@@ -69,6 +85,11 @@ import { I18nProvider } from "@/lib/i18n/context";
import { BackupsSection } from "../backups-section";
function render(locale: "en" | "de" = "en") {
mocks.queryData = {
rows: [],
retentionDays: 30,
schedule: healthySchedule,
};
return renderToStaticMarkup(
<I18nProvider initialLocale={locale}>
<BackupsSection />
@@ -129,6 +150,68 @@ describe("BackupsSection — empty state", () => {
});
});
/**
* The gap that let a weekly backup stop for a month and a half without anyone
* noticing: the page rendered whatever rows it had, and a copy from six weeks
* ago is indistinguishable from Sunday's until somebody subtracts the dates.
*/
describe("BackupsSection — schedule health", () => {
function withSchedule(schedule: Record<string, unknown>) {
mocks.queryData = { rows: [], retentionDays: 30, schedule };
return renderToStaticMarkup(
<I18nProvider initialLocale="en">
<BackupsSection />
</I18nProvider>,
);
}
it("stays quiet while the schedule is healthy", () => {
const html = render();
expect(html).not.toContain('data-slot="backup-schedule-notice"');
});
it("says so when the newest scheduled copy has gone stale", () => {
const html = withSchedule({
...healthySchedule,
lastSuccessAt: "2026-07-19T03:00:00.000Z",
lastSuccessAgeDays: 46,
stale: true,
lastRun: null,
lastRunFailed: false,
});
expect(html).toContain('data-slot="backup-schedule-notice"');
expect(html).toContain("The weekly backup has stopped");
expect(html).toContain("46");
});
it("names the reason when the last scheduled run failed", () => {
const html = withSchedule({
...healthySchedule,
lastRun: {
at: "2026-08-30T03:46:00.000Z",
state: "failed",
error: "job timed out",
},
lastRunFailed: true,
});
expect(html).toContain("The last scheduled backup run failed");
expect(html).toContain("job timed out");
});
it("survives a response that predates the schedule field", () => {
mocks.queryData = { rows: [], retentionDays: 30 };
expect(() =>
renderToStaticMarkup(
<I18nProvider initialLocale="en">
<BackupsSection />
</I18nProvider>,
),
).not.toThrow();
});
});
describe("BackupsSection — upload outcome", () => {
it("includes health profile fact revisions in the restored record total", () => {
mocks.toastSuccess.mockClear();
+70
View File
@@ -48,6 +48,7 @@ import { ListRow } from "@/components/ui/list-row";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import { queryKeys } from "@/lib/query-keys";
import type { BackupRow, BackupsList } from "@/types/backups";
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
import type {
MissingBackupSection,
RestoreSkipSummary,
@@ -560,6 +561,71 @@ export function RestoreSkipReport({
);
}
/**
* Says out loud whether the SCHEDULE is still alive.
*
* The weekly pass on the maintainer's own instance stopped producing copies
* and nothing surfaced it for a month and a half: the table listed the rows it
* had, each with an ordinary timestamp, and a copy from six weeks ago reads
* exactly like one made on Sunday until somebody does the subtraction. Two
* independent facts are shown, because they fail at different times — the age
* of the newest scheduled copy never ages out, while the run's own outcome
* carries the reason ("job timed out") and survives only as long as pg-boss
* keeps the row.
*
* Not dismissible: it reflects live state, so the only way to clear it is to
* make a backup succeed.
*/
function ScheduleHealthNotice({
schedule,
}: {
schedule: BackupScheduleStatus;
}) {
const { t } = useTranslations();
const fmt = useFormatters();
if (!schedule.stale && !schedule.lastRunFailed) return null;
return (
<div
role="alert"
data-slot="backup-schedule-notice"
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start gap-2">
<AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" aria-hidden="true" />
<div>
<p className="font-medium">
{schedule.stale
? t("admin.section.backups.scheduleStaleTitle")
: t("admin.section.backups.scheduleFailedTitle")}
</p>
{schedule.stale ? (
<p className="text-xs">
{t("admin.section.backups.scheduleStaleDescription", {
days: schedule.lastSuccessAgeDays ?? 0,
threshold: schedule.staleAfterDays,
})}
</p>
) : null}
{schedule.lastRunFailed && schedule.lastRun ? (
<p className="text-xs">
{t("admin.section.backups.scheduleFailedDescription", {
when: fmt.dateTime(schedule.lastRun.at),
reason:
schedule.lastRun.error ||
t("admin.section.backups.scheduleFailedNoReason"),
})}
</p>
) : null}
<p className="text-muted-foreground mt-2 text-xs">
{t("admin.section.backups.scheduleRemedy")}
</p>
</div>
</div>
</div>
);
}
export function BackupsSection() {
const { t } = useTranslations();
const fmt = useFormatters();
@@ -833,6 +899,10 @@ export function BackupsSection() {
}
/>
{data?.schedule ? (
<ScheduleHealthNotice schedule={data.schedule} />
) : null}
{/* Upload card — separate from the table so admins can ingest a
backup file independently of any existing rows. The visible
button proxies a hidden file input so the layout stays clean