mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 07:42:23 +00:00
feat: surface a stalled or failed backup schedule
The weekly pass can stop and leave no trace where anyone looks. The handler's own error never runs when the job is killed rather than thrown; `recordError()` increments an unnamed counter; and the admin status page's failing-queue panel looks back 72 hours, which for a weekly queue means a permanent failure is on screen three days in seven. The backups page itself showed an ordinary row with an ordinary timestamp, because a copy made six weeks ago reads exactly like Sunday's until somebody subtracts the dates. The listing now carries two facts that fail at different times: the age of the newest scheduled copy, which never leaves any retention window, and the terminal state of the last scheduled run, which is the only place the reason is written down. `readLastQueueRun` takes the newest run rather than a window, so a weekly queue's outcome stays readable until the next one replaces it, and it unwraps the value pg-boss stores a thrown error under, so the page says "job timed out" instead of raw JSON. The failing-queue reader gets the same unwrap. Manual uploads are excluded from the cadence judgement by the shape of the input, so an ad-hoc upload cannot make a dead cron look alive, and a fresh instance is not called overdue for having no copy yet. The listing also stops reading every blob into the web process to measure it; `octet_length` answers that in the database, which on an instance with large records was its own way of running out of memory.
This commit is contained in:
1 parent
d8196d5208
commit
154c93b5e1
16 files changed
+591
-16
No files matched your search
@@ -3,11 +3,27 @@ import { renderToStaticMarkup } from "react-dom/server";
|
||||
import de from "../../../../messages/de.json";
|
||||
import en from "../../../../messages/en.json";
|
||||
|
||||
const healthySchedule = {
|
||||
lastSuccessAt: "2026-09-01T03:00:00.000Z",
|
||||
lastSuccessAgeDays: 2,
|
||||
staleAfterDays: 10,
|
||||
stale: false,
|
||||
lastRun: {
|
||||
at: "2026-09-01T03:00:12.000Z",
|
||||
state: "completed" as const,
|
||||
error: null,
|
||||
},
|
||||
lastRunFailed: false,
|
||||
};
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
mutationOptions: [] as Array<{
|
||||
onSuccess?: (data: { summary: Record<string, number> }) => void;
|
||||
}>,
|
||||
toastSuccess: vi.fn(),
|
||||
// Mutable so one file can render both a healthy schedule and a broken one;
|
||||
// the query mock reads it on every render.
|
||||
queryData: { rows: [], retentionDays: 30 } as Record<string, unknown>,
|
||||
}));
|
||||
|
||||
/**
|
||||
@@ -34,7 +50,7 @@ vi.mock("sonner", () => ({
|
||||
|
||||
vi.mock("@tanstack/react-query", () => ({
|
||||
useQuery: () => ({
|
||||
data: { rows: [], retentionDays: 30 },
|
||||
data: mocks.queryData,
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
}),
|
||||
@@ -69,6 +85,11 @@ import { I18nProvider } from "@/lib/i18n/context";
|
||||
import { BackupsSection } from "../backups-section";
|
||||
|
||||
function render(locale: "en" | "de" = "en") {
|
||||
mocks.queryData = {
|
||||
rows: [],
|
||||
retentionDays: 30,
|
||||
schedule: healthySchedule,
|
||||
};
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale={locale}>
|
||||
<BackupsSection />
|
||||
@@ -129,6 +150,68 @@ describe("BackupsSection — empty state", () => {
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The gap that let a weekly backup stop for a month and a half without anyone
|
||||
* noticing: the page rendered whatever rows it had, and a copy from six weeks
|
||||
* ago is indistinguishable from Sunday's until somebody subtracts the dates.
|
||||
*/
|
||||
describe("BackupsSection — schedule health", () => {
|
||||
function withSchedule(schedule: Record<string, unknown>) {
|
||||
mocks.queryData = { rows: [], retentionDays: 30, schedule };
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<BackupsSection />
|
||||
</I18nProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
it("stays quiet while the schedule is healthy", () => {
|
||||
const html = render();
|
||||
expect(html).not.toContain('data-slot="backup-schedule-notice"');
|
||||
});
|
||||
|
||||
it("says so when the newest scheduled copy has gone stale", () => {
|
||||
const html = withSchedule({
|
||||
...healthySchedule,
|
||||
lastSuccessAt: "2026-07-19T03:00:00.000Z",
|
||||
lastSuccessAgeDays: 46,
|
||||
stale: true,
|
||||
lastRun: null,
|
||||
lastRunFailed: false,
|
||||
});
|
||||
|
||||
expect(html).toContain('data-slot="backup-schedule-notice"');
|
||||
expect(html).toContain("The weekly backup has stopped");
|
||||
expect(html).toContain("46");
|
||||
});
|
||||
|
||||
it("names the reason when the last scheduled run failed", () => {
|
||||
const html = withSchedule({
|
||||
...healthySchedule,
|
||||
lastRun: {
|
||||
at: "2026-08-30T03:46:00.000Z",
|
||||
state: "failed",
|
||||
error: "job timed out",
|
||||
},
|
||||
lastRunFailed: true,
|
||||
});
|
||||
|
||||
expect(html).toContain("The last scheduled backup run failed");
|
||||
expect(html).toContain("job timed out");
|
||||
});
|
||||
|
||||
it("survives a response that predates the schedule field", () => {
|
||||
mocks.queryData = { rows: [], retentionDays: 30 };
|
||||
expect(() =>
|
||||
renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<BackupsSection />
|
||||
</I18nProvider>,
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("BackupsSection — upload outcome", () => {
|
||||
it("includes health profile fact revisions in the restored record total", () => {
|
||||
mocks.toastSuccess.mockClear();
|
||||
|
||||
@@ -48,6 +48,7 @@ import { ListRow } from "@/components/ui/list-row";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
|
||||
import type {
|
||||
MissingBackupSection,
|
||||
RestoreSkipSummary,
|
||||
@@ -560,6 +561,71 @@ export function RestoreSkipReport({
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Says out loud whether the SCHEDULE is still alive.
|
||||
*
|
||||
* The weekly pass on the maintainer's own instance stopped producing copies
|
||||
* and nothing surfaced it for a month and a half: the table listed the rows it
|
||||
* had, each with an ordinary timestamp, and a copy from six weeks ago reads
|
||||
* exactly like one made on Sunday until somebody does the subtraction. Two
|
||||
* independent facts are shown, because they fail at different times — the age
|
||||
* of the newest scheduled copy never ages out, while the run's own outcome
|
||||
* carries the reason ("job timed out") and survives only as long as pg-boss
|
||||
* keeps the row.
|
||||
*
|
||||
* Not dismissible: it reflects live state, so the only way to clear it is to
|
||||
* make a backup succeed.
|
||||
*/
|
||||
function ScheduleHealthNotice({
|
||||
schedule,
|
||||
}: {
|
||||
schedule: BackupScheduleStatus;
|
||||
}) {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
if (!schedule.stale && !schedule.lastRunFailed) return null;
|
||||
|
||||
return (
|
||||
<div
|
||||
role="alert"
|
||||
data-slot="backup-schedule-notice"
|
||||
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
|
||||
>
|
||||
<div className="flex items-start gap-2">
|
||||
<AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" aria-hidden="true" />
|
||||
<div>
|
||||
<p className="font-medium">
|
||||
{schedule.stale
|
||||
? t("admin.section.backups.scheduleStaleTitle")
|
||||
: t("admin.section.backups.scheduleFailedTitle")}
|
||||
</p>
|
||||
{schedule.stale ? (
|
||||
<p className="text-xs">
|
||||
{t("admin.section.backups.scheduleStaleDescription", {
|
||||
days: schedule.lastSuccessAgeDays ?? 0,
|
||||
threshold: schedule.staleAfterDays,
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
{schedule.lastRunFailed && schedule.lastRun ? (
|
||||
<p className="text-xs">
|
||||
{t("admin.section.backups.scheduleFailedDescription", {
|
||||
when: fmt.dateTime(schedule.lastRun.at),
|
||||
reason:
|
||||
schedule.lastRun.error ||
|
||||
t("admin.section.backups.scheduleFailedNoReason"),
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
<p className="text-muted-foreground mt-2 text-xs">
|
||||
{t("admin.section.backups.scheduleRemedy")}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function BackupsSection() {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
@@ -833,6 +899,10 @@ export function BackupsSection() {
|
||||
}
|
||||
/>
|
||||
|
||||
{data?.schedule ? (
|
||||
<ScheduleHealthNotice schedule={data.schedule} />
|
||||
) : null}
|
||||
|
||||
{/* Upload card — separate from the table so admins can ingest a
|
||||
backup file independently of any existing rows. The visible
|
||||
button proxies a hidden file input so the layout stays clean
|
||||
|
||||
Reference in new issue
Block a user