feat(admin): say what a queued restore is waiting for

A restore never runs beside a backup pass or another large job: the
heavy lane puts its delivery back in the queue for a minute at a time.
The job's own row says queued for the whole wait, and the console said
the restore starts "as soon as the background worker picks it up",
which read as a stuck worker while a backup ran.

The status routes now ask the queue. A delivery the lane put back sits
in created with a start_after ahead of now, and the active row of a
heavy queue is what holds the lane. The job view carries
waiting: "backup" | "busy" | null, and the console says it waits for
the running backup, or for another large job, and since when it has
been queued, in all nine locales. A queue that cannot be asked answers
no wait and is reported at warn.

The job view and its reads move to backup-restore-view.ts, which keeps
the worker module under its size baseline.
This commit is contained in:
Marc-André Bombeck committed 2026-10-23 21:43:55 +02:00
1 parent 6a87810dbf
commit 7936ab9a57
28 files changed
+544 -199

No files matched your search

@@ -1,7 +1,7 @@
{
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 90,
"total": 100487,
"count": 89,
"total": 99600,
"entries": {
"app/medications/page-client.tsx": 839,
"components/charts/mood-chart.tsx": 1166,
@@ -67,7 +67,6 @@
"lib/insights/mood-aggregates.ts": 1053,
"lib/insights/status-cache.ts": 812,
"lib/integrations/status.ts": 1445,
"lib/jobs/backup-restore.ts": 887,
"lib/jobs/coach-nudge.ts": 1084,
"lib/jobs/insight-pregenerate.ts": 1259,
"lib/jobs/medication-intake-import.ts": 819,
+4
View File
@@ -39,6 +39,10 @@ const ALLOWED: Record<string, { hits: number; why: string }> = {
hits: 1,
why: "pg-boss start_after is timestamptz",
},
"src/lib/jobs/backup-restore-wait.ts": {
hits: 1,
why: "pg-boss start_after is timestamptz",
},
"src/lib/jobs/reminder/registrar-shared.ts": {
hits: 1,
why: "pg-boss tables are timestamptz",
@@ -7,10 +7,8 @@
*/
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import {
evictAfterRestore,
readBackupRestoreJob,
} from "@/lib/jobs/backup-restore";
import { evictAfterRestore } from "@/lib/jobs/backup-restore";
import { readBackupRestoreJob } from "@/lib/jobs/backup-restore-view";
import { annotate } from "@/lib/logging/context";
export const dynamic = "force-dynamic";
+2 -4
View File
@@ -8,10 +8,8 @@
*/
import { apiHandler, requireAdmin } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import {
evictAfterRestore,
listRecentBackupRestoreJobs,
} from "@/lib/jobs/backup-restore";
import { evictAfterRestore } from "@/lib/jobs/backup-restore";
import { listRecentBackupRestoreJobs } from "@/lib/jobs/backup-restore-view";
import { annotate } from "@/lib/logging/context";
export const dynamic = "force-dynamic";
@@ -26,7 +26,7 @@ import {
RestoreJobStatus,
visibleRestoreJobs,
} from "@/components/admin/backup-restore-status";
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore";
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore-view";
import { primeLocales } from "@/__tests__/helpers/prime-locales";
primeLocales("de");
@@ -54,6 +54,7 @@ function job(overrides: Partial<BackupRestoreJobView>): BackupRestoreJobView {
},
result: null,
failure: null,
waiting: null,
attempts: 1,
createdAt: "2026-09-25T08:00:00.000Z",
startedAt: "2026-09-25T08:00:01.000Z",
@@ -85,9 +86,29 @@ describe("RestoreJobStatus while it runs", () => {
expect(html).not.toContain('data-slot="restore-job-progress"');
});
it("says a queued restore is waiting", () => {
it("says a queued restore is waiting, and since when", () => {
const html = render(job({ status: "queued", phase: null, progress: null }));
expect(html).toContain("waiting to start");
expect(html).toContain("background worker picks it up");
expect(html).toContain("Queued at");
expect(html).not.toContain("data-waiting");
});
it("says it waits for the running backup, not for a stuck worker", () => {
const html = render(
job({ status: "queued", phase: null, progress: null, waiting: "backup" }),
);
expect(html).toContain('data-waiting="backup"');
expect(html).toContain("Waiting for the backup");
expect(html).not.toContain("background worker picks it up");
});
it("says it waits for another large job when that holds the server", () => {
const html = render(
job({ status: "queued", phase: null, progress: null, waiting: "busy" }),
);
expect(html).toContain('data-waiting="busy"');
expect(html).toContain("large job");
});
it("says when it was started again after a restart", () => {
@@ -179,4 +200,12 @@ describe("RestoreJobStatus in another language", () => {
expect(html).toContain("alex wird wiederhergestellt");
expect(html).toContain("Messwerte werden geschrieben");
});
it("says in German that it waits for the backup", () => {
const html = render(
job({ status: "queued", phase: null, progress: null, waiting: "backup" }),
"de",
);
expect(html).toContain("Wartet auf das Backup");
});
});
+35 -8
View File
@@ -28,10 +28,8 @@ import { Button } from "@/components/ui/button";
import { Progress } from "@/components/ui/progress";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import type { RestoreFailureCause } from "@/lib/export/restore-failure-cause";
import type {
BackupRestoreFailureCode,
BackupRestoreJobView,
} from "@/lib/jobs/backup-restore";
import type { BackupRestoreFailureCode } from "@/lib/jobs/backup-restore";
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore-view";
/** Whether a job still holds its account's one restore slot. */
export function isActiveRestore(job: BackupRestoreJobView): boolean {
@@ -147,6 +145,30 @@ function transactionFailureText(
}
}
/**
* What a queued job is waiting for. A restore never runs beside a backup or
* another large job (the heavy lane puts it back for a minute at a time), and
* a wait that said only "until the worker picks it up" read as a stuck
* worker for as long as a backup ran.
*/
function queuedText(
job: BackupRestoreJobView,
t: ReturnType<typeof useTranslations>["t"],
): string {
switch (job.waiting) {
case "backup":
return t("admin.section.backups.restoreJobWaitingBackupNote");
case "busy":
return t("admin.section.backups.restoreJobWaitingBusyNote");
case null:
return t("admin.section.backups.restoreJobQueuedNote");
default: {
const unhandled: never = job.waiting;
return unhandled;
}
}
}
/** What a running job is doing, and the share of it done when that is known. */
function runningLine(
job: BackupRestoreJobView,
@@ -213,16 +235,14 @@ export function RestoreJobStatus({
const line =
job.status === "running"
? runningLine(job, t, fmt)
: {
text: t("admin.section.backups.restoreJobQueuedNote"),
percent: null,
};
: { text: queuedText(job, t), percent: null };
return (
<div
role="status"
aria-live="polite"
data-slot="restore-job-status"
data-status={job.status}
data-waiting={job.waiting ?? undefined}
className="border-border bg-muted/30 space-y-2 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start gap-2">
@@ -248,6 +268,13 @@ export function RestoreJobStatus({
data-slot="restore-job-progress"
/>
) : null}
{job.status === "queued" ? (
<p className="text-muted-foreground text-xs">
{t("admin.section.backups.restoreJobQueuedSince", {
when: fmt.dateTime(job.createdAt),
})}
</p>
) : null}
{job.attempts > 1 ? (
<p className="text-muted-foreground text-xs">
{t("admin.section.backups.restoreJobResumed")}
@@ -10,7 +10,7 @@ import type {
MissingBackupSection,
RestoreSkipSummary,
} from "@/lib/export/restore-skips";
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore";
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore-view";
import { useTranslations } from "@/lib/i18n/context";
import { queryKeys } from "@/lib/query-keys";
import { randomId } from "@/lib/random-id";
+214
View File
@@ -0,0 +1,214 @@
/**
* The read side of a restore job: its row as the status routes answer it.
*
* The row's JSON columns are parsed defensively (a row written by an older
* build reads as nothing rather than as a malformed object), and a queued job
* says what it waits for (`backup-restore-wait.ts`). The worker and the
* admission check live in `backup-restore.ts`.
*/
import { z } from "zod/v4";
import { prisma } from "@/lib/db";
import type {
RestorePhase,
RestoreProgress,
} from "@/lib/export/restore-backup";
import { RESTORE_FAILURE_CAUSES } from "@/lib/export/restore-failure-cause";
import type {
BackupRestoreFailure,
BackupRestoreStatus,
} from "./backup-restore";
import { restoreWaits, type RestoreWait } from "./backup-restore-wait";
const failureSchema = z.object({
code: z.string(),
message: z.string(),
sections: z.array(z.string()).optional(),
keyIds: z.array(z.string()).optional(),
cause: z.enum(RESTORE_FAILURE_CAUSES).optional(),
});
const progressSchema = z.object({
measurementsChecked: z.number().int().nonnegative(),
measurementsTotal: z.number().int().nonnegative().nullable(),
measurementsWritten: z.number().int().nonnegative(),
sectionsDone: z.number().int().nonnegative(),
sectionsTotal: z.number().int().nonnegative(),
});
const resultSchema = z.object({
summary: z.record(z.string(), z.unknown()),
skipped: z.object({
links: z.number().int(),
catalogueKeys: z.array(
z.object({
catalogue: z.string(),
key: z.string(),
links: z.number().int(),
}),
),
}),
cleared: z.record(z.string(), z.number()),
});
export type BackupRestoreResult = z.infer<typeof resultSchema>;
/** What the status routes answer with. Counts and codes, never content. */
export interface BackupRestoreJobView {
id: string;
userId: string;
username: string | null;
backupId: string;
restoreInstanceSettings: boolean;
status: BackupRestoreStatus;
phase: RestorePhase | null;
progress: RestoreProgress | null;
result: BackupRestoreResult | null;
failure: BackupRestoreFailure | null;
/**
* Why a queued job has not started: the heavy lane put it back while a
* backup pass (`backup`) or another large job (`busy`) held it. Null while
* it only waits for the worker, and on every job that is not queued.
*/
waiting: RestoreWait | null;
attempts: number;
createdAt: string;
startedAt: string | null;
completedAt: string | null;
}
const PHASES: readonly RestorePhase[] = [
"validating",
"clearing",
"measurements",
"sections",
"rebuilding",
];
const STATUSES: readonly BackupRestoreStatus[] = [
"queued",
"running",
"succeeded",
"failed",
];
const viewSelect = {
id: true,
userId: true,
backupId: true,
restoreInstanceSettings: true,
status: true,
phase: true,
progress: true,
result: true,
failure: true,
attempts: true,
createdAt: true,
startedAt: true,
completedAt: true,
pgBossJobId: true,
user: { select: { username: true } },
} as const;
type ViewRow = {
id: string;
userId: string;
backupId: string;
restoreInstanceSettings: boolean;
status: string;
phase: string | null;
progress: unknown;
result: unknown;
failure: unknown;
attempts: number;
createdAt: Date;
startedAt: Date | null;
completedAt: Date | null;
pgBossJobId: string | null;
user: { username: string } | null;
};
function toView(
row: ViewRow,
waits: ReadonlyMap<string, RestoreWait>,
): BackupRestoreJobView {
const status = (STATUSES as readonly string[]).includes(row.status)
? (row.status as BackupRestoreStatus)
: "failed";
const phase =
row.phase && (PHASES as readonly string[]).includes(row.phase)
? (row.phase as RestorePhase)
: null;
const progress = progressSchema.safeParse(row.progress);
const result = resultSchema.safeParse(row.result);
const failure = failureSchema.safeParse(row.failure);
return {
id: row.id,
userId: row.userId,
username: row.user?.username ?? null,
backupId: row.backupId,
restoreInstanceSettings: row.restoreInstanceSettings,
status,
phase: status === "running" ? phase : null,
progress: progress.success ? progress.data : null,
result: result.success ? result.data : null,
failure: failure.success
? (failure.data as BackupRestoreFailure)
: status === "failed"
? { code: "unexpected", message: "The restore failed." }
: null,
waiting:
status === "queued" && row.pgBossJobId
? (waits.get(row.pgBossJobId) ?? null)
: null,
attempts: row.attempts,
createdAt: row.createdAt.toISOString(),
startedAt: row.startedAt?.toISOString() ?? null,
completedAt: row.completedAt?.toISOString() ?? null,
};
}
/** What the queued ones among `rows` wait for. */
function waitsOf(rows: readonly ViewRow[]) {
return restoreWaits(
rows.flatMap((row) =>
row.status === "queued" && row.pgBossJobId ? [row.pgBossJobId] : [],
),
);
}
/** One job, for the status route. Null when the id is unknown. */
export async function readBackupRestoreJob(
id: string,
): Promise<BackupRestoreJobView | null> {
const row = await prisma.backupRestoreJob.findUnique({
where: { id },
select: viewSelect,
});
if (!row) return null;
return toView(row, await waitsOf([row]));
}
/**
* The jobs the console shows: every queued or running one, and the finished
* ones of the last day, newest first. What a reload of the page needs to pick
* up a restore it was watching, and to show how the last one ended.
*/
export async function listRecentBackupRestoreJobs(
now: Date = new Date(),
): Promise<BackupRestoreJobView[]> {
const since = new Date(now.getTime() - 24 * 60 * 60 * 1000);
const rows = await prisma.backupRestoreJob.findMany({
where: {
OR: [
{ status: { in: ["queued", "running"] } },
{ createdAt: { gte: since } },
],
},
orderBy: { createdAt: "desc" },
take: 20,
select: viewSelect,
});
const waits = await waitsOf(rows);
return rows.map((row) => toView(row, waits));
}
+79
View File
@@ -0,0 +1,79 @@
/**
* What a queued restore is waiting for, read from the queue itself.
*
* A restore runs in the heavy lane (`heavy-lane.ts`): while a backup pass or
* another large job holds it, the restore goes back to its queue for a minute
* at a time, so it never runs beside one. That is deliberate, and invisible
* from the restore's own row, which says `queued` for the whole wait. The
* console said the restore starts "as soon as the background worker picks it
* up", which read as a stuck worker while a backup ran for half an hour.
*
* The queue knows. A delivery the lane put back sits in `created` with a
* `start_after` ahead of now, and the job holding the lane is the active row
* of a heavy queue. Reading both answers the question without a column that
* would have to be kept in step with the lane.
*/
import { prisma } from "@/lib/db";
import { HEAVY_QUEUES } from "@/lib/jobs/heavy-lane";
import { logCaught } from "@/lib/logging/signal";
import { BACKUP_RESTORE_QUEUE } from "./backup-restore-queue";
/**
* Why a queued restore has not started: a backup pass holds the lane, or
* something else does (an import, a nightly pass, the memory guard).
*/
export type RestoreWait = "backup" | "busy";
/** The queues whose holder the console names as "the backup". */
const BACKUP_PASS_QUEUES: ReadonlySet<string> = new Set([
"data-backup",
"data-backup-offhost",
]);
const OTHER_HEAVY_QUEUES = Object.keys(HEAVY_QUEUES).filter(
(queue) => queue !== BACKUP_RESTORE_QUEUE,
);
/**
* For each of `deliveries` (pg-boss job ids of queued restores) that the lane
* has put back, what holds the lane. A delivery not in the map is waiting
* for the worker to pick it up, nothing more.
*
* Never fails the read it serves: a queue that cannot be asked answers no
* waits, reported at warn, and the console says what it said before.
*/
export async function restoreWaits(
deliveries: readonly string[],
): Promise<Map<string, RestoreWait>> {
const waits = new Map<string, RestoreWait>();
if (deliveries.length === 0) return waits;
try {
const deferred = await prisma.$queryRaw<Array<{ id: string }>>`
SELECT id::text AS id
FROM pgboss.job
WHERE name = ${BACKUP_RESTORE_QUEUE}
AND id = ANY(${[...deliveries]}::uuid[])
AND state = 'created'
AND start_after > now()
`;
if (deferred.length === 0) return waits;
const holders = await prisma.$queryRaw<Array<{ name: string }>>`
SELECT name
FROM pgboss.job
WHERE state = 'active'
AND name = ANY(${OTHER_HEAVY_QUEUES}::text[])
ORDER BY started_on ASC
LIMIT 1
`;
const holder = holders[0]?.name;
const wait: RestoreWait =
holder !== undefined && BACKUP_PASS_QUEUES.has(holder)
? "backup"
: "busy";
for (const row of deferred) waits.set(row.id, wait);
} catch (err) {
logCaught("backup.restore.wait_unread", err);
}
return waits;
}
+4 -167
View File
@@ -51,7 +51,6 @@
import { createHash } from "node:crypto";
import type { Job, PgBoss } from "pg-boss";
import { z } from "zod/v4";
import { invalidateUserData } from "@/lib/cache/invalidate";
import { prisma, toJson } from "@/lib/db";
@@ -79,6 +78,10 @@ import {
} from "@/lib/export/restore-failure-cause";
import { logCaught } from "@/lib/logging/signal";
import { BACKUP_RESTORE_QUEUE } from "./backup-restore-queue";
import type {
BackupRestoreJobView,
BackupRestoreResult,
} from "./backup-restore-view";
/** Stable code of the 409 a second restore of the same account gets. */
export const BACKUP_RESTORE_ACTIVE_CODE = "backup.restore.active";
@@ -140,71 +143,6 @@ export interface BackupRestoreFailure {
cause?: RestoreFailureCause;
}
const failureSchema = z.object({
code: z.string(),
message: z.string(),
sections: z.array(z.string()).optional(),
keyIds: z.array(z.string()).optional(),
cause: z.enum(RESTORE_FAILURE_CAUSES).optional(),
});
const progressSchema = z.object({
measurementsChecked: z.number().int().nonnegative(),
measurementsTotal: z.number().int().nonnegative().nullable(),
measurementsWritten: z.number().int().nonnegative(),
sectionsDone: z.number().int().nonnegative(),
sectionsTotal: z.number().int().nonnegative(),
});
const resultSchema = z.object({
summary: z.record(z.string(), z.unknown()),
skipped: z.object({
links: z.number().int(),
catalogueKeys: z.array(
z.object({
catalogue: z.string(),
key: z.string(),
links: z.number().int(),
}),
),
}),
cleared: z.record(z.string(), z.number()),
});
export type BackupRestoreResult = z.infer<typeof resultSchema>;
/** What the status routes answer with. Counts and codes, never content. */
export interface BackupRestoreJobView {
id: string;
userId: string;
username: string | null;
backupId: string;
restoreInstanceSettings: boolean;
status: BackupRestoreStatus;
phase: RestorePhase | null;
progress: RestoreProgress | null;
result: BackupRestoreResult | null;
failure: BackupRestoreFailure | null;
attempts: number;
createdAt: string;
startedAt: string | null;
completedAt: string | null;
}
const PHASES: readonly RestorePhase[] = [
"validating",
"clearing",
"measurements",
"sections",
"rebuilding",
];
const STATUSES: readonly BackupRestoreStatus[] = [
"queued",
"running",
"succeeded",
"failed",
];
/**
* A failure code as a job fact. Job facts are lower-case words joined by
* underscores (`job-outcome.ts`), and two of the restore's codes are the
@@ -222,107 +160,6 @@ export function backupDigest(identity: string): string {
return createHash("sha256").update(identity).digest("hex");
}
const viewSelect = {
id: true,
userId: true,
backupId: true,
restoreInstanceSettings: true,
status: true,
phase: true,
progress: true,
result: true,
failure: true,
attempts: true,
createdAt: true,
startedAt: true,
completedAt: true,
user: { select: { username: true } },
} as const;
type ViewRow = {
id: string;
userId: string;
backupId: string;
restoreInstanceSettings: boolean;
status: string;
phase: string | null;
progress: unknown;
result: unknown;
failure: unknown;
attempts: number;
createdAt: Date;
startedAt: Date | null;
completedAt: Date | null;
user: { username: string } | null;
};
function toView(row: ViewRow): BackupRestoreJobView {
const status = (STATUSES as readonly string[]).includes(row.status)
? (row.status as BackupRestoreStatus)
: "failed";
const phase =
row.phase && (PHASES as readonly string[]).includes(row.phase)
? (row.phase as RestorePhase)
: null;
const progress = progressSchema.safeParse(row.progress);
const result = resultSchema.safeParse(row.result);
const failure = failureSchema.safeParse(row.failure);
return {
id: row.id,
userId: row.userId,
username: row.user?.username ?? null,
backupId: row.backupId,
restoreInstanceSettings: row.restoreInstanceSettings,
status,
phase: status === "running" ? phase : null,
progress: progress.success ? progress.data : null,
result: result.success ? result.data : null,
failure: failure.success
? (failure.data as BackupRestoreFailure)
: status === "failed"
? { code: "unexpected", message: "The restore failed." }
: null,
attempts: row.attempts,
createdAt: row.createdAt.toISOString(),
startedAt: row.startedAt?.toISOString() ?? null,
completedAt: row.completedAt?.toISOString() ?? null,
};
}
/** One job, for the status route. Null when the id is unknown. */
export async function readBackupRestoreJob(
id: string,
): Promise<BackupRestoreJobView | null> {
const row = await prisma.backupRestoreJob.findUnique({
where: { id },
select: viewSelect,
});
return row ? toView(row) : null;
}
/**
* The jobs the console shows: every queued or running one, and the finished
* ones of the last day, newest first. What a reload of the page needs to pick
* up a restore it was watching, and to show how the last one ended.
*/
export async function listRecentBackupRestoreJobs(
now: Date = new Date(),
): Promise<BackupRestoreJobView[]> {
const since = new Date(now.getTime() - 24 * 60 * 60 * 1000);
const rows = await prisma.backupRestoreJob.findMany({
where: {
OR: [
{ status: { in: ["queued", "running"] } },
{ createdAt: { gte: since } },
],
},
orderBy: { createdAt: "desc" },
take: 20,
select: viewSelect,
});
return rows.map(toView);
}
function failureJson(failure: BackupRestoreFailure) {
return toJson({
code: failure.code,
+6
View File
@@ -132,6 +132,12 @@ const restoreJob = z
})
.nullable(),
failure: restoreFailure.nullable(),
waiting: z
.enum(["backup", "busy"])
.nullable()
.describe(
"Why a `queued` job has not started. A restore never runs beside a backup or another large job: while one holds the server it goes back to the queue for a minute at a time. `backup` when a backup pass holds it, `busy` for anything else (an import, a nightly pass, the memory guard). Null while the job only waits for the worker, and on every job that is not queued.",
),
attempts: z.number().int(),
createdAt: z.string(),
startedAt: z.string().nullable(),