mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-12 14:16:12 +00:00
feat(admin): say what a queued restore is waiting for
A restore never runs beside a backup pass or another large job: the heavy lane puts its delivery back in the queue for a minute at a time. The job's own row says queued for the whole wait, and the console said the restore starts "as soon as the background worker picks it up", which read as a stuck worker while a backup ran. The status routes now ask the queue. A delivery the lane put back sits in created with a start_after ahead of now, and the active row of a heavy queue is what holds the lane. The job view carries waiting: "backup" | "busy" | null, and the console says it waits for the running backup, or for another large job, and since when it has been queued, in all nine locales. A queue that cannot be asked answers no wait and is reported at warn. The job view and its reads move to backup-restore-view.ts, which keeps the worker module under its size baseline.
This commit is contained in:
1 parent
6a87810dbf
commit
7936ab9a57
28 files changed
+544
-199
No files matched your search
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
|
||||
"count": 90,
|
||||
"total": 100487,
|
||||
"count": 89,
|
||||
"total": 99600,
|
||||
"entries": {
|
||||
"app/medications/page-client.tsx": 839,
|
||||
"components/charts/mood-chart.tsx": 1166,
|
||||
@@ -67,7 +67,6 @@
|
||||
"lib/insights/mood-aggregates.ts": 1053,
|
||||
"lib/insights/status-cache.ts": 812,
|
||||
"lib/integrations/status.ts": 1445,
|
||||
"lib/jobs/backup-restore.ts": 887,
|
||||
"lib/jobs/coach-nudge.ts": 1084,
|
||||
"lib/jobs/insight-pregenerate.ts": 1259,
|
||||
"lib/jobs/medication-intake-import.ts": 819,
|
||||
|
||||
@@ -39,6 +39,10 @@ const ALLOWED: Record<string, { hits: number; why: string }> = {
|
||||
hits: 1,
|
||||
why: "pg-boss start_after is timestamptz",
|
||||
},
|
||||
"src/lib/jobs/backup-restore-wait.ts": {
|
||||
hits: 1,
|
||||
why: "pg-boss start_after is timestamptz",
|
||||
},
|
||||
"src/lib/jobs/reminder/registrar-shared.ts": {
|
||||
hits: 1,
|
||||
why: "pg-boss tables are timestamptz",
|
||||
|
||||
@@ -7,10 +7,8 @@
|
||||
*/
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import {
|
||||
evictAfterRestore,
|
||||
readBackupRestoreJob,
|
||||
} from "@/lib/jobs/backup-restore";
|
||||
import { evictAfterRestore } from "@/lib/jobs/backup-restore";
|
||||
import { readBackupRestoreJob } from "@/lib/jobs/backup-restore-view";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -8,10 +8,8 @@
|
||||
*/
|
||||
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import {
|
||||
evictAfterRestore,
|
||||
listRecentBackupRestoreJobs,
|
||||
} from "@/lib/jobs/backup-restore";
|
||||
import { evictAfterRestore } from "@/lib/jobs/backup-restore";
|
||||
import { listRecentBackupRestoreJobs } from "@/lib/jobs/backup-restore-view";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -26,7 +26,7 @@ import {
|
||||
RestoreJobStatus,
|
||||
visibleRestoreJobs,
|
||||
} from "@/components/admin/backup-restore-status";
|
||||
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore";
|
||||
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore-view";
|
||||
import { primeLocales } from "@/__tests__/helpers/prime-locales";
|
||||
|
||||
primeLocales("de");
|
||||
@@ -54,6 +54,7 @@ function job(overrides: Partial<BackupRestoreJobView>): BackupRestoreJobView {
|
||||
},
|
||||
result: null,
|
||||
failure: null,
|
||||
waiting: null,
|
||||
attempts: 1,
|
||||
createdAt: "2026-09-25T08:00:00.000Z",
|
||||
startedAt: "2026-09-25T08:00:01.000Z",
|
||||
@@ -85,9 +86,29 @@ describe("RestoreJobStatus while it runs", () => {
|
||||
expect(html).not.toContain('data-slot="restore-job-progress"');
|
||||
});
|
||||
|
||||
it("says a queued restore is waiting", () => {
|
||||
it("says a queued restore is waiting, and since when", () => {
|
||||
const html = render(job({ status: "queued", phase: null, progress: null }));
|
||||
expect(html).toContain("waiting to start");
|
||||
expect(html).toContain("background worker picks it up");
|
||||
expect(html).toContain("Queued at");
|
||||
expect(html).not.toContain("data-waiting");
|
||||
});
|
||||
|
||||
it("says it waits for the running backup, not for a stuck worker", () => {
|
||||
const html = render(
|
||||
job({ status: "queued", phase: null, progress: null, waiting: "backup" }),
|
||||
);
|
||||
expect(html).toContain('data-waiting="backup"');
|
||||
expect(html).toContain("Waiting for the backup");
|
||||
expect(html).not.toContain("background worker picks it up");
|
||||
});
|
||||
|
||||
it("says it waits for another large job when that holds the server", () => {
|
||||
const html = render(
|
||||
job({ status: "queued", phase: null, progress: null, waiting: "busy" }),
|
||||
);
|
||||
expect(html).toContain('data-waiting="busy"');
|
||||
expect(html).toContain("large job");
|
||||
});
|
||||
|
||||
it("says when it was started again after a restart", () => {
|
||||
@@ -179,4 +200,12 @@ describe("RestoreJobStatus in another language", () => {
|
||||
expect(html).toContain("alex wird wiederhergestellt");
|
||||
expect(html).toContain("Messwerte werden geschrieben");
|
||||
});
|
||||
|
||||
it("says in German that it waits for the backup", () => {
|
||||
const html = render(
|
||||
job({ status: "queued", phase: null, progress: null, waiting: "backup" }),
|
||||
"de",
|
||||
);
|
||||
expect(html).toContain("Wartet auf das Backup");
|
||||
});
|
||||
});
|
||||
@@ -28,10 +28,8 @@ import { Button } from "@/components/ui/button";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import type { RestoreFailureCause } from "@/lib/export/restore-failure-cause";
|
||||
import type {
|
||||
BackupRestoreFailureCode,
|
||||
BackupRestoreJobView,
|
||||
} from "@/lib/jobs/backup-restore";
|
||||
import type { BackupRestoreFailureCode } from "@/lib/jobs/backup-restore";
|
||||
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore-view";
|
||||
|
||||
/** Whether a job still holds its account's one restore slot. */
|
||||
export function isActiveRestore(job: BackupRestoreJobView): boolean {
|
||||
@@ -147,6 +145,30 @@ function transactionFailureText(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What a queued job is waiting for. A restore never runs beside a backup or
|
||||
* another large job (the heavy lane puts it back for a minute at a time), and
|
||||
* a wait that said only "until the worker picks it up" read as a stuck
|
||||
* worker for as long as a backup ran.
|
||||
*/
|
||||
function queuedText(
|
||||
job: BackupRestoreJobView,
|
||||
t: ReturnType<typeof useTranslations>["t"],
|
||||
): string {
|
||||
switch (job.waiting) {
|
||||
case "backup":
|
||||
return t("admin.section.backups.restoreJobWaitingBackupNote");
|
||||
case "busy":
|
||||
return t("admin.section.backups.restoreJobWaitingBusyNote");
|
||||
case null:
|
||||
return t("admin.section.backups.restoreJobQueuedNote");
|
||||
default: {
|
||||
const unhandled: never = job.waiting;
|
||||
return unhandled;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** What a running job is doing, and the share of it done when that is known. */
|
||||
function runningLine(
|
||||
job: BackupRestoreJobView,
|
||||
@@ -213,16 +235,14 @@ export function RestoreJobStatus({
|
||||
const line =
|
||||
job.status === "running"
|
||||
? runningLine(job, t, fmt)
|
||||
: {
|
||||
text: t("admin.section.backups.restoreJobQueuedNote"),
|
||||
percent: null,
|
||||
};
|
||||
: { text: queuedText(job, t), percent: null };
|
||||
return (
|
||||
<div
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
data-slot="restore-job-status"
|
||||
data-status={job.status}
|
||||
data-waiting={job.waiting ?? undefined}
|
||||
className="border-border bg-muted/30 space-y-2 rounded-md border px-3 py-2 text-sm"
|
||||
>
|
||||
<div className="flex items-start gap-2">
|
||||
@@ -248,6 +268,13 @@ export function RestoreJobStatus({
|
||||
data-slot="restore-job-progress"
|
||||
/>
|
||||
) : null}
|
||||
{job.status === "queued" ? (
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{t("admin.section.backups.restoreJobQueuedSince", {
|
||||
when: fmt.dateTime(job.createdAt),
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
{job.attempts > 1 ? (
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{t("admin.section.backups.restoreJobResumed")}
|
||||
|
||||
@@ -10,7 +10,7 @@ import type {
|
||||
MissingBackupSection,
|
||||
RestoreSkipSummary,
|
||||
} from "@/lib/export/restore-skips";
|
||||
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore";
|
||||
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore-view";
|
||||
import { useTranslations } from "@/lib/i18n/context";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import { randomId } from "@/lib/random-id";
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
/**
|
||||
* The read side of a restore job: its row as the status routes answer it.
|
||||
*
|
||||
* The row's JSON columns are parsed defensively (a row written by an older
|
||||
* build reads as nothing rather than as a malformed object), and a queued job
|
||||
* says what it waits for (`backup-restore-wait.ts`). The worker and the
|
||||
* admission check live in `backup-restore.ts`.
|
||||
*/
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import type {
|
||||
RestorePhase,
|
||||
RestoreProgress,
|
||||
} from "@/lib/export/restore-backup";
|
||||
import { RESTORE_FAILURE_CAUSES } from "@/lib/export/restore-failure-cause";
|
||||
|
||||
import type {
|
||||
BackupRestoreFailure,
|
||||
BackupRestoreStatus,
|
||||
} from "./backup-restore";
|
||||
import { restoreWaits, type RestoreWait } from "./backup-restore-wait";
|
||||
|
||||
const failureSchema = z.object({
|
||||
code: z.string(),
|
||||
message: z.string(),
|
||||
sections: z.array(z.string()).optional(),
|
||||
keyIds: z.array(z.string()).optional(),
|
||||
cause: z.enum(RESTORE_FAILURE_CAUSES).optional(),
|
||||
});
|
||||
|
||||
const progressSchema = z.object({
|
||||
measurementsChecked: z.number().int().nonnegative(),
|
||||
measurementsTotal: z.number().int().nonnegative().nullable(),
|
||||
measurementsWritten: z.number().int().nonnegative(),
|
||||
sectionsDone: z.number().int().nonnegative(),
|
||||
sectionsTotal: z.number().int().nonnegative(),
|
||||
});
|
||||
|
||||
const resultSchema = z.object({
|
||||
summary: z.record(z.string(), z.unknown()),
|
||||
skipped: z.object({
|
||||
links: z.number().int(),
|
||||
catalogueKeys: z.array(
|
||||
z.object({
|
||||
catalogue: z.string(),
|
||||
key: z.string(),
|
||||
links: z.number().int(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
cleared: z.record(z.string(), z.number()),
|
||||
});
|
||||
|
||||
export type BackupRestoreResult = z.infer<typeof resultSchema>;
|
||||
|
||||
/** What the status routes answer with. Counts and codes, never content. */
|
||||
export interface BackupRestoreJobView {
|
||||
id: string;
|
||||
userId: string;
|
||||
username: string | null;
|
||||
backupId: string;
|
||||
restoreInstanceSettings: boolean;
|
||||
status: BackupRestoreStatus;
|
||||
phase: RestorePhase | null;
|
||||
progress: RestoreProgress | null;
|
||||
result: BackupRestoreResult | null;
|
||||
failure: BackupRestoreFailure | null;
|
||||
/**
|
||||
* Why a queued job has not started: the heavy lane put it back while a
|
||||
* backup pass (`backup`) or another large job (`busy`) held it. Null while
|
||||
* it only waits for the worker, and on every job that is not queued.
|
||||
*/
|
||||
waiting: RestoreWait | null;
|
||||
attempts: number;
|
||||
createdAt: string;
|
||||
startedAt: string | null;
|
||||
completedAt: string | null;
|
||||
}
|
||||
|
||||
const PHASES: readonly RestorePhase[] = [
|
||||
"validating",
|
||||
"clearing",
|
||||
"measurements",
|
||||
"sections",
|
||||
"rebuilding",
|
||||
];
|
||||
const STATUSES: readonly BackupRestoreStatus[] = [
|
||||
"queued",
|
||||
"running",
|
||||
"succeeded",
|
||||
"failed",
|
||||
];
|
||||
|
||||
const viewSelect = {
|
||||
id: true,
|
||||
userId: true,
|
||||
backupId: true,
|
||||
restoreInstanceSettings: true,
|
||||
status: true,
|
||||
phase: true,
|
||||
progress: true,
|
||||
result: true,
|
||||
failure: true,
|
||||
attempts: true,
|
||||
createdAt: true,
|
||||
startedAt: true,
|
||||
completedAt: true,
|
||||
pgBossJobId: true,
|
||||
user: { select: { username: true } },
|
||||
} as const;
|
||||
|
||||
type ViewRow = {
|
||||
id: string;
|
||||
userId: string;
|
||||
backupId: string;
|
||||
restoreInstanceSettings: boolean;
|
||||
status: string;
|
||||
phase: string | null;
|
||||
progress: unknown;
|
||||
result: unknown;
|
||||
failure: unknown;
|
||||
attempts: number;
|
||||
createdAt: Date;
|
||||
startedAt: Date | null;
|
||||
completedAt: Date | null;
|
||||
pgBossJobId: string | null;
|
||||
user: { username: string } | null;
|
||||
};
|
||||
|
||||
function toView(
|
||||
row: ViewRow,
|
||||
waits: ReadonlyMap<string, RestoreWait>,
|
||||
): BackupRestoreJobView {
|
||||
const status = (STATUSES as readonly string[]).includes(row.status)
|
||||
? (row.status as BackupRestoreStatus)
|
||||
: "failed";
|
||||
const phase =
|
||||
row.phase && (PHASES as readonly string[]).includes(row.phase)
|
||||
? (row.phase as RestorePhase)
|
||||
: null;
|
||||
const progress = progressSchema.safeParse(row.progress);
|
||||
const result = resultSchema.safeParse(row.result);
|
||||
const failure = failureSchema.safeParse(row.failure);
|
||||
return {
|
||||
id: row.id,
|
||||
userId: row.userId,
|
||||
username: row.user?.username ?? null,
|
||||
backupId: row.backupId,
|
||||
restoreInstanceSettings: row.restoreInstanceSettings,
|
||||
status,
|
||||
phase: status === "running" ? phase : null,
|
||||
progress: progress.success ? progress.data : null,
|
||||
result: result.success ? result.data : null,
|
||||
failure: failure.success
|
||||
? (failure.data as BackupRestoreFailure)
|
||||
: status === "failed"
|
||||
? { code: "unexpected", message: "The restore failed." }
|
||||
: null,
|
||||
waiting:
|
||||
status === "queued" && row.pgBossJobId
|
||||
? (waits.get(row.pgBossJobId) ?? null)
|
||||
: null,
|
||||
attempts: row.attempts,
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
startedAt: row.startedAt?.toISOString() ?? null,
|
||||
completedAt: row.completedAt?.toISOString() ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
/** What the queued ones among `rows` wait for. */
|
||||
function waitsOf(rows: readonly ViewRow[]) {
|
||||
return restoreWaits(
|
||||
rows.flatMap((row) =>
|
||||
row.status === "queued" && row.pgBossJobId ? [row.pgBossJobId] : [],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** One job, for the status route. Null when the id is unknown. */
|
||||
export async function readBackupRestoreJob(
|
||||
id: string,
|
||||
): Promise<BackupRestoreJobView | null> {
|
||||
const row = await prisma.backupRestoreJob.findUnique({
|
||||
where: { id },
|
||||
select: viewSelect,
|
||||
});
|
||||
if (!row) return null;
|
||||
return toView(row, await waitsOf([row]));
|
||||
}
|
||||
|
||||
/**
|
||||
* The jobs the console shows: every queued or running one, and the finished
|
||||
* ones of the last day, newest first. What a reload of the page needs to pick
|
||||
* up a restore it was watching, and to show how the last one ended.
|
||||
*/
|
||||
export async function listRecentBackupRestoreJobs(
|
||||
now: Date = new Date(),
|
||||
): Promise<BackupRestoreJobView[]> {
|
||||
const since = new Date(now.getTime() - 24 * 60 * 60 * 1000);
|
||||
const rows = await prisma.backupRestoreJob.findMany({
|
||||
where: {
|
||||
OR: [
|
||||
{ status: { in: ["queued", "running"] } },
|
||||
{ createdAt: { gte: since } },
|
||||
],
|
||||
},
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: 20,
|
||||
select: viewSelect,
|
||||
});
|
||||
const waits = await waitsOf(rows);
|
||||
return rows.map((row) => toView(row, waits));
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* What a queued restore is waiting for, read from the queue itself.
|
||||
*
|
||||
* A restore runs in the heavy lane (`heavy-lane.ts`): while a backup pass or
|
||||
* another large job holds it, the restore goes back to its queue for a minute
|
||||
* at a time, so it never runs beside one. That is deliberate, and invisible
|
||||
* from the restore's own row, which says `queued` for the whole wait. The
|
||||
* console said the restore starts "as soon as the background worker picks it
|
||||
* up", which read as a stuck worker while a backup ran for half an hour.
|
||||
*
|
||||
* The queue knows. A delivery the lane put back sits in `created` with a
|
||||
* `start_after` ahead of now, and the job holding the lane is the active row
|
||||
* of a heavy queue. Reading both answers the question without a column that
|
||||
* would have to be kept in step with the lane.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { HEAVY_QUEUES } from "@/lib/jobs/heavy-lane";
|
||||
import { logCaught } from "@/lib/logging/signal";
|
||||
|
||||
import { BACKUP_RESTORE_QUEUE } from "./backup-restore-queue";
|
||||
|
||||
/**
|
||||
* Why a queued restore has not started: a backup pass holds the lane, or
|
||||
* something else does (an import, a nightly pass, the memory guard).
|
||||
*/
|
||||
export type RestoreWait = "backup" | "busy";
|
||||
|
||||
/** The queues whose holder the console names as "the backup". */
|
||||
const BACKUP_PASS_QUEUES: ReadonlySet<string> = new Set([
|
||||
"data-backup",
|
||||
"data-backup-offhost",
|
||||
]);
|
||||
|
||||
const OTHER_HEAVY_QUEUES = Object.keys(HEAVY_QUEUES).filter(
|
||||
(queue) => queue !== BACKUP_RESTORE_QUEUE,
|
||||
);
|
||||
|
||||
/**
|
||||
* For each of `deliveries` (pg-boss job ids of queued restores) that the lane
|
||||
* has put back, what holds the lane. A delivery not in the map is waiting
|
||||
* for the worker to pick it up, nothing more.
|
||||
*
|
||||
* Never fails the read it serves: a queue that cannot be asked answers no
|
||||
* waits, reported at warn, and the console says what it said before.
|
||||
*/
|
||||
export async function restoreWaits(
|
||||
deliveries: readonly string[],
|
||||
): Promise<Map<string, RestoreWait>> {
|
||||
const waits = new Map<string, RestoreWait>();
|
||||
if (deliveries.length === 0) return waits;
|
||||
try {
|
||||
const deferred = await prisma.$queryRaw<Array<{ id: string }>>`
|
||||
SELECT id::text AS id
|
||||
FROM pgboss.job
|
||||
WHERE name = ${BACKUP_RESTORE_QUEUE}
|
||||
AND id = ANY(${[...deliveries]}::uuid[])
|
||||
AND state = 'created'
|
||||
AND start_after > now()
|
||||
`;
|
||||
if (deferred.length === 0) return waits;
|
||||
const holders = await prisma.$queryRaw<Array<{ name: string }>>`
|
||||
SELECT name
|
||||
FROM pgboss.job
|
||||
WHERE state = 'active'
|
||||
AND name = ANY(${OTHER_HEAVY_QUEUES}::text[])
|
||||
ORDER BY started_on ASC
|
||||
LIMIT 1
|
||||
`;
|
||||
const holder = holders[0]?.name;
|
||||
const wait: RestoreWait =
|
||||
holder !== undefined && BACKUP_PASS_QUEUES.has(holder)
|
||||
? "backup"
|
||||
: "busy";
|
||||
for (const row of deferred) waits.set(row.id, wait);
|
||||
} catch (err) {
|
||||
logCaught("backup.restore.wait_unread", err);
|
||||
}
|
||||
return waits;
|
||||
}
|
||||
@@ -51,7 +51,6 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import type { Job, PgBoss } from "pg-boss";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import { invalidateUserData } from "@/lib/cache/invalidate";
|
||||
import { prisma, toJson } from "@/lib/db";
|
||||
@@ -79,6 +78,10 @@ import {
|
||||
} from "@/lib/export/restore-failure-cause";
|
||||
import { logCaught } from "@/lib/logging/signal";
|
||||
import { BACKUP_RESTORE_QUEUE } from "./backup-restore-queue";
|
||||
import type {
|
||||
BackupRestoreJobView,
|
||||
BackupRestoreResult,
|
||||
} from "./backup-restore-view";
|
||||
|
||||
/** Stable code of the 409 a second restore of the same account gets. */
|
||||
export const BACKUP_RESTORE_ACTIVE_CODE = "backup.restore.active";
|
||||
@@ -140,71 +143,6 @@ export interface BackupRestoreFailure {
|
||||
cause?: RestoreFailureCause;
|
||||
}
|
||||
|
||||
const failureSchema = z.object({
|
||||
code: z.string(),
|
||||
message: z.string(),
|
||||
sections: z.array(z.string()).optional(),
|
||||
keyIds: z.array(z.string()).optional(),
|
||||
cause: z.enum(RESTORE_FAILURE_CAUSES).optional(),
|
||||
});
|
||||
|
||||
const progressSchema = z.object({
|
||||
measurementsChecked: z.number().int().nonnegative(),
|
||||
measurementsTotal: z.number().int().nonnegative().nullable(),
|
||||
measurementsWritten: z.number().int().nonnegative(),
|
||||
sectionsDone: z.number().int().nonnegative(),
|
||||
sectionsTotal: z.number().int().nonnegative(),
|
||||
});
|
||||
|
||||
const resultSchema = z.object({
|
||||
summary: z.record(z.string(), z.unknown()),
|
||||
skipped: z.object({
|
||||
links: z.number().int(),
|
||||
catalogueKeys: z.array(
|
||||
z.object({
|
||||
catalogue: z.string(),
|
||||
key: z.string(),
|
||||
links: z.number().int(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
cleared: z.record(z.string(), z.number()),
|
||||
});
|
||||
|
||||
export type BackupRestoreResult = z.infer<typeof resultSchema>;
|
||||
|
||||
/** What the status routes answer with. Counts and codes, never content. */
|
||||
export interface BackupRestoreJobView {
|
||||
id: string;
|
||||
userId: string;
|
||||
username: string | null;
|
||||
backupId: string;
|
||||
restoreInstanceSettings: boolean;
|
||||
status: BackupRestoreStatus;
|
||||
phase: RestorePhase | null;
|
||||
progress: RestoreProgress | null;
|
||||
result: BackupRestoreResult | null;
|
||||
failure: BackupRestoreFailure | null;
|
||||
attempts: number;
|
||||
createdAt: string;
|
||||
startedAt: string | null;
|
||||
completedAt: string | null;
|
||||
}
|
||||
|
||||
const PHASES: readonly RestorePhase[] = [
|
||||
"validating",
|
||||
"clearing",
|
||||
"measurements",
|
||||
"sections",
|
||||
"rebuilding",
|
||||
];
|
||||
const STATUSES: readonly BackupRestoreStatus[] = [
|
||||
"queued",
|
||||
"running",
|
||||
"succeeded",
|
||||
"failed",
|
||||
];
|
||||
|
||||
/**
|
||||
* A failure code as a job fact. Job facts are lower-case words joined by
|
||||
* underscores (`job-outcome.ts`), and two of the restore's codes are the
|
||||
@@ -222,107 +160,6 @@ export function backupDigest(identity: string): string {
|
||||
return createHash("sha256").update(identity).digest("hex");
|
||||
}
|
||||
|
||||
const viewSelect = {
|
||||
id: true,
|
||||
userId: true,
|
||||
backupId: true,
|
||||
restoreInstanceSettings: true,
|
||||
status: true,
|
||||
phase: true,
|
||||
progress: true,
|
||||
result: true,
|
||||
failure: true,
|
||||
attempts: true,
|
||||
createdAt: true,
|
||||
startedAt: true,
|
||||
completedAt: true,
|
||||
user: { select: { username: true } },
|
||||
} as const;
|
||||
|
||||
type ViewRow = {
|
||||
id: string;
|
||||
userId: string;
|
||||
backupId: string;
|
||||
restoreInstanceSettings: boolean;
|
||||
status: string;
|
||||
phase: string | null;
|
||||
progress: unknown;
|
||||
result: unknown;
|
||||
failure: unknown;
|
||||
attempts: number;
|
||||
createdAt: Date;
|
||||
startedAt: Date | null;
|
||||
completedAt: Date | null;
|
||||
user: { username: string } | null;
|
||||
};
|
||||
|
||||
function toView(row: ViewRow): BackupRestoreJobView {
|
||||
const status = (STATUSES as readonly string[]).includes(row.status)
|
||||
? (row.status as BackupRestoreStatus)
|
||||
: "failed";
|
||||
const phase =
|
||||
row.phase && (PHASES as readonly string[]).includes(row.phase)
|
||||
? (row.phase as RestorePhase)
|
||||
: null;
|
||||
const progress = progressSchema.safeParse(row.progress);
|
||||
const result = resultSchema.safeParse(row.result);
|
||||
const failure = failureSchema.safeParse(row.failure);
|
||||
return {
|
||||
id: row.id,
|
||||
userId: row.userId,
|
||||
username: row.user?.username ?? null,
|
||||
backupId: row.backupId,
|
||||
restoreInstanceSettings: row.restoreInstanceSettings,
|
||||
status,
|
||||
phase: status === "running" ? phase : null,
|
||||
progress: progress.success ? progress.data : null,
|
||||
result: result.success ? result.data : null,
|
||||
failure: failure.success
|
||||
? (failure.data as BackupRestoreFailure)
|
||||
: status === "failed"
|
||||
? { code: "unexpected", message: "The restore failed." }
|
||||
: null,
|
||||
attempts: row.attempts,
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
startedAt: row.startedAt?.toISOString() ?? null,
|
||||
completedAt: row.completedAt?.toISOString() ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
/** One job, for the status route. Null when the id is unknown. */
|
||||
export async function readBackupRestoreJob(
|
||||
id: string,
|
||||
): Promise<BackupRestoreJobView | null> {
|
||||
const row = await prisma.backupRestoreJob.findUnique({
|
||||
where: { id },
|
||||
select: viewSelect,
|
||||
});
|
||||
return row ? toView(row) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The jobs the console shows: every queued or running one, and the finished
|
||||
* ones of the last day, newest first. What a reload of the page needs to pick
|
||||
* up a restore it was watching, and to show how the last one ended.
|
||||
*/
|
||||
export async function listRecentBackupRestoreJobs(
|
||||
now: Date = new Date(),
|
||||
): Promise<BackupRestoreJobView[]> {
|
||||
const since = new Date(now.getTime() - 24 * 60 * 60 * 1000);
|
||||
const rows = await prisma.backupRestoreJob.findMany({
|
||||
where: {
|
||||
OR: [
|
||||
{ status: { in: ["queued", "running"] } },
|
||||
{ createdAt: { gte: since } },
|
||||
],
|
||||
},
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: 20,
|
||||
select: viewSelect,
|
||||
});
|
||||
return rows.map(toView);
|
||||
}
|
||||
|
||||
function failureJson(failure: BackupRestoreFailure) {
|
||||
return toJson({
|
||||
code: failure.code,
|
||||
|
||||
@@ -132,6 +132,12 @@ const restoreJob = z
|
||||
})
|
||||
.nullable(),
|
||||
failure: restoreFailure.nullable(),
|
||||
waiting: z
|
||||
.enum(["backup", "busy"])
|
||||
.nullable()
|
||||
.describe(
|
||||
"Why a `queued` job has not started. A restore never runs beside a backup or another large job: while one holds the server it goes back to the queue for a minute at a time. `backup` when a backup pass holds it, `busy` for anything else (an import, a nightly pass, the memory guard). Null while the job only waits for the worker, and on every job that is not queued.",
|
||||
),
|
||||
attempts: z.number().int(),
|
||||
createdAt: z.string(),
|
||||
startedAt: z.string().nullable(),
|
||||
|
||||
Reference in new issue
Block a user