mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-18 08:02:12 +00:00
feat(admin): say what a queued restore is waiting for
A restore never runs beside a backup pass or another large job: the heavy lane puts its delivery back in the queue for a minute at a time. The job's own row says queued for the whole wait, and the console said the restore starts "as soon as the background worker picks it up", which read as a stuck worker while a backup ran. The status routes now ask the queue. A delivery the lane put back sits in created with a start_after ahead of now, and the active row of a heavy queue is what holds the lane. The job view carries waiting: "backup" | "busy" | null, and the console says it waits for the running backup, or for another large job, and since when it has been queued, in all nine locales. A queue that cannot be asked answers no wait and is reported at warn. The job view and its reads move to backup-restore-view.ts, which keeps the worker module under its size baseline.
This commit is contained in:
1 parent
6a87810dbf
commit
7936ab9a57
28 files changed
+544
-199
No files matched your search
@@ -461,8 +461,9 @@ describe("a restore queued against a copy that is then replaced", () => {
|
||||
await import("@/app/api/admin/backups/[id]/restore/route");
|
||||
const { getGlobalBoss, setGlobalBoss } =
|
||||
await import("@/lib/jobs/boss-instance");
|
||||
const { readBackupRestoreJob, runBackupRestoreJob } =
|
||||
await import("@/lib/jobs/backup-restore");
|
||||
const { runBackupRestoreJob } = await import("@/lib/jobs/backup-restore");
|
||||
const { readBackupRestoreJob } =
|
||||
await import("@/lib/jobs/backup-restore-view");
|
||||
|
||||
const previous = getGlobalBoss();
|
||||
setGlobalBoss(acceptingBoss);
|
||||
|
||||
@@ -45,10 +45,10 @@ import { runJob } from "@/lib/jobs/run-job";
|
||||
import {
|
||||
BACKUP_RESTORE_MAX_ATTEMPTS,
|
||||
handleBackupRestore,
|
||||
readBackupRestoreJob,
|
||||
runBackupRestoreJob,
|
||||
sweepInterruptedRestores,
|
||||
} from "@/lib/jobs/backup-restore";
|
||||
import { readBackupRestoreJob } from "@/lib/jobs/backup-restore-view";
|
||||
import { BACKUP_RESTORE_QUEUE } from "@/lib/jobs/backup-restore-queue";
|
||||
import { backupPayloadSchema } from "@/lib/validations/backup";
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
|
||||
import { storeBackupBlob } from "@/lib/export/store-backup-blob";
|
||||
import { RESTORE_FAILURE_CAUSE_MESSAGES } from "@/lib/export/restore-failure-cause";
|
||||
import { readBackupRestoreJob } from "@/lib/jobs/backup-restore";
|
||||
import { readBackupRestoreJob } from "@/lib/jobs/backup-restore-view";
|
||||
|
||||
vi.mock("next/headers", async () => {
|
||||
const { cookieJar, headerJar } = await import("./mock-next-headers");
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* What a queued restore says it waits for, read from a real pg-boss.
|
||||
*
|
||||
* A restore never runs beside a backup pass or another large job: the heavy
|
||||
* lane puts its delivery back in the queue for a minute at a time
|
||||
* (`deferLaneJobs`). Its own row says `queued` throughout, and the console
|
||||
* used to say only that it starts "as soon as the background worker picks it
|
||||
* up", which read as a stuck worker while a backup ran. The status read now
|
||||
* asks the queue: a delivery that went back, and the heavy job that holds
|
||||
* the lane.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { PgBoss } from "pg-boss";
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { readBackupRestoreJob } from "@/lib/jobs/backup-restore-view";
|
||||
import { BACKUP_RESTORE_QUEUE } from "@/lib/jobs/backup-restore-queue";
|
||||
import { restoreWaits } from "@/lib/jobs/backup-restore-wait";
|
||||
import { deferLaneJobs } from "@/lib/jobs/heavy-lane";
|
||||
|
||||
import { makeUser } from "../kit/factories";
|
||||
import { truncateAllTables } from "./setup";
|
||||
|
||||
let boss: PgBoss;
|
||||
|
||||
const QUEUES = [BACKUP_RESTORE_QUEUE, "data-backup", "rollup-full-backfill"];
|
||||
|
||||
beforeAll(async () => {
|
||||
await truncateAllTables(prisma);
|
||||
boss = new PgBoss({ connectionString: process.env.DATABASE_URL });
|
||||
await boss.start();
|
||||
for (const queue of QUEUES) await boss.createQueue(queue);
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
for (const queue of QUEUES) await boss.deleteAllJobs(queue);
|
||||
await boss?.stop({ graceful: false });
|
||||
await truncateAllTables(prisma);
|
||||
});
|
||||
|
||||
/** A restore delivery the lane has put back, as `heavyLane` leaves it. */
|
||||
async function deferredRestore(): Promise<string> {
|
||||
const id = await boss.send(BACKUP_RESTORE_QUEUE, { restoreJobId: "r" });
|
||||
const [fetched] = await boss.fetch(BACKUP_RESTORE_QUEUE);
|
||||
expect(fetched?.id).toBe(id);
|
||||
expect((await deferLaneJobs(BACKUP_RESTORE_QUEUE, [fetched!])).deferred).toBe(
|
||||
1,
|
||||
);
|
||||
return id!;
|
||||
}
|
||||
|
||||
/** A job of `queue` running, so it holds the lane. */
|
||||
async function running(queue: string): Promise<string> {
|
||||
const id = await boss.send(queue, {});
|
||||
const [fetched] = await boss.fetch(queue);
|
||||
expect(fetched?.id).toBe(id);
|
||||
return id!;
|
||||
}
|
||||
|
||||
describe("restoreWaits", () => {
|
||||
it("names the backup when a backup pass holds the lane", async () => {
|
||||
const restore = await deferredRestore();
|
||||
const backup = await running("data-backup");
|
||||
expect(await restoreWaits([restore])).toEqual(
|
||||
new Map([[restore, "backup"]]),
|
||||
);
|
||||
await boss.complete("data-backup", backup);
|
||||
|
||||
// Another large job holding it is named as one.
|
||||
const fold = await running("rollup-full-backfill");
|
||||
expect((await restoreWaits([restore])).get(restore)).toBe("busy");
|
||||
await boss.complete("rollup-full-backfill", fold);
|
||||
|
||||
// Put back with nothing running (the memory guard, or a holder that has
|
||||
// just finished): still a wait for room, not for the worker.
|
||||
expect((await restoreWaits([restore])).get(restore)).toBe("busy");
|
||||
await boss.deleteJob(BACKUP_RESTORE_QUEUE, restore);
|
||||
});
|
||||
|
||||
it("says nothing about a delivery that only waits for the worker", async () => {
|
||||
const queued = await boss.send(BACKUP_RESTORE_QUEUE, { restoreJobId: "q" });
|
||||
const backup = await running("data-backup");
|
||||
expect(await restoreWaits([queued!])).toEqual(new Map());
|
||||
await boss.complete("data-backup", backup);
|
||||
await boss.deleteJob(BACKUP_RESTORE_QUEUE, queued!);
|
||||
});
|
||||
|
||||
it("reaches the status read of the job behind the delivery", async () => {
|
||||
const owner = await makeUser({});
|
||||
const restore = await deferredRestore();
|
||||
const job = await prisma.backupRestoreJob.create({
|
||||
data: {
|
||||
userId: owner.id,
|
||||
actorUserId: owner.id,
|
||||
backupId: "backup-1",
|
||||
backupDigest: "digest",
|
||||
status: "queued",
|
||||
pgBossJobId: restore,
|
||||
},
|
||||
});
|
||||
const backup = await running("data-backup");
|
||||
expect((await readBackupRestoreJob(job.id))?.waiting).toBe("backup");
|
||||
await boss.complete("data-backup", backup);
|
||||
|
||||
// A job that is no longer queued waits for nothing, whatever its last
|
||||
// delivery says.
|
||||
await prisma.backupRestoreJob.update({
|
||||
where: { id: job.id },
|
||||
data: { status: "running" },
|
||||
});
|
||||
expect((await readBackupRestoreJob(job.id))?.waiting).toBeNull();
|
||||
await boss.deleteJob(BACKUP_RESTORE_QUEUE, restore);
|
||||
});
|
||||
});
|
||||
@@ -191,7 +191,8 @@ describe("restore against a running consolidation (#1031)", () => {
|
||||
const restore = await startRestore(backupId);
|
||||
|
||||
const [folded] = await Promise.all([consolidation, restore.done]);
|
||||
const { readBackupRestoreJob } = await import("@/lib/jobs/backup-restore");
|
||||
const { readBackupRestoreJob } =
|
||||
await import("@/lib/jobs/backup-restore-view");
|
||||
const job = await readBackupRestoreJob(restore.jobId);
|
||||
|
||||
expect(
|
||||
|
||||
@@ -17,10 +17,8 @@ import type { PgBoss } from "pg-boss";
|
||||
|
||||
import { POST as queueRestore } from "@/app/api/admin/backups/[id]/restore/route";
|
||||
import { getGlobalBoss, setGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import {
|
||||
readBackupRestoreJob,
|
||||
runBackupRestoreJob,
|
||||
} from "@/lib/jobs/backup-restore";
|
||||
import { runBackupRestoreJob } from "@/lib/jobs/backup-restore";
|
||||
import { readBackupRestoreJob } from "@/lib/jobs/backup-restore-view";
|
||||
|
||||
/** The status the synchronous route answered each refusal with. */
|
||||
const LEGACY_STATUS: Record<string, number> = {
|
||||
|
||||
Reference in new issue
Block a user