/** * The full-history admission lane against a real pg-boss on a real Postgres: * what happens to an import that runs past its job's expiry. * * Up to 12.37, measured with the lane's worker shape (localConcurrency 1, * groupConcurrency 1, exclusive policy, retries with backoff), a two-second * expiry, a one-second retry delay and a handler that ignores the abort * signal and runs for eight seconds, pg-boss failed the job at the expiry, * fired the signal, freed the worker and group slots, and started the retry * in the same worker BESIDE the run it had given up on: two runs of one job * at once, which the lane's lock existed to absorb. * * 12.38 changed that. At the expiry it still fails the job and fires the * signal, but the worker fetches nothing more until the handler returns (and * records a `handler_overrun` warning). The retry starts after the overrun * ends, in the same worker, so the lane's one-at-a-time now holds without the * lock's help; the lock stays as the guard for a second worker or instance. * The first test keeps that as an executable statement about pg-boss; the * second runs the lane's own handler under the same conditions. */ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { PgBoss, type Job } from "pg-boss"; const runner = vi.hoisted(() => ({ running: 0, maxRunning: 0, calls: 0, holdMs: 8_000, })); vi.mock("@/lib/jobs/google-health-backfill", () => ({ // Ignores the budget and the signal on purpose: the case the lock covers // is a run that does not stop when pg-boss gives up on it. runGoogleHealthBackfillForUser: async () => { runner.calls += 1; runner.running += 1; runner.maxRunning = Math.max(runner.maxRunning, runner.running); await new Promise((resolve) => setTimeout(resolve, runner.holdMs)); runner.running -= 1; return { imported: 1 }; }, })); import { integrationBackfillAdmissionSendOptions, type IntegrationBackfillAdmissionPayload, } from "@/lib/jobs/integration-backfill-admission"; import { drainIntegrationBackfillAdmission } from "@/lib/jobs/integration-backfill-drain"; import { withIntegrationBackfillLock } from "@/lib/jobs/integration-backfill-lock"; import { runJob } from "@/lib/jobs/run-job"; let boss: PgBoss; const suffix = `${Date.now()}-${Math.floor(Math.random() * 1e6)}`; /** Short enough to cross in a test, the lane's shape otherwise. */ const EXPIRY_OVERRIDE = { expireInSeconds: 2, retryDelay: 1 } as const; const WORK_OPTIONS = { localConcurrency: 1, groupConcurrency: 1, pollingIntervalSeconds: 0.5, } as const; const payload: IntegrationBackfillAdmissionPayload = { kind: "google-health-backfill", data: { userId: `lane-user-${suffix}`, enqueuedAt: new Date().toISOString() }, }; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); /** * Waits for a state rather than a fixed time: the retry is fetched about a * second and a half after the overrun ends, later on a loaded machine, and a * fixed window turned that lateness into a red run. The deadline only bounds * a run that never gets there. */ async function until(done: () => boolean, deadlineMs = 40_000): Promise { const end = Date.now() + deadlineMs; while (!done() && Date.now() < end) await sleep(100); } beforeAll(async () => { boss = new PgBoss({ connectionString: process.env.DATABASE_URL }); await boss.start(); }, 120_000); afterAll(async () => { await boss?.stop({ graceful: false }); }); describe("an import that outlives its expiry", () => { it("pg-boss retries the expired job only once the run it gave up on returns", async () => { const queue = `lane-expiry-raw-${suffix}`; await boss.createQueue(queue, { policy: "exclusive" }); const runs: Array<{ id: string; start: number; end?: number }> = []; let running = 0; let maxRunning = 0; await boss.work(queue, WORK_OPTIONS, async ([job]: Job[]) => { const run: { id: string; start: number; end?: number } = { id: job!.id, start: Date.now(), }; runs.push(run); running += 1; maxRunning = Math.max(maxRunning, running); await sleep(8_000); running -= 1; run.end = Date.now(); }); await boss.send(queue, payload, { ...integrationBackfillAdmissionSendOptions(payload), ...EXPIRY_OVERRIDE, }); await until(() => runs.length >= 2); await boss.offWork(queue); expect(runs.length).toBeGreaterThanOrEqual(2); expect(new Set(runs.map((r) => r.id)).size).toBe(1); expect(maxRunning).toBe(1); expect(runs[1]!.start).toBeGreaterThanOrEqual(runs[0]!.end!); }, 90_000); it("the lane's handler runs one import at a time while the expired run overruns", async () => { const queue = `lane-expiry-drain-${suffix}`; await boss.createQueue(queue, { policy: "exclusive" }); const handler = runJob(queue, drainIntegrationBackfillAdmission); const deliveries: string[] = []; await boss.work( queue, WORK_OPTIONS, async (jobs: Job[]) => { deliveries.push(jobs[0]!.id); return handler(jobs); }, ); await boss.send(queue, payload, { ...integrationBackfillAdmissionSendOptions(payload), ...EXPIRY_OVERRIDE, }); // The run holds for eight seconds, past its two-second expiry; the retry // waits for it and starts after. The retry's own run is let finish so the // call count is read at rest. await until(() => deliveries.length >= 2); await boss.offWork(queue); await until( () => runner.running === 0 && runner.calls >= deliveries.length, ); expect(deliveries.length).toBeGreaterThanOrEqual(2); expect(new Set(deliveries).size).toBe(1); expect(runner.maxRunning).toBe(1); expect(runner.calls).toBe(deliveries.length); }, 100_000); }); describe("withIntegrationBackfillLock", () => { it("admits one run per identity, others alongside, and the next once it ends", async () => { let release!: () => void; const held = new Promise((resolve) => (release = resolve)); const first = withIntegrationBackfillLock("kind|a", () => held); await sleep(300); const same = await withIntegrationBackfillLock("kind|a", async () => 1); const other = await withIntegrationBackfillLock("kind|b", async () => 2); expect(same).toEqual({ ran: false }); expect(other).toEqual({ ran: true, result: 2 }); release(); await first; const after = await withIntegrationBackfillLock("kind|a", async () => 3); expect(after).toEqual({ ran: true, result: 3 }); }); it("releases the lock when the run throws", async () => { await expect( withIntegrationBackfillLock("kind|c", async () => { throw new Error("boom"); }), ).rejects.toThrow("boom"); const after = await withIntegrationBackfillLock("kind|c", async () => 4); expect(after).toEqual({ ran: true, result: 4 }); }); });