mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 23:33:29 +00:00
Up to 12.37 pg-boss retried a job whose handler outlived its expiry in the same worker beside the run it had given up on, and the lane test pinned that as an executable statement. 12.38 fails the job and fires the signal as before, but the worker fetches nothing more until the handler returns, and the retry starts after it. The test now pins that: one run at a time, the retry starting after the overrun ends, and the lane's handler importing once per delivery with never two at once. The lane's lock stays, as the guard for a second worker or instance.
172 lines
6.2 KiB
TypeScript
172 lines
6.2 KiB
TypeScript
/**
|
|
* The full-history admission lane against a real pg-boss on a real Postgres:
|
|
* what happens to an import that runs past its job's expiry.
|
|
*
|
|
* Up to 12.37, measured with the lane's worker shape (localConcurrency 1,
|
|
* groupConcurrency 1, exclusive policy, retries with backoff), a two-second
|
|
* expiry, a one-second retry delay and a handler that ignores the abort
|
|
* signal and runs for eight seconds, pg-boss failed the job at the expiry,
|
|
* fired the signal, freed the worker and group slots, and started the retry
|
|
* in the same worker BESIDE the run it had given up on: two runs of one job
|
|
* at once, which the lane's lock existed to absorb.
|
|
*
|
|
* 12.38 changed that. At the expiry it still fails the job and fires the
|
|
* signal, but the worker fetches nothing more until the handler returns (and
|
|
* records a `handler_overrun` warning). The retry starts after the overrun
|
|
* ends, in the same worker, so the lane's one-at-a-time now holds without the
|
|
* lock's help; the lock stays as the guard for a second worker or instance.
|
|
* The first test keeps that as an executable statement about pg-boss; the
|
|
* second runs the lane's own handler under the same conditions.
|
|
*/
|
|
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
|
import { PgBoss, type Job } from "pg-boss";
|
|
|
|
const runner = vi.hoisted(() => ({
|
|
running: 0,
|
|
maxRunning: 0,
|
|
calls: 0,
|
|
holdMs: 8_000,
|
|
}));
|
|
|
|
vi.mock("@/lib/jobs/google-health-backfill", () => ({
|
|
// Ignores the budget and the signal on purpose: the case the lock covers
|
|
// is a run that does not stop when pg-boss gives up on it.
|
|
runGoogleHealthBackfillForUser: async () => {
|
|
runner.calls += 1;
|
|
runner.running += 1;
|
|
runner.maxRunning = Math.max(runner.maxRunning, runner.running);
|
|
await new Promise((resolve) => setTimeout(resolve, runner.holdMs));
|
|
runner.running -= 1;
|
|
return { imported: 1 };
|
|
},
|
|
}));
|
|
|
|
import {
|
|
integrationBackfillAdmissionSendOptions,
|
|
type IntegrationBackfillAdmissionPayload,
|
|
} from "@/lib/jobs/integration-backfill-admission";
|
|
import { drainIntegrationBackfillAdmission } from "@/lib/jobs/integration-backfill-drain";
|
|
import { withIntegrationBackfillLock } from "@/lib/jobs/integration-backfill-lock";
|
|
import { runJob } from "@/lib/jobs/run-job";
|
|
|
|
let boss: PgBoss;
|
|
const suffix = `${Date.now()}-${Math.floor(Math.random() * 1e6)}`;
|
|
|
|
/** Short enough to cross in a test, the lane's shape otherwise. */
|
|
const EXPIRY_OVERRIDE = { expireInSeconds: 2, retryDelay: 1 } as const;
|
|
|
|
const WORK_OPTIONS = {
|
|
localConcurrency: 1,
|
|
groupConcurrency: 1,
|
|
pollingIntervalSeconds: 0.5,
|
|
} as const;
|
|
|
|
const payload: IntegrationBackfillAdmissionPayload = {
|
|
kind: "google-health-backfill",
|
|
data: { userId: `lane-user-${suffix}`, enqueuedAt: new Date().toISOString() },
|
|
};
|
|
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
beforeAll(async () => {
|
|
boss = new PgBoss({ connectionString: process.env.DATABASE_URL });
|
|
await boss.start();
|
|
}, 120_000);
|
|
|
|
afterAll(async () => {
|
|
await boss?.stop({ graceful: false });
|
|
});
|
|
|
|
describe("an import that outlives its expiry", () => {
|
|
it("pg-boss retries the expired job only once the run it gave up on returns", async () => {
|
|
const queue = `lane-expiry-raw-${suffix}`;
|
|
await boss.createQueue(queue, { policy: "exclusive" });
|
|
const runs: Array<{ id: string; start: number; end?: number }> = [];
|
|
let running = 0;
|
|
let maxRunning = 0;
|
|
await boss.work(queue, WORK_OPTIONS, async ([job]: Job<object>[]) => {
|
|
const run: { id: string; start: number; end?: number } = {
|
|
id: job!.id,
|
|
start: Date.now(),
|
|
};
|
|
runs.push(run);
|
|
running += 1;
|
|
maxRunning = Math.max(maxRunning, running);
|
|
await sleep(8_000);
|
|
running -= 1;
|
|
run.end = Date.now();
|
|
});
|
|
await boss.send(queue, payload, {
|
|
...integrationBackfillAdmissionSendOptions(payload),
|
|
...EXPIRY_OVERRIDE,
|
|
});
|
|
|
|
await sleep(12_000);
|
|
await boss.offWork(queue);
|
|
|
|
expect(runs.length).toBeGreaterThanOrEqual(2);
|
|
expect(new Set(runs.map((r) => r.id)).size).toBe(1);
|
|
expect(maxRunning).toBe(1);
|
|
expect(runs[1]!.start).toBeGreaterThanOrEqual(runs[0]!.end!);
|
|
}, 60_000);
|
|
|
|
it("the lane's handler runs one import at a time while the expired run overruns", async () => {
|
|
const queue = `lane-expiry-drain-${suffix}`;
|
|
await boss.createQueue(queue, { policy: "exclusive" });
|
|
const handler = runJob(queue, drainIntegrationBackfillAdmission);
|
|
const deliveries: string[] = [];
|
|
await boss.work(
|
|
queue,
|
|
WORK_OPTIONS,
|
|
async (jobs: Job<IntegrationBackfillAdmissionPayload>[]) => {
|
|
deliveries.push(jobs[0]!.id);
|
|
return handler(jobs);
|
|
},
|
|
);
|
|
await boss.send(queue, payload, {
|
|
...integrationBackfillAdmissionSendOptions(payload),
|
|
...EXPIRY_OVERRIDE,
|
|
});
|
|
|
|
// The run holds for eight seconds, past its two-second expiry; the retry
|
|
// waits for it and starts after.
|
|
await sleep(12_000);
|
|
await boss.offWork(queue);
|
|
await sleep(8_500);
|
|
|
|
expect(deliveries.length).toBeGreaterThanOrEqual(2);
|
|
expect(new Set(deliveries).size).toBe(1);
|
|
expect(runner.maxRunning).toBe(1);
|
|
expect(runner.calls).toBe(deliveries.length);
|
|
}, 60_000);
|
|
});
|
|
|
|
describe("withIntegrationBackfillLock", () => {
|
|
it("admits one run per identity, others alongside, and the next once it ends", async () => {
|
|
let release!: () => void;
|
|
const held = new Promise<void>((resolve) => (release = resolve));
|
|
const first = withIntegrationBackfillLock("kind|a", () => held);
|
|
await sleep(300);
|
|
|
|
const same = await withIntegrationBackfillLock("kind|a", async () => 1);
|
|
const other = await withIntegrationBackfillLock("kind|b", async () => 2);
|
|
expect(same).toEqual({ ran: false });
|
|
expect(other).toEqual({ ran: true, result: 2 });
|
|
|
|
release();
|
|
await first;
|
|
const after = await withIntegrationBackfillLock("kind|a", async () => 3);
|
|
expect(after).toEqual({ ran: true, result: 3 });
|
|
});
|
|
|
|
it("releases the lock when the run throws", async () => {
|
|
await expect(
|
|
withIntegrationBackfillLock("kind|c", async () => {
|
|
throw new Error("boom");
|
|
}),
|
|
).rejects.toThrow("boom");
|
|
const after = await withIntegrationBackfillLock("kind|c", async () => 4);
|
|
expect(after).toEqual({ ran: true, result: 4 });
|
|
});
|
|
});
|