fix(jobs): run one heavy job at a time per process

Web and worker share one process and one heap limit, and the jobs that
hold a large working set (the Apple Health and Health Connect imports,
restore, the weekly and off-host backups, the restore drill, the rollup
backfill, the nightly status batch, the pre-generation walk and the
narrative walk) ran side by side whenever their schedules or an upload
met, so the peak was their sum. `heavyLane` queues them in the process,
first come first served. A waiter leaves the line when pg-boss expires it
and its retry takes the place; a one-shot whose input lives outside
pg-boss keeps its place. The runtime guard binds every heavy queue in the
lane, keeps `lockedPass` outside it, and allows the one-shot rule only
where the send options forbid a retry.
This commit is contained in:
Marc-André Bombeck committed 2026-10-20 03:35:23 +02:00
1 parent d9e56482d9
commit 1ee73908ae
7 files changed
+388 -51

No files matched your search

@@ -42,6 +42,7 @@ import {
type SourceSite,
} from "@/lib/jobs/queue-runtime";
import { SCHEDULED_AND_SENT_QUEUES } from "@/lib/jobs/reminder/registrar-shared";
import { HEAVY_QUEUES } from "@/lib/jobs/heavy-lane";
const SRC = join(process.cwd(), "src");
const REGISTRARS = [
@@ -415,3 +416,46 @@ describe("a cron-fed queue polls slowly, a sent one does not", () => {
expect(dynamic.has("whoop-recovery-sync")).toBe(true);
});
});
/** The text of the first `const NAME = …` in the tree, up to its end. */
function constBody(texts: ReadonlyMap<string, string>, name: string): string {
for (const [file, raw] of texts) {
if (new RegExp(`\\bconst\\s+${name}\\b`).test(raw)) {
return bodyOf(texts, { file, fn: name }).split(/\n\n/)[0];
}
}
return "";
}
describe("heavy jobs share the lane", () => {
const all = scan(source);
const laned = all.bindings.filter((b) => /\bheavyLane\s*\(/.test(b.text));
it("binds every heavy queue in the lane, and nothing else", () => {
expect(laned.map((b) => b.queue).sort()).toEqual(
Object.keys(HEAVY_QUEUES).sort(),
);
});
it("holds a lockedPass outside the lane, so an expiry retry does not queue twice", () => {
const inverted = laned
.filter((b) => /\blockedPass\s*\(/.test(b.text))
.filter((b) => b.text.indexOf("lockedPass") > b.text.indexOf("heavyLane"))
.map((b) => b.queue);
expect(inverted).toEqual([]);
});
it("keeps a waiter past its expiry only where nothing retries the job", () => {
const problems = Object.entries(HEAVY_QUEUES)
.filter(([, heavy]) => heavy.oneShot)
.flatMap(([queue]) => {
const entry = QUEUE_RUNTIME[queue];
if (entry?.runtime !== "long") return [`${queue}: not a long queue`];
const noRetry = entry.expiryVia.some((v) =>
/\bretryLimit:\s*0\b/.test(constBody(source, v)),
);
return noRetry ? [] : [`${queue}: its send options allow a retry`];
});
expect(problems).toEqual([]);
});
});