mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-19 08:12:52 +00:00
fix(jobs): run one heavy job at a time per process
Web and worker share one process and one heap limit, and the jobs that hold a large working set (the Apple Health and Health Connect imports, restore, the weekly and off-host backups, the restore drill, the rollup backfill, the nightly status batch, the pre-generation walk and the narrative walk) ran side by side whenever their schedules or an upload met, so the peak was their sum. `heavyLane` queues them in the process, first come first served. A waiter leaves the line when pg-boss expires it and its retry takes the place; a one-shot whose input lives outside pg-boss keeps its place. The runtime guard binds every heavy queue in the lane, keeps `lockedPass` outside it, and allows the one-shot rule only where the send options forbid a retry.
This commit is contained in:
1 parent
d9e56482d9
commit
1ee73908ae
7 files changed
+388
-51
No files matched your search
@@ -42,6 +42,7 @@ import {
|
||||
type SourceSite,
|
||||
} from "@/lib/jobs/queue-runtime";
|
||||
import { SCHEDULED_AND_SENT_QUEUES } from "@/lib/jobs/reminder/registrar-shared";
|
||||
import { HEAVY_QUEUES } from "@/lib/jobs/heavy-lane";
|
||||
|
||||
const SRC = join(process.cwd(), "src");
|
||||
const REGISTRARS = [
|
||||
@@ -415,3 +416,46 @@ describe("a cron-fed queue polls slowly, a sent one does not", () => {
|
||||
expect(dynamic.has("whoop-recovery-sync")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/** The text of the first `const NAME = …` in the tree, up to its end. */
|
||||
function constBody(texts: ReadonlyMap<string, string>, name: string): string {
|
||||
for (const [file, raw] of texts) {
|
||||
if (new RegExp(`\\bconst\\s+${name}\\b`).test(raw)) {
|
||||
return bodyOf(texts, { file, fn: name }).split(/\n\n/)[0];
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
describe("heavy jobs share the lane", () => {
|
||||
const all = scan(source);
|
||||
const laned = all.bindings.filter((b) => /\bheavyLane\s*\(/.test(b.text));
|
||||
|
||||
it("binds every heavy queue in the lane, and nothing else", () => {
|
||||
expect(laned.map((b) => b.queue).sort()).toEqual(
|
||||
Object.keys(HEAVY_QUEUES).sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("holds a lockedPass outside the lane, so an expiry retry does not queue twice", () => {
|
||||
const inverted = laned
|
||||
.filter((b) => /\blockedPass\s*\(/.test(b.text))
|
||||
.filter((b) => b.text.indexOf("lockedPass") > b.text.indexOf("heavyLane"))
|
||||
.map((b) => b.queue);
|
||||
expect(inverted).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps a waiter past its expiry only where nothing retries the job", () => {
|
||||
const problems = Object.entries(HEAVY_QUEUES)
|
||||
.filter(([, heavy]) => heavy.oneShot)
|
||||
.flatMap(([queue]) => {
|
||||
const entry = QUEUE_RUNTIME[queue];
|
||||
if (entry?.runtime !== "long") return [`${queue}: not a long queue`];
|
||||
const noRetry = entry.expiryVia.some((v) =>
|
||||
/\bretryLimit:\s*0\b/.test(constBody(source, v)),
|
||||
);
|
||||
return noRetry ? [] : [`${queue}: its send options allow a retry`];
|
||||
});
|
||||
expect(problems).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user