refactor(jobs): declare every queue in a typed registry and load handlers on first job

The five hand-written registrars (about 5,500 lines, one 997-line function)
are replaced by `jobs/registry/`: one `defineJob` entry per queue with its
cron, schedule send options, policy, work options, lock, lane and a handler
loader. `registerJobGroup` creates, schedules, retires and binds each group
in the order the registrars used; the worker boot walks the groups and runs
their boot discovery.

Handlers load on their queue's first job. The facts a declaration needs at
boot (names, crons, concurrency, send options) moved out of the heavy job
modules into light `*-queue.ts` siblings, and their importers point there,
so loading the registry evaluates 288 modules instead of the 1,164 the
registrars pulled in. The formerly inline handlers moved verbatim into
handler modules under `jobs/reminder/`.

Fitbit's queues live in `fitbit/jobs/definitions.ts` and enter the registry
through one spread, so removing the integration later is that folder and
that line.

`tests/integration/queue-registration-parity.test.ts` proves pg-boss holds
exactly what the registrars left (queues, schedules, bindings, byte for
byte) and loads every handler once. The queue tests and the runtime guard
assert on the registry objects instead of reading registrar source.
This commit is contained in:
Marc-André Bombeck committed 2026-10-23 12:36:09 +02:00
1 parent df6e182e0b
commit 4807a70c1d
193 files changed
+6830 -7950

No files matched your search

+8 -13
View File
@@ -1,7 +1,7 @@
{
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 121,
"total": 143770,
"count": 116,
"total": 137648,
"entries": {
"app/api/dashboard/summary/route.ts": 1032,
"app/api/documents/inbound/route.ts": 811,
@@ -89,18 +89,13 @@
"lib/insights/mood-status.ts": 814,
"lib/insights/status-cache.ts": 812,
"lib/integrations/status.ts": 1445,
"lib/jobs/apple-health-import-worker.ts": 801,
"lib/jobs/backup-restore.ts": 888,
"lib/jobs/coach-nudge.ts": 1089,
"lib/jobs/insight-pregenerate.ts": 1266,
"lib/jobs/medication-intake-import.ts": 822,
"lib/jobs/offhost-backup.ts": 1168,
"lib/jobs/backup-restore.ts": 887,
"lib/jobs/coach-nudge.ts": 1084,
"lib/jobs/insight-pregenerate.ts": 1259,
"lib/jobs/medication-intake-import.ts": 819,
"lib/jobs/offhost-backup.ts": 1147,
"lib/jobs/reaction-line.ts": 824,
"lib/jobs/reminder/medication-reminder-check.ts": 818,
"lib/jobs/reminder/register-integration-sync.ts": 1051,
"lib/jobs/reminder/register-maintenance.ts": 2197,
"lib/jobs/reminder/register-rollup.ts": 1156,
"lib/jobs/reminder/register-status.ts": 878,
"lib/managed-profiles/handover.ts": 986,
"lib/mcp/record-search.ts": 1467,
"lib/mcp/rich-reads.ts": 1467,
@@ -117,7 +112,7 @@
"lib/rollups/measurement-rollups.ts": 1496,
"lib/search/search-records.ts": 996,
"lib/sharing/grants.ts": 1046,
"lib/support/report.ts": 905,
"lib/support/report.ts": 903,
"lib/telegram-webhook-handlers.ts": 1341,
"lib/validations/inbound-documents.ts": 1130,
"lib/whoop/client.ts": 917,
@@ -1,7 +1,7 @@
{
"$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 410,
"total": 147795,
"count": 403,
"total": 144626,
"entries": {
"app/achievements/page.tsx :: AchievementsPage": 206,
"app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314,
@@ -368,13 +368,6 @@
"lib/jobs/reaction-line.ts :: runReactionLine": 337,
"lib/jobs/reminder/medication-reminder-check.ts :: handleReminderCheck": 619,
"lib/jobs/reminder/medication-reminder-check.ts :: handleReminderCheck > withBackgroundEvent(arg)": 614,
"lib/jobs/reminder/register-integration-sync.ts :: enqueueIntegrationSyncBootDiscovery": 224,
"lib/jobs/reminder/register-integration-sync.ts :: registerIntegrationSyncQueues": 320,
"lib/jobs/reminder/register-maintenance.ts :: enqueueMaintenanceBootDiscovery": 232,
"lib/jobs/reminder/register-maintenance.ts :: registerMaintenanceQueues": 997,
"lib/jobs/reminder/register-rollup.ts :: enqueueRollupBootDiscovery": 238,
"lib/jobs/reminder/register-rollup.ts :: registerRollupQueues": 616,
"lib/jobs/reminder/register-status.ts :: registerStatusQueues": 542,
"lib/jobs/workout-insight-generate.ts :: runWorkoutInsightGenerate": 324,
"lib/labs/biomarker-merge.ts :: runMerge": 254,
"lib/labs/ocr-extract.ts :: runOcrExtraction": 207,
+2 -20
View File
@@ -1,7 +1,7 @@
{
"$comment": "tests that read a source file by path, per test file under src/. Maintained by src/__tests__/test-realism-ratchet.test.ts.",
"count": 367,
"total": 367,
"count": 349,
"total": 349,
"entries": {
"__tests__/account-payload-consumer-guard.test.ts": 1,
"__tests__/admin-data-wipe-completeness.test.ts": 1,
@@ -314,44 +314,26 @@
"lib/jobs/__tests__/ai-job-capability-inventory.test.ts": 1,
"lib/jobs/__tests__/apple-health-import-ecg.test.ts": 1,
"lib/jobs/__tests__/apple-health-import-export-date.test.ts": 1,
"lib/jobs/__tests__/apple-health-import-reconcile.test.ts": 1,
"lib/jobs/__tests__/apple-health-import-send-policy.test.ts": 1,
"lib/jobs/__tests__/coach-memory-refresh-queue.test.ts": 1,
"lib/jobs/__tests__/coach-message-cleanup-queue.test.ts": 1,
"lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts": 1,
"lib/jobs/__tests__/cycle-reminder-queue.test.ts": 1,
"lib/jobs/__tests__/daily-briefing.test.ts": 1,
"lib/jobs/__tests__/data-arrival-provider-isolation.test.ts": 1,
"lib/jobs/__tests__/data-arrival-queue.test.ts": 1,
"lib/jobs/__tests__/data-backup-send-policy.test.ts": 1,
"lib/jobs/__tests__/drain-cumulative-queue.test.ts": 1,
"lib/jobs/__tests__/fitbit-queues.test.ts": 1,
"lib/jobs/__tests__/geo-backfill.test.ts": 1,
"lib/jobs/__tests__/geolite2-fetch-queue.test.ts": 1,
"lib/jobs/__tests__/google-health-queues.test.ts": 1,
"lib/jobs/__tests__/insight-pregenerate.test.ts": 1,
"lib/jobs/__tests__/insight-status-generate.test.ts": 1,
"lib/jobs/__tests__/intake-slot-dedup-queue.test.ts": 1,
"lib/jobs/__tests__/integration-backfill-admission.test.ts": 1,
"lib/jobs/__tests__/job-failures-retention.test.ts": 1,
"lib/jobs/__tests__/job-outcome-source-inventory.test.ts": 1,
"lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts": 1,
"lib/jobs/__tests__/mean-consolidation-queue.test.ts": 1,
"lib/jobs/__tests__/measurement-reminder-queue.test.ts": 1,
"lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts": 1,
"lib/jobs/__tests__/nightscout-queues.test.ts": 1,
"lib/jobs/__tests__/period-narrative-warm.test.ts": 1,
"lib/jobs/__tests__/queue-policy-tables.test.ts": 1,
"lib/jobs/__tests__/read-side-effect-queues.test.ts": 1,
"lib/jobs/__tests__/recovery-score-queue.test.ts": 1,
"lib/jobs/__tests__/sleep-timeline-queues.test.ts": 1,
"lib/jobs/__tests__/status-cron-candidates.test.ts": 1,
"lib/jobs/__tests__/step-consolidation-repair-queue.test.ts": 1,
"lib/jobs/__tests__/stress-strain-retention-queue.test.ts": 1,
"lib/jobs/__tests__/subsystem-surface.test.ts": 1,
"lib/jobs/__tests__/tls-pin-monitor-queue.test.ts": 1,
"lib/jobs/__tests__/whoop-queues.test.ts": 1,
"lib/jobs/__tests__/withings-ecg-worker.test.ts": 1,
"lib/jobs/__tests__/withings-queues.test.ts": 1,
"lib/jobs/__tests__/workout-insight-queue.test.ts": 1,
"lib/jobs/reminder/__tests__/integration-cohort-outcomes.test.ts": 1,
@@ -1,7 +1,7 @@
{
"$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.",
"count": 2954,
"total": 10499,
"count": 2924,
"total": 10176,
"entries": {
"__tests__/account-payload-consumer-guard.test.ts": 2,
"__tests__/acting-account-boundary-guard.test.ts": 7,
@@ -2288,59 +2288,34 @@
"lib/integrations/metric-freshness.ts": 1,
"lib/integrations/status.ts": 19,
"lib/intl/__tests__/formatter-cache.test.ts": 1,
"lib/jobs/__tests__/apple-health-import-reconcile.test.ts": 2,
"lib/jobs/__tests__/apple-health-import-send-policy.test.ts": 1,
"lib/jobs/__tests__/coach-memory-refresh-queue.test.ts": 2,
"lib/jobs/__tests__/coach-message-cleanup-queue.test.ts": 1,
"lib/jobs/__tests__/apple-health-import-reconcile.test.ts": 1,
"lib/jobs/__tests__/coach-nudge.test.ts": 3,
"lib/jobs/__tests__/coach-reminder-sweep.test.ts": 1,
"lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts": 1,
"lib/jobs/__tests__/cycle-reminder-queue.test.ts": 2,
"lib/jobs/__tests__/cycle-reminder.test.ts": 2,
"lib/jobs/__tests__/data-arrival-queue.test.ts": 1,
"lib/jobs/__tests__/drain-cumulative-queue.test.ts": 4,
"lib/jobs/__tests__/feedback-aggregator.test.ts": 2,
"lib/jobs/__tests__/fitbit-backfill.test.ts": 1,
"lib/jobs/__tests__/fitbit-queues.test.ts": 2,
"lib/jobs/__tests__/geo-backfill.test.ts": 3,
"lib/jobs/__tests__/google-health-queues.test.ts": 1,
"lib/jobs/__tests__/geo-backfill.test.ts": 1,
"lib/jobs/__tests__/google-health-sleep-repair.test.ts": 1,
"lib/jobs/__tests__/hero-line-json-envelope.test.ts": 1,
"lib/jobs/__tests__/insight-pregenerate-retry-enqueue.test.ts": 1,
"lib/jobs/__tests__/insight-pregenerate.test.ts": 16,
"lib/jobs/__tests__/insight-status-generate.test.ts": 2,
"lib/jobs/__tests__/insight-pregenerate.test.ts": 14,
"lib/jobs/__tests__/intake-auto-skip.test.ts": 3,
"lib/jobs/__tests__/intake-auto-take.test.ts": 1,
"lib/jobs/__tests__/intake-slot-dedup-queue.test.ts": 6,
"lib/jobs/__tests__/job-run-report.test.ts": 1,
"lib/jobs/__tests__/mean-consolidation-queue.test.ts": 2,
"lib/jobs/__tests__/measurement-reminder-queue.test.ts": 2,
"lib/jobs/__tests__/measurement-reminder.test.ts": 6,
"lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts": 2,
"lib/jobs/__tests__/medication-inventory-expire.test.ts": 1,
"lib/jobs/__tests__/medication-low-stock.test.ts": 2,
"lib/jobs/__tests__/medication-reminder-check.test.ts": 2,
"lib/jobs/__tests__/mood-reminder.test.ts": 2,
"lib/jobs/__tests__/nightscout-queues.test.ts": 3,
"lib/jobs/__tests__/period-narrative-warm.test.ts": 1,
"lib/jobs/__tests__/reaction-line-degradation.test.ts": 2,
"lib/jobs/__tests__/recovery-score-queue.test.ts": 2,
"lib/jobs/__tests__/recovery-score.test.ts": 1,
"lib/jobs/__tests__/reminder-phases.test.ts": 1,
"lib/jobs/__tests__/reminder-satisfy.test.ts": 3,
"lib/jobs/__tests__/report-worker-error.test.ts": 1,
"lib/jobs/__tests__/run-job.test.ts": 1,
"lib/jobs/__tests__/sleep-timeline-backfill.test.ts": 1,
"lib/jobs/__tests__/sleep-timeline-queues.test.ts": 2,
"lib/jobs/__tests__/status-cron-candidates.test.ts": 2,
"lib/jobs/__tests__/step-consolidation-repair-queue.test.ts": 1,
"lib/jobs/__tests__/stress-strain-retention-queue.test.ts": 2,
"lib/jobs/__tests__/tls-pin-monitor-queue.test.ts": 2,
"lib/jobs/__tests__/whoop-backfill.test.ts": 1,
"lib/jobs/__tests__/whoop-queues.test.ts": 2,
"lib/jobs/__tests__/whoop-resource-sync.test.ts": 1,
"lib/jobs/__tests__/withings-queues.test.ts": 2,
"lib/jobs/__tests__/workout-insight-queue.test.ts": 1,
"lib/jobs/apple-health-import-worker.ts": 4,
"lib/jobs/coach-message-cleanup.ts": 1,
"lib/jobs/coach-nudge-ai.ts": 3,
@@ -2396,7 +2371,7 @@
"lib/jobs/recovery-score.ts": 1,
"lib/jobs/reminder-phases.ts": 1,
"lib/jobs/reminder-satisfy.ts": 1,
"lib/jobs/reminder-worker.ts": 4,
"lib/jobs/reminder-worker.ts": 3,
"lib/jobs/reminder/__tests__/cleanup-handlers-visibility.test.ts": 1,
"lib/jobs/reminder/__tests__/queue-retention.test.ts": 1,
"lib/jobs/reminder/cleanup-handlers.ts": 7,
@@ -2410,11 +2385,6 @@
"lib/jobs/reminder/oura-sync.ts": 1,
"lib/jobs/reminder/polar-sync.ts": 2,
"lib/jobs/reminder/poll-cohort.ts": 2,
"lib/jobs/reminder/register-integration-sync.ts": 63,
"lib/jobs/reminder/register-maintenance.ts": 114,
"lib/jobs/reminder/register-reminders.ts": 15,
"lib/jobs/reminder/register-rollup.ts": 37,
"lib/jobs/reminder/register-status.ts": 38,
"lib/jobs/reminder/registrar-shared.ts": 2,
"lib/jobs/reminder/strava-sync.ts": 1,
"lib/jobs/reminder/whoop-sync.ts": 3,
-2
View File
@@ -63,8 +63,6 @@ const NO_FLOOR_NEEDED: Readonly<Record<string, string>> = {
"named walks, each must positively match the keyset cursor",
"components/charts/__tests__/touch-action-guard.test.ts":
"named chart wrappers, each must positively declare touch-pan-y",
"__tests__/moodlog-removal-guard.test.ts":
"absence checks over named paths, columns and keys; no matcher to drift",
// Unit tests of runtime code whose name ends in "guard". They read no
// source and sweep nothing.
+41 -1
View File
@@ -197,6 +197,8 @@ export interface ModuleImports {
value: string[];
/** True when every statement is an `export … from` (or an import). */
reexportOnly: boolean;
/** The value imports evaluated when the module loads: `value` without `import("…")`. */
staticValue: string[];
}
/**
@@ -240,6 +242,7 @@ export function moduleImports(sf: ts.SourceFile): ModuleImports {
}
other += 1;
}
const staticValue = [...value];
const visit = (node: ts.Node): void => {
if (
ts.isCallExpression(node) &&
@@ -252,7 +255,7 @@ export function moduleImports(sf: ts.SourceFile): ModuleImports {
ts.forEachChild(node, visit);
};
visit(sf);
return { value, reexportOnly: reexports > 0 && other === 0 };
return { value, staticValue, reexportOnly: reexports > 0 && other === 0 };
}
export interface ImportGraph {
@@ -374,3 +377,40 @@ export function backEdgeCycles(edges: Map<string, string[]>): string[][] {
for (const v of [...edges.keys()].sort()) walk(v);
return cycles;
}
/**
* Every module under `src/` that loading `entries` evaluates: static value
* imports and re-exports, transitively. A dynamic `import("…")` is not
* followed, since it loads only when it runs. Paths are relative to `src/`.
*/
export function staticImportClosure(entries: readonly string[]): Set<string> {
const cache = ts.createModuleResolutionCache(
process.cwd(),
(x) => x,
COMPILER_OPTIONS,
);
const seen = new Set<string>();
const stack = [...entries];
while (stack.length > 0) {
const rel = stack.pop()!;
if (seen.has(rel)) continue;
seen.add(rel);
const abs = join(SRC, rel);
for (const spec of moduleImports(parse(rel, readFileSync(abs, "utf8")))
.staticValue) {
const hit = ts.resolveModuleName(
spec,
abs,
COMPILER_OPTIONS,
ts.sys,
cache,
).resolvedModule;
if (!hit || hit.isExternalLibraryImport) continue;
if (!hit.resolvedFileName.startsWith(`${SRC}/`)) continue;
const target = hit.resolvedFileName.slice(SRC.length + 1);
if (target.startsWith("generated/") || target.endsWith(".d.ts")) continue;
stack.push(target);
}
}
return seen;
}
+117 -188
View File
@@ -11,23 +11,23 @@
* `src/lib/jobs/queue-runtime.ts` declares every queue `short`, `budgeted` or
* `long`. This guard holds the tree to it:
*
* - every queue bound with `createAndWork` is in the table, and every table
* entry is bound;
* - every `boss.send` and every schedule tuple names a queue in the table;
* - every queue the worker registry declares (`jobs/registry/`) is in the
* table, and every table entry is declared;
* - every `boss.send` and every scheduled queue names a queue in the table;
* - a `long` queue has an expiry between the default and pg-boss's 24-hour
* ceiling, every send and schedule for it carries `expireInSeconds` (or an
* identifier the table names as its carrier), it stops on the job's budget
* where the table says (`jobBudget` / `jobDeadline`, or a declared
* one-shot), and it is exclusive per identity where the table says
* (`lockedPass` at the binding, or the named marker);
* ceiling, every send for it carries `expireInSeconds` (or an identifier
* the table names as its carrier) and its schedule's send options do, it
* stops on the job's budget where the table says (`jobBudget` /
* `jobDeadline`, or a declared one-shot), and it is exclusive per identity
* where the table says (the definition's `lock`, or the named marker);
* - a `budgeted` queue stops on the budget and relies on the default expiry;
* - a `short` queue's sends set no expiry of their own, which would mean the
* job is longer than the table claims.
*
* Floors fail the guard if a matcher stops matching, and the "plants" cases
* run the checks over a planted long queue that lacks each protection in turn
* (the mutation proof kept in the suite). Removing `lockedPass` from the
* pr-detection binding, or the expiry from its cron tuple, fails the real
* (the mutation proof kept in the suite). Removing the `lock` from the
* pr-detection definition, or the expiry from its schedule, fails the real
* checks below.
*/
import { describe, expect, it } from "vitest";
@@ -39,19 +39,14 @@ import {
DEFAULT_JOB_EXPIRE_SECONDS,
QUEUE_RUNTIME,
type QueueRuntime,
type SourceSite,
} from "@/lib/jobs/queue-runtime";
import { SCHEDULED_AND_SENT_QUEUES } from "@/lib/jobs/reminder/registrar-shared";
import { HEAVY_QUEUES } from "@/lib/jobs/heavy-lane";
import type { JobDefinition } from "@/lib/jobs/registry/define-job";
import { allJobDefinitions } from "@/lib/jobs/registry/index";
const SRC = join(process.cwd(), "src");
const REGISTRARS = [
"lib/jobs/reminder/register-integration-sync.ts",
"lib/jobs/reminder/register-maintenance.ts",
"lib/jobs/reminder/register-reminders.ts",
"lib/jobs/reminder/register-rollup.ts",
"lib/jobs/reminder/register-status.ts",
];
const DEFINITIONS = allJobDefinitions();
/** pg-boss refuses an expiry of 24 hours or more. */
const MAX_EXPIRE_SECONDS = 24 * 60 * 60;
@@ -120,27 +115,6 @@ interface Site {
text: string;
}
/** Every `createAndWork(boss, QUEUE, …)` in the registrars. */
function bindings(
texts: ReadonlyMap<string, string>,
definitions: Definitions,
): Site[] {
const out: Site[] = [];
for (const file of REGISTRARS) {
const text = code(texts.get(file) ?? "");
for (const m of text.matchAll(
/createAndWork\s*(?:<[^(]*?>)?\(\s*boss\s*,\s*([A-Z][A-Z0-9_]*)/g,
)) {
const open = text.indexOf("(", m.index);
const queue = resolve(definitions, file, m[1]);
if (queue) {
out.push({ file, queue, text: text.slice(open, closeOf(text, open)) });
}
}
}
return out;
}
/** Every `boss.send(QUEUE, …)` / `getGlobalBoss()?.send(QUEUE, …)`. */
function sends(
texts: ReadonlyMap<string, string>,
@@ -162,31 +136,6 @@ function sends(
return out;
}
/** Every `[QUEUE, CRON, options?]` tuple in a registrar's `schedules`. */
function schedules(
texts: ReadonlyMap<string, string>,
definitions: Definitions,
): Site[] {
const out: Site[] = [];
for (const file of REGISTRARS) {
const text = code(texts.get(file) ?? "");
const m = /const\s+schedules\s*:\s*ScheduleEntry\[\]\s*=\s*\[/.exec(text);
if (!m) continue;
const open = m.index + m[0].length - 1;
const body = text.slice(open + 1, closeOf(text, open) - 1);
for (let i = 0; i < body.length; i += 1) {
if (body[i] !== "[") continue;
const end = closeOf(body, i);
const tuple = body.slice(i, end);
const ident = /^\[\s*([A-Z][A-Z0-9_]*)/.exec(tuple)?.[1];
const queue = ident ? resolve(definitions, file, ident) : null;
if (queue) out.push({ file, queue, text: tuple });
i = end - 1;
}
}
return out;
}
/** The text of `fn` (a function or a const) in `file`. */
function bodyOf(
texts: ReadonlyMap<string, string>,
@@ -201,14 +150,6 @@ function bodyOf(
return text.slice(m.index, end < 0 ? undefined : end + 2);
}
function siteText(
texts: ReadonlyMap<string, string>,
site: SourceSite,
binding: Site | undefined,
): string {
return site === "binding" ? (binding?.text ?? "") : bodyOf(texts, site);
}
const STOPS = /\bjob(?:Budget|Deadline)\s*\(/;
/** Every finding for one queue's declaration against the tree. */
@@ -216,13 +157,12 @@ function check(
queue: string,
entry: QueueRuntime,
texts: ReadonlyMap<string, string>,
all: { bindings: Site[]; sends: Site[]; schedules: Site[] },
all: { definitions: readonly JobDefinition[]; sends: Site[] },
): string[] {
const problems: string[] = [];
const binding = all.bindings.find((b) => b.queue === queue);
const carriers = [...all.sends, ...all.schedules].filter(
(s) => s.queue === queue,
);
const definition = all.definitions.find((d) => d.queue === queue);
const sends = all.sends.filter((s) => s.queue === queue);
const scheduledExpiry = definition?.scheduleOptions?.expireInSeconds;
if (entry.runtime === "long") {
if (
!(entry.expireInSeconds > DEFAULT_JOB_EXPIRE_SECONDS) ||
@@ -230,85 +170,86 @@ function check(
) {
problems.push(`${queue}: expiry ${entry.expireInSeconds}s out of range`);
}
for (const site of carriers) {
for (const site of sends) {
const carried =
/\bexpireInSeconds\b/.test(site.text) ||
entry.expiryVia.some((v) => new RegExp(`\\b${v}\\b`).test(site.text));
if (!carried) {
problems.push(
`${queue}: ${site.file} sends or schedules it without an expiry`,
);
problems.push(`${queue}: ${site.file} sends it without an expiry`);
}
}
if (definition?.cron && typeof scheduledExpiry !== "number") {
problems.push(`${queue}: its schedule carries no expiry`);
}
if (typeof entry.stop === "object" && "oneShot" in entry.stop) {
if (entry.stop.oneShot.length < 20) {
problems.push(`${queue}: a one-shot needs its reason`);
}
} else if (!STOPS.test(siteText(texts, entry.stop, binding))) {
} else if (!STOPS.test(bodyOf(texts, entry.stop))) {
problems.push(`${queue}: does not stop on the job's budget`);
}
if (entry.exclusive === "lockedPass") {
if (!/\blockedPass\s*\(/.test(binding?.text ?? "")) {
problems.push(`${queue}: binding holds no lockedPass`);
if (!definition?.lock) {
problems.push(`${queue}: definition holds no lock`);
}
} else if (
!siteText(texts, entry.exclusive.at, binding).includes(
entry.exclusive.marker,
)
!bodyOf(texts, entry.exclusive.at).includes(entry.exclusive.marker)
) {
problems.push(`${queue}: no "${entry.exclusive.marker}" where declared`);
}
} else {
if (entry.runtime === "budgeted") {
if (!STOPS.test(siteText(texts, entry.budget, binding))) {
if (!STOPS.test(bodyOf(texts, entry.budget))) {
problems.push(`${queue}: does not stop on the job's budget`);
}
}
for (const site of carriers) {
for (const site of sends) {
if (/\bexpireInSeconds\b/.test(site.text)) {
problems.push(
`${queue}: ${site.file} sets an expiry on a queue declared ${entry.runtime}`,
);
}
}
if (scheduledExpiry !== undefined) {
problems.push(
`${queue}: its schedule sets an expiry on a queue declared ${entry.runtime}`,
);
}
}
return problems;
}
function scan(texts: ReadonlyMap<string, string>) {
const definitions = definitionsOf(texts);
return {
bindings: bindings(texts, definitions),
sends: sends(texts, definitions),
schedules: schedules(texts, definitions),
};
return { sends: sends(texts, definitionsOf(texts)) };
}
describe("every queue declares its runtime", () => {
const all = scan(source);
const bound = new Set(all.bindings.map((b) => b.queue));
const all = { definitions: DEFINITIONS, ...scan(source) };
const declared = new Set(DEFINITIONS.map((d) => d.queue));
it("finds the bindings, sends and schedules it is meant to police", () => {
// Pinned below the counts on 2026-10-01 (112 / 64 / 74).
expect(all.bindings.length).toBeGreaterThanOrEqual(105);
it("finds the declarations, sends and schedules it is meant to police", () => {
// Pinned below the counts on 2026-10-23 (122 / 64 / 78).
expect(DEFINITIONS.length).toBeGreaterThanOrEqual(115);
expect(all.sends.length).toBeGreaterThanOrEqual(50);
expect(all.schedules.length).toBeGreaterThanOrEqual(70);
expect(DEFINITIONS.filter((d) => d.cron).length).toBeGreaterThanOrEqual(70);
expect(
Object.values(QUEUE_RUNTIME).filter((e) => e.runtime === "long").length,
).toBeGreaterThanOrEqual(15);
});
it("every bound queue is in the table, and every entry is bound", () => {
expect([...bound].filter((q) => !(q in QUEUE_RUNTIME)).sort()).toEqual([]);
it("every declared queue is in the table, and every entry is declared", () => {
expect([...declared].filter((q) => !(q in QUEUE_RUNTIME)).sort()).toEqual(
[],
);
expect(
Object.keys(QUEUE_RUNTIME)
.filter((q) => !bound.has(q))
.filter((q) => !declared.has(q))
.sort(),
).toEqual([]);
});
it("every send and schedule names a queue in the table", () => {
const unknown = [...all.sends, ...all.schedules]
it("every send names a queue in the table", () => {
const unknown = all.sends
.filter((s) => !(s.queue in QUEUE_RUNTIME))
.map((s) => `${s.file} → ${s.queue}`);
expect(unknown).toEqual([]);
@@ -322,51 +263,56 @@ describe("every queue declares its runtime", () => {
});
it("plants: a long queue missing any protection is caught", () => {
const registrar = REGISTRARS[1];
const planted = (binding: string, schedule: string) =>
const file = "lib/jobs/planted.ts";
const texts = (handler: string) =>
new Map([
...source,
[
registrar,
`const PLANT_QUEUE = "planted-long";\n` +
`const schedules: ScheduleEntry[] = [\n ${schedule},\n];\n` +
`async function register(boss) {\n await createAndWork(boss, PLANT_QUEUE, { localConcurrency: 1 }, ${binding});\n}\n`,
],
[file, `export async function plant(jobs) {\n ${handler}\n}\n`],
]);
const entry: QueueRuntime = {
runtime: "long",
expireInSeconds: 3600,
expiryVia: [],
stop: "binding",
stop: { file, fn: "plant" },
exclusive: "lockedPass",
why: "planted",
};
const findings = (binding: string, schedule: string) => {
const texts = planted(binding, schedule);
return check("planted-long", entry, texts, scan(texts));
const planted = (overrides: Partial<JobDefinition>): JobDefinition => ({
queue: "planted-long",
cron: "0 3 * * *",
scheduleOptions: { expireInSeconds: 3600 },
work: { localConcurrency: 1 },
lock: () => "pass",
load: async () => async () => ({ ok: true, did: {} }),
...overrides,
});
const findings = (
handler: string,
overrides: Partial<JobDefinition> = {},
) => {
const t = texts(handler);
return check("planted-long", entry, t, {
definitions: [planted(overrides)],
...scan(t),
});
};
const safe = [
"lockedPass(PLANT_QUEUE, () => WHOLE_PASS, async (jobs) => run(jobBudget(jobs)))",
"[PLANT_QUEUE, CRON, { expireInSeconds: 3600 }]",
] as const;
expect(findings(...safe)).toEqual([]);
expect(findings(safe[0], "[PLANT_QUEUE, CRON]")).toEqual([
`planted-long: ${registrar} sends or schedules it without an expiry`,
const budgeted = "return run(jobBudget(jobs));";
expect(findings(budgeted)).toEqual([]);
expect(findings(budgeted, { scheduleOptions: { retryLimit: 0 } })).toEqual([
"planted-long: its schedule carries no expiry",
]);
expect(
findings("lockedPass(PLANT_QUEUE, () => WHOLE_PASS, run)", safe[1]),
).toEqual(["planted-long: does not stop on the job's budget"]);
expect(findings("async (jobs) => run(jobBudget(jobs))", safe[1])).toEqual([
"planted-long: binding holds no lockedPass",
expect(findings("return run(jobs);")).toEqual([
"planted-long: does not stop on the job's budget",
]);
expect(findings(budgeted, { lock: undefined })).toEqual([
"planted-long: definition holds no lock",
]);
expect(
check(
"planted-long",
{ ...entry, expireInSeconds: 900 },
...((): [ReadonlyMap<string, string>, ReturnType<typeof scan>] => {
const texts = planted(...safe);
return [texts, scan(texts)];
})(),
texts(budgeted),
{ definitions: [planted({})], ...scan(texts(budgeted)) },
),
).toEqual(["planted-long: expiry 900s out of range"]);
});
@@ -374,12 +320,13 @@ describe("every queue declares its runtime", () => {
/**
* Queues a file sends to through a variable (`boss.send(queueName, …)`): every
* known queue name the file spells out, as a constant or a map value, counts
* as sent, because the matcher above cannot tell which one the variable holds.
* known queue the file names, as a literal, a map value or an imported
* `*_QUEUE` constant, counts as sent, because the matcher above cannot tell which one the variable holds.
*/
function dynamicallySent(texts: ReadonlyMap<string, string>): Set<string> {
const definitions = definitionsOf(texts);
const out = new Set<string>();
for (const [, raw] of texts) {
for (const [file, raw] of texts) {
const text = code(raw);
if (
!/\b(?:boss|getGlobalBoss\(\)\??)\s*\.\s*send\s*(?:<[^(]*?>)?\(\s*[a-z]/.test(
@@ -391,13 +338,20 @@ function dynamicallySent(texts: ReadonlyMap<string, string>): Set<string> {
for (const m of text.matchAll(/"([a-z0-9-]+)"/g)) {
if (m[1] in QUEUE_RUNTIME) out.add(m[1]);
}
// A queue named through an imported constant counts the same.
for (const m of text.matchAll(/\b([A-Z][A-Z0-9_]*_QUEUE)\b/g)) {
const queue = resolve(definitions, file, m[1]);
if (queue && queue in QUEUE_RUNTIME) out.add(queue);
}
}
return out;
}
describe("a cron-fed queue polls slowly, a sent one does not", () => {
const all = scan(source);
const scheduled = new Set(all.schedules.map((s) => s.queue));
const scheduled = new Set(
DEFINITIONS.filter((d) => d.cron).map((d) => d.queue),
);
const sent = new Set([
...all.sends.map((s) => s.queue),
...dynamicallySent(source),
@@ -418,26 +372,14 @@ describe("a cron-fed queue polls slowly, a sent one does not", () => {
});
describe("heavy jobs share the lane", () => {
const all = scan(source);
const laned = all.bindings.filter((b) => /\bheavyLane\s*\(/.test(b.text));
const laned = DEFINITIONS.filter((d) => d.lane);
it("binds every heavy queue in the lane, and nothing else", () => {
expect(laned.map((b) => b.queue).sort()).toEqual(
it("puts every heavy queue in the lane, and nothing else", () => {
expect(laned.map((d) => d.queue).sort()).toEqual(
Object.keys(HEAVY_QUEUES).sort(),
);
});
it("holds a lockedPass outside the lane, so an expiry retry finds the run going", () => {
// Outside, a retry pg-boss starts while the expired run still holds the
// lane finds the lock and settles; inside, the lane would defer it and
// run the job a second time once the first is done.
const inverted = laned
.filter((b) => /\blockedPass\s*\(/.test(b.text))
.filter((b) => b.text.indexOf("lockedPass") > b.text.indexOf("heavyLane"))
.map((b) => b.queue);
expect(inverted).toEqual([]);
});
it("declares every heavy queue long, so its expiry, stop and exclusivity are proven", () => {
// A heavy job is the one most likely to run past pg-boss's default
// expiry, and the retry that follows an expiry runs it a second time.
@@ -464,41 +406,28 @@ describe("heavy jobs share the lane", () => {
// A deferred job returns from active to created. Under `short` (one
// queued job per key) and `stately` (one per state) a twin already
// queued would refuse it, and the job would fail instead of waiting.
const policies = policiesOf(source);
expect(policies.size).toBeGreaterThanOrEqual(40);
const refused = Object.keys(HEAVY_QUEUES)
.filter((queue) => /^(?:short|stately)$/.test(policies.get(queue) ?? ""))
.sort();
expect(refused).toEqual([]);
// The planted case: the same check over a table that declares one.
const planted = new Map(source);
const file = REGISTRARS[2];
planted.set(
file,
`const DATA_BACKUP_QUEUE = "data-backup";\nconst queuePolicies: QueuePolicyTable = {\n [DATA_BACKUP_QUEUE]: { policy: "short", reason: "x" },\n};\n`,
const refusing = (definitions: readonly JobDefinition[]) =>
definitions
.filter(
(d) => d.lane && /^(?:short|stately)$/.test(d.policy?.policy ?? ""),
)
.map((d) => d.queue)
.sort();
expect(DEFINITIONS.filter((d) => d.policy).length).toBeGreaterThanOrEqual(
40,
);
expect(policiesOf(planted).get("data-backup")).toBe("short");
expect(refusing(DEFINITIONS)).toEqual([]);
// The planted case: the same check over a heavy queue declared `short`.
expect(
refusing([
{
queue: "data-backup",
policy: { policy: "short", reason: "x" },
work: { localConcurrency: 1 },
lane: true,
load: async () => async () => ({ ok: true, did: {} }),
},
]),
).toEqual(["data-backup"]);
});
});
/** Every `[QUEUE]: { policy: "…" }` in a registrar's `queuePolicies`. */
function policiesOf(texts: ReadonlyMap<string, string>): Map<string, string> {
const definitions = definitionsOf(texts);
const out = new Map<string, string>();
for (const file of REGISTRARS) {
const text = code(texts.get(file) ?? "");
const m = /const\s+queuePolicies\s*:\s*QueuePolicyTable\s*=\s*\{/.exec(
text,
);
if (!m) continue;
const open = m.index + m[0].length - 1;
const body = text.slice(open, closeOf(text, open));
for (const entry of body.matchAll(
/\[\s*([A-Z][A-Z0-9_]*)\s*\]\s*:\s*\{\s*policy\s*:\s*"([a-z]+)"/g,
)) {
const queue = resolve(definitions, file, entry[1]);
if (queue) out.set(queue, entry[2]);
}
}
return out;
}
+5 -7
View File
@@ -18,11 +18,12 @@
*/
import { readFileSync, existsSync } from "node:fs";
import path from "node:path";
import { allJobDefinitions } from "@/lib/jobs/registry/index";
import { describe, expect, it } from "vitest";
import { INTEGRATION_CADENCE } from "@/lib/integrations/sync-verdict";
import { INTEGRATION_DISPLAY_NAMES } from "@/components/settings/integrations/integration-fallback-row";
import { readSource } from "@/__tests__/helpers/source-files";
const ROOT = path.resolve(__dirname, "../..");
@@ -76,12 +77,9 @@ describe("moodLog integration stays removed", () => {
});
it("registers no moodlog queue, cron or handler binding", () => {
const registrar = readSource(
"src/lib/jobs/reminder/register-integration-sync.ts",
"export async function registerIntegrationSyncQueues",
);
expect(registrar).not.toContain("moodlog");
expect(registrar).not.toContain("MoodLog");
const queues = allJobDefinitions().map((job) => job.queue);
expect(queues.length).toBeGreaterThan(100);
expect(queues.filter((queue) => /mood-?log/i.test(queue))).toEqual([]);
});
it("carries no moodlog integration key on the status envelope", () => {