diff --git a/src/__tests__/job-queue-runtime-guard.test.ts b/src/__tests__/job-queue-runtime-guard.test.ts new file mode 100644 index 000000000..5110a5c93 --- /dev/null +++ b/src/__tests__/job-queue-runtime-guard.test.ts @@ -0,0 +1,371 @@ +/** + * Structural guard: every pg-boss queue declares how long its job can run, and + * a long one is safe against pg-boss's expiry. + * + * pg-boss gives a job fifteen minutes unless the send says otherwise, then + * fails it, fires its abort signal and retries it while the handler keeps + * going. The full-history imports ran beside themselves that way (#1066), and + * so could every other job that legitimately needs longer: the nightly status + * passes, the pre-generation, the record scan, the key rotation, the backups. + * + * `src/lib/jobs/queue-runtime.ts` declares every queue `short`, `budgeted` or + * `long`. This guard holds the tree to it: + * + * - every queue bound with `createAndWork` is in the table, and every table + * entry is bound; + * - every `boss.send` and every schedule tuple names a queue in the table; + * - a `long` queue has an expiry between the default and pg-boss's 24-hour + * ceiling, every send and schedule for it carries `expireInSeconds` (or an + * identifier the table names as its carrier), it stops on the job's budget + * where the table says (`jobBudget` / `jobDeadline`, or a declared + * one-shot), and it is exclusive per identity where the table says + * (`lockedPass` at the binding, or the named marker); + * - a `budgeted` queue stops on the budget and relies on the default expiry; + * - a `short` queue's sends set no expiry of their own, which would mean the + * job is longer than the table claims. + * + * Floors fail the guard if a matcher stops matching, and the "plants" cases + * run the checks over a planted long queue that lacks each protection in turn + * (the mutation proof kept in the suite). Removing `lockedPass` from the + * pr-detection binding, or the expiry from its cron tuple, fails the real + * checks below. + */ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { walkSourceFiles } from "./helpers/source-files"; +import { + DEFAULT_JOB_EXPIRE_SECONDS, + QUEUE_RUNTIME, + type QueueRuntime, + type SourceSite, +} from "@/lib/jobs/queue-runtime"; + +const SRC = join(process.cwd(), "src"); +const REGISTRARS = [ + "lib/jobs/reminder/register-integration-sync.ts", + "lib/jobs/reminder/register-maintenance.ts", + "lib/jobs/reminder/register-reminders.ts", + "lib/jobs/reminder/register-rollup.ts", + "lib/jobs/reminder/register-status.ts", +]; +/** pg-boss refuses an expiry of 24 hours or more. */ +const MAX_EXPIRE_SECONDS = 24 * 60 * 60; + +const files = walkSourceFiles(SRC, { floor: 3000 }) + .filter((p) => !p.startsWith("generated/")) + .filter((p) => !p.includes("__tests__")) + .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx")); +const source = new Map( + files.map((f) => [f, readFileSync(join(SRC, f), "utf8")]), +); + +/** Strip comments so documentation that quotes code never matches. */ +function code(text: string): string { + return text + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/(^|[^:])\/\/.*$/gm, "$1"); +} + +/** Index just past the bracket that closes the one at `open`. */ +function closeOf(text: string, open: number): number { + const pair: Record = { "(": ")", "[": "]", "{": "}" }; + const want = pair[text[open]]; + let depth = 1; + let i = open + 1; + while (depth > 0 && i < text.length) { + if (text[i] === text[open]) depth += 1; + else if (text[i] === want) depth -= 1; + i += 1; + } + return i; +} + +type Definitions = Map>; + +/** `const NAME = "queue-name"` definitions, per file. */ +function definitionsOf(texts: ReadonlyMap): Definitions { + const definitions: Definitions = new Map(); + for (const [file, text] of texts) { + for (const m of text.matchAll( + /\bconst\s+([A-Z][A-Z0-9_]*)\s*(?::\s*string\s*)?=\s*"([a-z0-9-]+)"/g, + )) { + const list = definitions.get(m[1]) ?? []; + list.push({ file, name: m[2] }); + definitions.set(m[1], list); + } + } + return definitions; +} + +/** Resolve a queue identifier as seen from `file`, preferring its own. */ +function resolve( + definitions: Definitions, + file: string, + ident: string, +): string | null { + const defs = definitions.get(ident) ?? []; + const own = defs.find((d) => d.file === file); + if (own) return own.name; + const names = new Set(defs.map((d) => d.name)); + return names.size === 1 ? [...names][0] : null; +} + +interface Site { + file: string; + queue: string; + text: string; +} + +/** Every `createAndWork(boss, QUEUE, …)` in the registrars. */ +function bindings( + texts: ReadonlyMap, + definitions: Definitions, +): Site[] { + const out: Site[] = []; + for (const file of REGISTRARS) { + const text = code(texts.get(file) ?? ""); + for (const m of text.matchAll( + /createAndWork\s*(?:<[^(]*?>)?\(\s*boss\s*,\s*([A-Z][A-Z0-9_]*)/g, + )) { + const open = text.indexOf("(", m.index); + const queue = resolve(definitions, file, m[1]); + if (queue) { + out.push({ file, queue, text: text.slice(open, closeOf(text, open)) }); + } + } + } + return out; +} + +/** Every `boss.send(QUEUE, …)` / `getGlobalBoss()?.send(QUEUE, …)`. */ +function sends( + texts: ReadonlyMap, + definitions: Definitions, +): Site[] { + const out: Site[] = []; + for (const [file, raw] of texts) { + const text = code(raw); + for (const m of text.matchAll( + /\b(?:boss|getGlobalBoss\(\)\??)\s*\.\s*send\s*(?:<[^(]*?>)?\(\s*([A-Z][A-Z0-9_]*)/g, + )) { + const open = text.indexOf("(", text.indexOf("send", m.index)); + const queue = resolve(definitions, file, m[1]); + if (queue) { + out.push({ file, queue, text: text.slice(open, closeOf(text, open)) }); + } + } + } + return out; +} + +/** Every `[QUEUE, CRON, options?]` tuple in a registrar's `schedules`. */ +function schedules( + texts: ReadonlyMap, + definitions: Definitions, +): Site[] { + const out: Site[] = []; + for (const file of REGISTRARS) { + const text = code(texts.get(file) ?? ""); + const m = /const\s+schedules\s*:\s*ScheduleEntry\[\]\s*=\s*\[/.exec(text); + if (!m) continue; + const open = m.index + m[0].length - 1; + const body = text.slice(open + 1, closeOf(text, open) - 1); + for (let i = 0; i < body.length; i += 1) { + if (body[i] !== "[") continue; + const end = closeOf(body, i); + const tuple = body.slice(i, end); + const ident = /^\[\s*([A-Z][A-Z0-9_]*)/.exec(tuple)?.[1]; + const queue = ident ? resolve(definitions, file, ident) : null; + if (queue) out.push({ file, queue, text: tuple }); + i = end - 1; + } + } + return out; +} + +/** The text of `fn` (a function or a const) in `file`. */ +function bodyOf( + texts: ReadonlyMap, + site: { file: string; fn: string }, +): string { + const text = code(texts.get(site.file) ?? ""); + const m = new RegExp( + `(?:function\\s+${site.fn}\\b|const\\s+${site.fn}\\b)`, + ).exec(text); + if (!m) return ""; + const end = text.indexOf("\n}", m.index); + return text.slice(m.index, end < 0 ? undefined : end + 2); +} + +function siteText( + texts: ReadonlyMap, + site: SourceSite, + binding: Site | undefined, +): string { + return site === "binding" ? (binding?.text ?? "") : bodyOf(texts, site); +} + +const STOPS = /\bjob(?:Budget|Deadline)\s*\(/; + +/** Every finding for one queue's declaration against the tree. */ +function check( + queue: string, + entry: QueueRuntime, + texts: ReadonlyMap, + all: { bindings: Site[]; sends: Site[]; schedules: Site[] }, +): string[] { + const problems: string[] = []; + const binding = all.bindings.find((b) => b.queue === queue); + const carriers = [...all.sends, ...all.schedules].filter( + (s) => s.queue === queue, + ); + if (entry.runtime === "long") { + if ( + !(entry.expireInSeconds > DEFAULT_JOB_EXPIRE_SECONDS) || + !(entry.expireInSeconds < MAX_EXPIRE_SECONDS) + ) { + problems.push(`${queue}: expiry ${entry.expireInSeconds}s out of range`); + } + for (const site of carriers) { + const carried = + /\bexpireInSeconds\b/.test(site.text) || + entry.expiryVia.some((v) => new RegExp(`\\b${v}\\b`).test(site.text)); + if (!carried) { + problems.push( + `${queue}: ${site.file} sends or schedules it without an expiry`, + ); + } + } + if (typeof entry.stop === "object" && "oneShot" in entry.stop) { + if (entry.stop.oneShot.length < 20) { + problems.push(`${queue}: a one-shot needs its reason`); + } + } else if (!STOPS.test(siteText(texts, entry.stop, binding))) { + problems.push(`${queue}: does not stop on the job's budget`); + } + if (entry.exclusive === "lockedPass") { + if (!/\blockedPass\s*\(/.test(binding?.text ?? "")) { + problems.push(`${queue}: binding holds no lockedPass`); + } + } else if ( + !siteText(texts, entry.exclusive.at, binding).includes( + entry.exclusive.marker, + ) + ) { + problems.push(`${queue}: no "${entry.exclusive.marker}" where declared`); + } + } else { + if (entry.runtime === "budgeted") { + if (!STOPS.test(siteText(texts, entry.budget, binding))) { + problems.push(`${queue}: does not stop on the job's budget`); + } + } + for (const site of carriers) { + if (/\bexpireInSeconds\b/.test(site.text)) { + problems.push( + `${queue}: ${site.file} sets an expiry on a queue declared ${entry.runtime}`, + ); + } + } + } + return problems; +} + +function scan(texts: ReadonlyMap) { + const definitions = definitionsOf(texts); + return { + bindings: bindings(texts, definitions), + sends: sends(texts, definitions), + schedules: schedules(texts, definitions), + }; +} + +describe("every queue declares its runtime", () => { + const all = scan(source); + const bound = new Set(all.bindings.map((b) => b.queue)); + + it("finds the bindings, sends and schedules it is meant to police", () => { + // Pinned below the counts on 2026-10-01 (112 / 64 / 74). + expect(all.bindings.length).toBeGreaterThanOrEqual(105); + expect(all.sends.length).toBeGreaterThanOrEqual(50); + expect(all.schedules.length).toBeGreaterThanOrEqual(70); + expect( + Object.values(QUEUE_RUNTIME).filter((e) => e.runtime === "long").length, + ).toBeGreaterThanOrEqual(15); + }); + + it("every bound queue is in the table, and every entry is bound", () => { + expect([...bound].filter((q) => !(q in QUEUE_RUNTIME)).sort()).toEqual([]); + expect( + Object.keys(QUEUE_RUNTIME) + .filter((q) => !bound.has(q)) + .sort(), + ).toEqual([]); + }); + + it("every send and schedule names a queue in the table", () => { + const unknown = [...all.sends, ...all.schedules] + .filter((s) => !(s.queue in QUEUE_RUNTIME)) + .map((s) => `${s.file} → ${s.queue}`); + expect(unknown).toEqual([]); + }); + + it("every queue meets its declaration", () => { + const problems = Object.entries(QUEUE_RUNTIME).flatMap(([queue, entry]) => + check(queue, entry, source, all), + ); + expect(problems).toEqual([]); + }); + + it("plants: a long queue missing any protection is caught", () => { + const registrar = REGISTRARS[1]; + const planted = (binding: string, schedule: string) => + new Map([ + ...source, + [ + registrar, + `const PLANT_QUEUE = "planted-long";\n` + + `const schedules: ScheduleEntry[] = [\n ${schedule},\n];\n` + + `async function register(boss) {\n await createAndWork(boss, PLANT_QUEUE, { localConcurrency: 1 }, ${binding});\n}\n`, + ], + ]); + const entry: QueueRuntime = { + runtime: "long", + expireInSeconds: 3600, + expiryVia: [], + stop: "binding", + exclusive: "lockedPass", + why: "planted", + }; + const findings = (binding: string, schedule: string) => { + const texts = planted(binding, schedule); + return check("planted-long", entry, texts, scan(texts)); + }; + const safe = [ + "lockedPass(PLANT_QUEUE, () => WHOLE_PASS, async (jobs) => run(jobBudget(jobs)))", + "[PLANT_QUEUE, CRON, { expireInSeconds: 3600 }]", + ] as const; + expect(findings(...safe)).toEqual([]); + expect(findings(safe[0], "[PLANT_QUEUE, CRON]")).toEqual([ + `planted-long: ${registrar} sends or schedules it without an expiry`, + ]); + expect( + findings("lockedPass(PLANT_QUEUE, () => WHOLE_PASS, run)", safe[1]), + ).toEqual(["planted-long: does not stop on the job's budget"]); + expect(findings("async (jobs) => run(jobBudget(jobs))", safe[1])).toEqual([ + "planted-long: binding holds no lockedPass", + ]); + expect( + check( + "planted-long", + { ...entry, expireInSeconds: 900 }, + ...((): [ReadonlyMap, ReturnType] => { + const texts = planted(...safe); + return [texts, scan(texts)]; + })(), + ), + ).toEqual(["planted-long: expiry 900s out of range"]); + }); +}); diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts new file mode 100644 index 000000000..caf376c3d --- /dev/null +++ b/src/lib/jobs/queue-runtime.ts @@ -0,0 +1,398 @@ +/** + * How long one job of each queue can run, and what keeps a long one safe. + * + * pg-boss gives every job an expiry, fifteen minutes unless the send says + * otherwise. Past it pg-boss fails the job, fires its abort signal and retries + * it, and it does not stop the handler, so a job that legitimately needs longer + * ends up running twice beside itself. No queue policy prevents that: the + * policies constrain job rows, and an expired row is no longer active. Every + * queue bound with `createAndWork` is therefore declared here as one of: + * + * - `short`: a job cannot come near fifteen minutes (one account's bounded + * window, one document, a forwarder, a bounded delete). It relies on the + * default expiry, and the reason says why that is enough. + * - `budgeted`: a pass over a long history that stops itself at three + * quarters of the default expiry (`jobBudget`) and hands the rest to its + * next run, so it ends before fifteen minutes by construction. + * - `long`: a job that may run for hours. It carries an explicit + * `expireInSeconds` on every send and schedule, stops itself before that + * expiry (`jobBudget` / `jobDeadline`), and holds something per identity + * that keeps a second run from working beside it: the lock `lockedPass` + * takes, a lock of its own, or a row it claims. The one exception is a + * one-shot job with `retryLimit: 0`, which pg-boss never retries and which + * cannot stop half-way without losing its input. + * + * `src/__tests__/job-queue-runtime-guard.test.ts` holds every binding, send + * and schedule in the tree to this table. + */ +import { INTEGRATION_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/integration-backfill-admission"; +import { NIGHTLY_INSIGHT_PASS_EXPIRE_SECONDS } from "@/lib/jobs/reminder/registrar-shared"; +import { INSIGHT_PREGENERATE_EXPIRE_SECONDS } from "@/lib/jobs/insight-pregenerate-shared"; +import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "@/lib/jobs/period-narrative-shared"; +import { PR_DETECTION_EXPIRE_SECONDS } from "@/lib/jobs/pr-detection"; +import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key-rotate"; +import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill"; +import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy"; +import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup"; +import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore"; +import { APPLE_HEALTH_IMPORT_SEND_OPTIONS } from "@/lib/jobs/apple-health-import-worker"; +import { MEDICATION_INTAKE_IMPORT_SEND_OPTIONS } from "@/lib/jobs/medication-intake-import"; + +/** pg-boss's default job expiry, in seconds. */ +export const DEFAULT_JOB_EXPIRE_SECONDS = 15 * 60; + +/** Where a check lives: `binding` is the `createAndWork` call itself. */ +export type SourceSite = "binding" | { file: string; fn: string }; + +export type QueueRuntime = + | { runtime: "short"; why: string } + | { runtime: "budgeted"; budget: SourceSite; why: string } + | { + runtime: "long"; + expireInSeconds: number; + /** + * Identifiers that carry the expiry into a send or a schedule. A site + * that names none of them must set `expireInSeconds` itself. + */ + expiryVia: readonly string[]; + /** Where the handler stops on the job's budget, or `oneShot`. */ + stop: SourceSite | { oneShot: string }; + /** + * What keeps a second run from working beside this one: `lockedPass` + * at the binding, or a marker in the named function. + */ + exclusive: "lockedPass" | { marker: string; at: SourceSite }; + why: string; + }; + +const short = (why: string): QueueRuntime => ({ runtime: "short", why }); + +const nightlyStatusPass = (fn: string): QueueRuntime => ({ + runtime: "long", + expireInSeconds: NIGHTLY_INSIGHT_PASS_EXPIRE_SECONDS, + expiryVia: ["nightlyInsightPassOptions"], + stop: { file: "lib/jobs/reminder/insights-handlers.ts", fn }, + exclusive: "lockedPass", + why: "Nightly status pass: one provider call per candidate account, serially, over the whole instance.", +}); + +const cohortSync = (provider: string): QueueRuntime => + short( + `${provider} sync: a recent window per connection (incremental since the last sync); a webhook job is one connection.`, + ); + +const oauthStateCleanup = short("One DELETE of expired OAuth state rows."); + +const forwarder = short( + "Forwards each job to integration-backfill-admission; the import itself runs there.", +); + +export const QUEUE_RUNTIME: Readonly> = { + // ── Integration sync ─────────────────────────────────────────────── + "integration-backfill-admission": { + runtime: "long", + expireInSeconds: INTEGRATION_BACKFILL_EXPIRE_SECONDS, + expiryVia: ["integrationBackfillAdmissionSendOptions"], + stop: { + file: "lib/jobs/integration-backfill-drain.ts", + fn: "drainIntegrationBackfillAdmission", + }, + exclusive: { + marker: "withIntegrationBackfillLock", + at: { + file: "lib/jobs/integration-backfill-drain.ts", + fn: "drainIntegrationBackfillAdmission", + }, + }, + why: "Full-history provider imports: thousands of pages on a dense account.", + }, + "withings-fallback-sync": cohortSync("Withings"), + "withings-activity-sync": cohortSync("Withings activity"), + "withings-sleep-sync": cohortSync("Withings sleep"), + "withings-ecg-sync": cohortSync("Withings ECG"), + "withings-oauth-state-cleanup": oauthStateCleanup, + "whoop-recovery-sync": cohortSync("WHOOP recovery"), + "whoop-sleep-sync": cohortSync("WHOOP sleep"), + "whoop-workout-sync": cohortSync("WHOOP workout"), + "whoop-cycle-sync": cohortSync("WHOOP cycle"), + "whoop-backfill": forwarder, + "whoop-oauth-state-cleanup": oauthStateCleanup, + "oidc-native-handoff-cleanup": short("One DELETE of expired handoff rows."), + "fitbit-sync": cohortSync("Fitbit"), + "fitbit-backfill": forwarder, + "fitbit-sleep-repair": forwarder, + "fitbit-oauth-state-cleanup": oauthStateCleanup, + "google-health-sync": cohortSync("Google Health"), + "google-health-backfill": forwarder, + "google-health-sleep-repair": forwarder, + "google-health-oauth-state-cleanup": oauthStateCleanup, + "sleep-timeline-backfill": forwarder, + "lab-biomarker-backfill": forwarder, + "nightscout-sync": cohortSync("Nightscout"), + "polar-sync": cohortSync("Polar"), + "oura-sync": cohortSync("Oura"), + "strava-sync": cohortSync("Strava"), + "strava-backfill": forwarder, + + // ── Status and insights ──────────────────────────────────────────── + "insights-general-status": nightlyStatusPass("handleGeneralStatusGenerate"), + "insights-blood-pressure-status": nightlyStatusPass( + "handleBloodPressureStatusGenerate", + ), + "insights-weight-status": nightlyStatusPass("handleWeightStatusGenerate"), + "insights-pulse-status": nightlyStatusPass("handlePulseStatusGenerate"), + "insights-bmi-status": nightlyStatusPass("handleBmiStatusGenerate"), + "insights-mood-status": nightlyStatusPass("handleMoodStatusGenerate"), + "insights-medication-compliance-status": nightlyStatusPass( + "handleMedicationComplianceStatusGenerate", + ), + "insight-pregenerate": { + runtime: "long", + expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS, + expiryVia: ["INSIGHT_PREGENERATE_EXPIRE_SECONDS"], + stop: { + file: "lib/jobs/reminder/insights-handlers.ts", + fn: "handleInsightPregenerateJob", + }, + exclusive: "lockedPass", + why: "Up to 200 accounts, a comprehensive generation each (a provider call bounded by the account's timeout, up to ten minutes) plus the status warm.", + }, + "morning-digest-refresh": short( + "One account, one comprehensive generation, bounded by the provider timeout.", + ), + "insight-status-generate": short( + "One account, one metric, one provider call.", + ), + "data-arrival": short("One arrival's reactions."), + "reaction-line-generate": short("One provider call for one reaction line."), + "workout-insight-generate": short("One workout's insight."), + "recovery-score-compute": short( + "Arithmetic over a recent window, capped at 500 accounts; no provider call.", + ), + "stress-score-compute": short( + "Arithmetic over a recent window, capped at 500 accounts; no provider call.", + ), + "strain-score-compute": short( + "Arithmetic over a recent window, capped at 500 accounts; no provider call.", + ), + "period-narrative-warm": { + runtime: "long", + expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS, + expiryVia: ["PERIOD_NARRATIVE_EXPIRE_SECONDS"], + stop: "binding", + exclusive: "lockedPass", + why: "On a boundary night up to 200 accounts with a provider call per period each; a single-account warm is short.", + }, + "coach-memory-refresh": short("One conversation's summary and facts."), + "coach-nudge": short( + "Light deterministic checks per account; the provider part is capped at 25 calls and 90 seconds a tick.", + ), + "coach-reminder-sweep": short("Batches of 200 to 500 rows, no provider."), + "coach-plan-review": short("A batch of 100 plans, folded in SQL."), + "medication-low-stock": short("A light inventory check per account."), + "daily-briefing": short("A light cohort scan every fifteen minutes."), + + // ── Rollups ──────────────────────────────────────────────────────── + "rollup-recompute": short("One bucket."), + "rollup-full-backfill": short( + "One account's rollup tiers: four aggregate statements, each held to the 60-second statement timeout.", + ), + "mood-rollup-recompute": short("One bucket."), + "mood-rollup-full-backfill": short( + "One account's mood entries, a few a day.", + ), + "medication-compliance-full-backfill": short( + "One account's 90-day compliance window.", + ), + "step-consolidation": short( + "One account's legacy step rows, one transaction per day.", + ), + "step-consolidation-repair": short( + "One account's tombstoned step rows inside the retention window.", + ), + "cumulative-pr-rederive": short( + "One account's record detection; measured at 20 seconds for 1.25 million rows.", + ), + "mean-consolidation": { + runtime: "budgeted", + budget: "binding", + why: "One account's whole history, a day at a time; stops at three quarters of the default expiry and resumes.", + }, + "dense-intraday-retention": { + runtime: "budgeted", + budget: "binding", + why: "A dense account's history, a day at a time; stops at three quarters of the default expiry and resumes.", + }, + "dense-intraday-hourly-rebuild": { + runtime: "budgeted", + budget: "binding", + why: "A dense account's hourly rows, a day at a time; stops at three quarters of the default expiry and resumes.", + }, + "drain-per-sample-cumulative": { + runtime: "budgeted", + budget: "binding", + why: "Every account's per-sample drains; stops at three quarters of the default expiry and hands the rest to a continuation job.", + }, + + // ── Reminders ────────────────────────────────────────────────────── + "medication-reminder-check": short("The due slots of the current minute."), + "mood-reminder-check": short("Accounts whose local reminder hour is now."), + "cycle-reminder-check": short("Accounts whose local reminder hour is now."), + "measurement-reminder-check": short( + "Accounts whose local reminder hour is now.", + ), + "reminder-satisfy": short("One account's open reminders."), + + // ── Maintenance ──────────────────────────────────────────────────── + "data-backup": { + runtime: "long", + expireInSeconds: DATA_BACKUP_SEND_OPTIONS.expireInSeconds, + expiryVia: ["DATA_BACKUP_SEND_OPTIONS"], + stop: { + file: "lib/jobs/reminder/backup-handlers.ts", + fn: "handleDataBackup", + }, + exclusive: "lockedPass", + why: "Serialises every account's whole record.", + }, + "data-backup-offhost": { + runtime: "long", + expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS, + expiryVia: ["OFFHOST_BACKUP_EXPIRE_SECONDS"], + stop: { + file: "lib/jobs/reminder/backup-handlers.ts", + fn: "handleOffhostBackup", + }, + exclusive: "lockedPass", + why: "Uploads every account's record off-host.", + }, + "data-restore-drill": short("Fetches and stream-parses one off-host object."), + "offhost-backup-purge": short("At most 200 delete requests per run."), + "backup-restore": { + runtime: "long", + expireInSeconds: BACKUP_RESTORE_EXPIRE_SECONDS, + expiryVia: ["BACKUP_RESTORE_SEND_OPTIONS", "BACKUP_RESTORE_EXPIRE_SECONDS"], + stop: { file: "lib/jobs/backup-restore.ts", fn: "handleBackupRestore" }, + exclusive: { + marker: "restore_claimed", + at: { file: "lib/jobs/backup-restore.ts", fn: "runBackupRestoreJob" }, + }, + why: "Restores a whole account in one transaction.", + }, + "host-metric-sample": short("One host sample."), + "feedback-aggregator": short("One aggregate over the feedback table."), + "geo-backfill": short("At most 500 audit rows per pass."), + "geolite2-fetch": short("One database download."), + "tls-pin-monitor": short("One TLS handshake."), + "mood-reminder-cleanup": short("One retention DELETE."), + "push-attempt-cleanup": short("One retention DELETE."), + "arrival-reaction-cleanup": short("One retention DELETE."), + "rate-limit-cleanup": short("One retention DELETE."), + "idempotency-cleanup": short("One retention DELETE."), + "step-up-elevation-cleanup": short("One retention DELETE."), + "coach-message-cleanup": short("One retention DELETE."), + "mcp-token-cleanup": short("One retention DELETE."), + "audit-log-cleanup": short("At most 40 batches of 5 000 rows per leg."), + "measurement-tombstone-cleanup": short( + "At most 40 batches of 5 000 rows per leg.", + ), + "document-tombstone-purge": short("One deleteMany."), + "document-summary-reaper": short("One updateMany."), + "cycle-prediction-refresh": short( + "Arithmetic per eligible account, no provider call.", + ), + "mood-prognosis-refresh": short( + "Arithmetic per eligible account, no provider call.", + ), + "achievement-unlock-sweep": short( + "A bounded achievement read per account, no provider call.", + ), + "pr-detection": { + runtime: "long", + expireInSeconds: PR_DETECTION_EXPIRE_SECONDS, + expiryVia: ["PR_DETECTION_EXPIRE_SECONDS"], + stop: { + file: "lib/jobs/reminder/ops-handlers.ts", + fn: "handlePrDetection", + }, + exclusive: "lockedPass", + why: "The fallback cron scans every account's all-time history every thirty minutes; an ingest job is one account.", + }, + "medication-inventory-expire": short("One updateMany."), + "intake-auto-skip": short("One updateMany."), + "apple-health-import-v2": { + runtime: "long", + expireInSeconds: APPLE_HEALTH_IMPORT_SEND_OPTIONS.expireInSeconds, + expiryVia: ["APPLE_HEALTH_IMPORT_SEND_OPTIONS"], + stop: { + oneShot: + "The run consumes and unlinks its staged upload, so it can neither stop half-way nor be retried; the send sets retryLimit 0.", + }, + exclusive: { + marker: "retryLimit: 0", + at: { + file: "lib/jobs/apple-health-import-worker.ts", + fn: "APPLE_HEALTH_IMPORT_SEND_OPTIONS", + }, + }, + why: "Parses an export that can be several gigabytes.", + }, + "apple-health-import": short( + "The legacy queue: moves each job onto apple-health-import-v2.", + ), + "apple-health-import-reconcile": short("One updateMany."), + "medication-intake-import": { + runtime: "long", + expireInSeconds: MEDICATION_INTAKE_IMPORT_SEND_OPTIONS.expireInSeconds, + expiryVia: ["MEDICATION_INTAKE_IMPORT_SEND_OPTIONS"], + stop: { + file: "lib/jobs/medication-intake-import.ts", + fn: "handleMedicationIntakeImport", + }, + exclusive: { + marker: "FOR UPDATE", + at: { + file: "lib/jobs/medication-intake-import.ts", + fn: "processNextChunk", + }, + }, + why: "Imports a dose-history file chunk by chunk; its length follows the file.", + }, + "intake-slot-dedup": short("One account's duplicate dose slots."), + "note-encryption-backfill": short( + "Pages of 200 rows of one account's plaintext notes, light per row.", + ), + "med-notes-encryption-backfill": short( + "Pages of 200 rows of one account's plaintext notes, light per row.", + ), + "free-text-encryption-backfill": short( + "Pages of 200 rows of one account's plaintext fields, light per row.", + ), + "encryption-key-rotate": { + runtime: "long", + expireInSeconds: ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS, + expiryVia: ["ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS"], + stop: { + file: "lib/jobs/encryption-key-rotate.ts", + fn: "handleEncryptionKeyRotate", + }, + exclusive: "lockedPass", + why: "Re-encrypts every encrypted column on the instance.", + }, + "document-content-index-backfill": { + runtime: "long", + expireInSeconds: CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS, + expiryVia: ["CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS"], + stop: "binding", + exclusive: "lockedPass", + why: "Up to 200 documents of one account, a provider transcription each.", + }, + "document-index": short("One document."), + "document-thumbnail": short("One document."), + "document-summary": short("One document."), + "document-summary-catchup": short("Enqueues at most 200 documents."), + "document-thumbnail-backfill": short("Enqueues at most 1 000 documents."), + "environment-fetch": short( + "A fan-out of sends, or one account's bounded archive range.", + ), +};