From 119af56f6ffea5f4baa13a51644fc7b6dd0ce693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Mon, 27 Jul 2026 22:41:02 +0200 Subject: [PATCH 1/6] feat(jobs): derive job completion from a returned outcome MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pg-boss handler signalled success by falling off the end of a function. Every binding in this tree resolved to undefined and nothing read the value, so a handler that caught an error, logged a warning and returned was indistinguishable from one that finished its work: pg-boss marked both completed and the failed-job readers saw nothing. JobOutcome makes the completion a value the handler has to construct. runJob turns ok:false into a reported, rethrown failure so the queue's retry policy applies. createAndWork is the one binding path; its handler type is (jobs) => Promise, so a handler that returns nothing no longer compiles. The type cannot make a handler honest — jobDone() after a swallow still compiles. It converts omission into commission, which is the smaller and far more reviewable class. --- src/lib/jobs/job-outcome.ts | 74 +++++++++++++++++++ src/lib/jobs/reminder/registrar-shared.ts | 45 +++++++++++- src/lib/jobs/run-job.ts | 88 +++++++++++++++++++++++ 3 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 src/lib/jobs/job-outcome.ts create mode 100644 src/lib/jobs/run-job.ts diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts new file mode 100644 index 000000000..d90b90e05 --- /dev/null +++ b/src/lib/jobs/job-outcome.ts @@ -0,0 +1,74 @@ +/** + * The value a background-job handler returns to say what it did. + * + * Before this type existed, a pg-boss handler signalled success by falling + * off the end of a function. Every one of the queue bindings in this tree + * resolved to `undefined`, and none of them was read anywhere, so a handler + * that caught an error, logged a warning and returned was indistinguishable + * from one that finished its work. pg-boss marked both `completed`; the + * operator surfaces that read `pgboss.job` saw nothing; the nightly pass + * looked healthy for as long as it kept failing. + * + * A returned `JobOutcome` closes that specific gap: completion is now + * derived from a value the handler had to construct, and `runJob` turns + * `ok: false` into a failed pg-boss job. What the type CANNOT do is make a + * handler honest — `jobDone()` after swallowing an exception compiles and + * passes every check in this repository. It converts omission (forgetting to + * signal a failure) into commission (claiming a success that did not + * happen), which is a smaller and much more reviewable class of defect. + * + * `did` is the small pinned-shape bag of facts about the run — counts, + * identifiers, booleans. It rides into the failure report and, once the + * failure ledger lands, into the operator surface. Keep it to scalars that + * a dashboard can key on; it is not a place for payload data or free text. + */ + +/** A single fact about a run. Scalars only — never payload data. */ +export type JobFact = string | number | boolean; + +/** The pinned-shape bag of facts a handler reports about its run. */ +export type JobFacts = Readonly>; + +/** + * What a handler did. `ok: true` means the work named by the queue actually + * happened; `ok: false` means it did not, and `runJob` will fail the pg-boss + * job so the failure reaches the retry policy and the operator surfaces. + * + * A run that legitimately had nothing to do is `ok: true` with a zero count + * in `did` — absence of work is not failure. A run that could not do its + * work (provider down, query timed out, key missing) is `ok: false` even + * when the handler previously chose to carry on. + */ +export type JobOutcome = + | { + readonly ok: true; + readonly did: JobFacts; + } + | { + readonly ok: false; + /** Short, stable, greppable. `"nightscout fetch failed"`, not a stack. */ + readonly reason: string; + /** The thrown value, when there was one. Preserved for the stack. */ + readonly cause?: unknown; + /** Whatever the run managed before it failed. */ + readonly did?: JobFacts; + }; + +/** The run did its work. Pass the counts that make the run auditable. */ +export function jobDone(did: JobFacts = {}): JobOutcome { + return { ok: true, did }; +} + +/** + * The run did not do its work. `runJob` reports this and rethrows, so the + * pg-boss job fails and the queue's retry policy applies — check that policy + * when converting a handler that used to swallow, because a deterministic + * failure now retries instead of passing quietly. + */ +export function jobFailed( + reason: string, + cause?: unknown, + did?: JobFacts, +): JobOutcome { + return { ok: false, reason, cause, did }; +} diff --git a/src/lib/jobs/reminder/registrar-shared.ts b/src/lib/jobs/reminder/registrar-shared.ts index ceb84ea2b..178c580e6 100644 --- a/src/lib/jobs/reminder/registrar-shared.ts +++ b/src/lib/jobs/reminder/registrar-shared.ts @@ -16,13 +16,18 @@ * assert a single aggregate `allQueues` (defence in depth: the per-registrar * arrays AND the boot-level union both have to agree). */ -import { PgBoss } from "pg-boss"; +import { PgBoss, type WorkOptions } from "pg-boss"; import { prisma } from "@/lib/db"; import { workerLog } from "./shared"; import { withBackgroundEvent } from "@/lib/logging/background"; import { annotate } from "@/lib/logging/context"; +import { + runJob, + type JobHandler, + type JobWithMetadataHandler, +} from "@/lib/jobs/run-job"; /** * Cron schedule tuple: `[queueName, cronExpression, sendOptions?]`. The @@ -198,3 +203,41 @@ export async function createAndSchedule( ); } } + +/** + * Bind a handler to a queue. The one legal way to reach `boss.work` in this + * tree — the `healthlog/job-handler-outcome` lint rule refuses a bare + * `boss.work(` anywhere else. + * + * The point of routing every binding through here is the handler type: + * `JobHandler` resolves to `Promise`, so a handler cannot be + * bound unless it returns a value declaring what it did. `runJob` then fails + * the pg-boss job on `ok: false` instead of completing it. Falling off the + * end of a handler, which used to read as success, is now a type error. + * + * Argument order matches `boss.work(queue, options, handler)` so a binding + * reads the same as it did before, with `boss` moved to the front. + */ +export async function createAndWork( + boss: PgBoss, + queue: string, + options: WorkOptions & { includeMetadata: true }, + handler: JobWithMetadataHandler, +): Promise; +export async function createAndWork( + boss: PgBoss, + queue: string, + options: WorkOptions, + handler: JobHandler, +): Promise; +export async function createAndWork( + boss: PgBoss, + queue: string, + options: WorkOptions, + handler: JobHandler | JobWithMetadataHandler, +): Promise { + // `JobWithMetadata` extends `Job`, so the metadata handler accepts + // everything the plain one does; the overloads above are what keeps the + // call sites honest about which shape they asked pg-boss for. + await boss.work(queue, options, runJob(queue, handler as JobHandler)); +} diff --git a/src/lib/jobs/run-job.ts b/src/lib/jobs/run-job.ts new file mode 100644 index 000000000..27fcd268a --- /dev/null +++ b/src/lib/jobs/run-job.ts @@ -0,0 +1,88 @@ +/** + * The wrapper that turns a returned `JobOutcome` into a pg-boss result. + * + * `boss.work(queue, opts, handler)` completes a job when the handler's + * promise resolves, whatever it resolves to. Wrapping every handler in + * `runJob` moves that decision from "the function returned" to "the function + * said it succeeded": an `ok: false` outcome is reported and rethrown, so + * pg-boss records a failed job, the queue's retry policy applies, and the + * failed-job readers in `job-failures.ts` surface it to the operator. + * + * This changes runtime behaviour, not only visibility. Handlers that used to + * catch, warn and return now fail their job. For a queue whose failure mode + * is deterministic — a DELETE that times out will time out again — retrying + * is wasted load, so the retry policy of every converted queue has to be a + * decision rather than a default. + * + * The wrapper deliberately does not open a wide event of its own. Handlers + * own their `withBackgroundEvent` scope and annotate inside it; a second + * event here would double every worker line for no added fact. + */ +import type { Job, JobWithMetadata } from "pg-boss"; + +import type { JobFact, JobFacts, JobOutcome } from "./job-outcome"; +import { reportWorkerError } from "./report-worker-error"; + +/** A converted handler: it says what it did instead of just finishing. */ +export type JobHandler = (jobs: Job[]) => Promise; + +/** The same, for a queue bound with `includeMetadata: true`. */ +export type JobWithMetadataHandler = ( + jobs: JobWithMetadata[], +) => Promise; + +/** + * The error a failed outcome becomes. Carries the queue and the handler's + * own reason so the pg-boss `output` column, the GlitchTip event and the + * container log all name the same thing. + */ +export class JobFailure extends Error { + readonly queue: string; + readonly reason: string; + + constructor(queue: string, reason: string, cause?: unknown) { + super(`${queue}: ${reason}`, cause === undefined ? undefined : { cause }); + this.name = "JobFailure"; + this.queue = queue; + this.reason = reason; + } +} + +function factMeta(did: JobFacts | undefined): Record { + const meta: Record = {}; + for (const [key, value] of Object.entries(did ?? {})) meta[key] = value; + return meta; +} + +/** + * Wrap a handler for `boss.work`. Resolves on `ok: true`; reports and + * rethrows on `ok: false`. + * + * A handler that throws is left alone — the throw already fails the job, and + * re-wrapping it would only bury the original stack. `runJob` exists for the + * case the throw never happens. + */ +export function runJob( + queue: string, + handler: (jobs: J[]) => Promise, +): (jobs: J[]) => Promise { + return async (jobs: J[]): Promise => { + const outcome = await handler(jobs); + if (outcome.ok) return; + + const failure = new JobFailure(queue, outcome.reason, outcome.cause); + const causeMessage = + outcome.cause instanceof Error + ? outcome.cause.message + : typeof outcome.cause === "string" + ? outcome.cause + : undefined; + + await reportWorkerError(queue, failure, { + ...factMeta(outcome.did), + ...(causeMessage === undefined ? {} : { cause: causeMessage }), + }); + + throw failure; + }; +} From 67aa3338dae0adf1de9c42c4d7cc70212ed045a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Mon, 27 Jul 2026 22:56:59 +0200 Subject: [PATCH 2/6] feat(jobs): return an outcome from every named queue handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sixty-two handler functions across twenty-four modules now resolve to a JobOutcome instead of undefined. The counts they already wrote to their wide event ride out in `did`, so a run that deleted nothing and a run that could not run are no longer the same shape. Three rules decided each conversion: - a catch that swallowed becomes jobFailed, keeping its warning line; - a catch that already rethrew keeps its throw, because the throw already fails the job and rewrapping it buries the stack; - a fan-out pass is judged on the pass. One account's revoked grant belongs on that account's integration ledger, not in a queue-wide failure that retries the whole cohort, so per-leg failures ride out as a users_failed count — visible for the first time. The retention cleanups are the behaviour change worth naming: every one of them used to warn and return, so a purge that had not run for weeks looked like a purge with nothing to do. They fail now. Their registrar sets retryLimit accordingly in the commit that converts the bindings. JobFailure carries the thrown error's stack; without it every one of the 101 queues would report the same runJob frame. --- src/lib/jobs/apple-health-import-worker.ts | 34 ++++- src/lib/jobs/data-arrival.ts | 13 +- src/lib/jobs/document-purge.ts | 11 +- src/lib/jobs/encryption-key-rotate.ts | 14 +- src/lib/jobs/environment-fetch.ts | 18 ++- src/lib/jobs/job-outcome.ts | 7 + src/lib/jobs/medication-intake-import.ts | 15 +- src/lib/jobs/reaction-line.ts | 13 +- src/lib/jobs/reminder/backup-handlers.ts | 46 +++++- src/lib/jobs/reminder/cleanup-handlers.ts | 138 +++++++++++++----- src/lib/jobs/reminder/fitbit-sync.ts | 37 ++++- src/lib/jobs/reminder/google-health-sync.ts | 35 ++++- src/lib/jobs/reminder/insights-handlers.ts | 88 ++++++++--- .../jobs/reminder/medication-maintenance.ts | 19 ++- .../reminder/medication-reminder-check.ts | 26 +++- src/lib/jobs/reminder/mood-cycle-checks.ts | 57 +++++++- src/lib/jobs/reminder/ops-handlers.ts | 97 ++++++++---- src/lib/jobs/reminder/poll-cohort.ts | 27 +++- src/lib/jobs/reminder/register-reminders.ts | 29 +++- src/lib/jobs/reminder/whoop-sync.ts | 42 +++++- src/lib/jobs/reminder/withings-sync.ts | 90 ++++++++++-- src/lib/jobs/restore-drill.ts | 27 +++- src/lib/jobs/run-job.ts | 6 + src/lib/jobs/workout-insight-generate.ts | 13 +- 24 files changed, 729 insertions(+), 173 deletions(-) diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index 9bf197017..b71dfa0ab 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -31,6 +31,7 @@ import type { Job } from "pg-boss"; import { extractExportXml } from "@/lib/import/unzip-export-xml"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { streamParseExportXml, type ImportJobProgress, @@ -66,23 +67,26 @@ export const IMPORT_JOB_RECONCILE_CRON = "*/15 * * * *"; * the same `reconcileOrphanImportJobs()` the boot path calls — the * heartbeat + live pg-boss-job check already make it safe to re-run * on a healthy worker (a live import's fresh heartbeat and `active` - * pg-boss state both keep its row untouched). Errors are logged, not - * rethrown — a reconcile miss must never spam pg-boss's retry/backoff - * machinery for a background sweep this cheap to just re-run in 15 - * minutes. + * pg-boss state both keep its row untouched). A sweep that throws now + * reports a failed outcome instead of only leaving a warning behind: + * this pass exists to end a silent stall, and a stall in the pass + * itself was the one condition nothing surfaced. The next tick is 15 + * minutes away regardless, so nothing rests on the retry. */ export async function handleImportJobReconcileTick( jobs: Job[], -): Promise { +): Promise { void jobs; - await withBackgroundEvent( + return withBackgroundEvent( "job.apple_health_import_reconcile", async (evt) => { try { await reconcileOrphanImportJobs(); } catch (err) { evt.addWarning(`apple-health-import-reconcile failed: ${err}`); + return jobFailed("apple health import reconcile failed", err); } + return jobDone(); }, ); } @@ -230,7 +234,7 @@ async function writeProgress( */ export async function handleAppleHealthImport( job: Job, -): Promise { +): Promise { const { userId, uploadPath, uploadBytes, triggeredByAdminId } = job.data; const prisma = getWorkerPrisma(); @@ -280,7 +284,7 @@ export async function handleAppleHealthImport( ` — row is already terminal (${importJob.status}); the staged upload` + " was consumed by the first run and a re-run could only mask its outcome", ); - return; + return jobDone({ skipped: "already_terminal" }); } // Extracted-XML path, hoisted so the failure path can clean it up — @@ -364,6 +368,12 @@ export async function handleAppleHealthImport( // the end, scoped to the user's full measurement span, so // post-import reads of the analytics + comprehensive surfaces // hit the warm rollup table on first paint. + // + // The miss rides out as a fact rather than as a failed job: the + // import is committed and its mirror row already reads `done`, so + // failing here would report a completed import as failed and + // contradict the row the status poll surfaces to the operator. + let rollupFailed = false; try { const span = await prisma.measurement.aggregate({ where: { userId }, @@ -382,6 +392,7 @@ export async function handleAppleHealthImport( } catch (rollupErr) { // Rollup failure is non-fatal — the next read falls through to // live aggregation. Log but don't poison the import. + rollupFailed = true; console.warn( `[apple-health-import] Rollup recompute failed for user ${userId}`, rollupErr, @@ -392,6 +403,13 @@ export async function handleAppleHealthImport( // periodically swept on the host. safeUnlink(unzip.xmlPath); safeUnlink(uploadPath); + + return jobDone({ + records_read: result.totals.recordsRead, + rows_upserted: result.totals.rowsUpserted, + duration_ms: result.totals.durationMs, + rollup_failed: rollupFailed, + }); } catch (err) { // v1.28.33 (issue #486) — a missing staging file is an operational // condition, not a parse failure: `/tmp` is wiped on a container diff --git a/src/lib/jobs/data-arrival.ts b/src/lib/jobs/data-arrival.ts index d51132bd9..cb4318ac6 100644 --- a/src/lib/jobs/data-arrival.ts +++ b/src/lib/jobs/data-arrival.ts @@ -30,6 +30,7 @@ import type { Job } from "pg-boss"; import { invalidateUserDashboardSnapshot } from "@/lib/cache/invalidate"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { annotate } from "@/lib/logging/context"; import { withBackgroundEvent } from "@/lib/logging/background"; @@ -248,14 +249,17 @@ export async function runDataArrival( export async function handleDataArrival( jobs: Job[], -): Promise { - await withBackgroundEvent("job.data_arrival", async (evt) => { +): Promise { + return withBackgroundEvent("job.data_arrival", async (evt) => { + let processed = 0; + let skipped = 0; for (const job of jobs) { const arrival = job.data; try { const outcome = await runDataArrival(getWorkerPrisma(), arrival); if (outcome.status === "skipped") { + skipped++; annotate({ action: { name: `arrival.${arrival.kind}.skipped` }, meta: { reason: outcome.reason, source: arrival.source }, @@ -264,6 +268,7 @@ export async function handleDataArrival( continue; } + processed++; annotate({ action: { name: `arrival.${arrival.kind}.processed` }, meta: { @@ -285,5 +290,9 @@ export async function handleDataArrival( throw err; } } + // Both statuses are successes — a refused arrival is a decision the spine + // made, not work it failed to do. The counts are what make the split + // between them readable at all. + return jobDone({ total: jobs.length, processed, skipped }); }); } diff --git a/src/lib/jobs/document-purge.ts b/src/lib/jobs/document-purge.ts index 8f136be07..eebb5eb7e 100644 --- a/src/lib/jobs/document-purge.ts +++ b/src/lib/jobs/document-purge.ts @@ -11,6 +11,7 @@ */ import { type Job } from "pg-boss"; import { withBackgroundEvent } from "@/lib/logging/background"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { getWorkerPrisma } from "@/lib/jobs/reminder/shared"; import type { PrismaClient } from "@/generated/prisma/client"; @@ -43,16 +44,22 @@ export async function purgeExpiredDocumentTombstones( return count; } -export async function handleDocumentPurge(jobs: Job[]) { +export async function handleDocumentPurge( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.document_tombstone_purge", async (evt) => { + return withBackgroundEvent("job.document_tombstone_purge", async (evt) => { const prisma = getWorkerPrisma(); try { const purged = await purgeExpiredDocumentTombstones(prisma); evt.setAction({ name: "documents.vault.purge" }); evt.addMeta("document_purge_deleted", purged); + // A night with no expired tombstone purges nothing and is still a run + // that did its work — the zero is the fact, not an absence of one. + return jobDone({ document_purge_deleted: purged }); } catch (err) { evt.addWarning(`document-tombstone-purge failed: ${err}`); + return jobFailed("document tombstone purge failed", err); } }); } diff --git a/src/lib/jobs/encryption-key-rotate.ts b/src/lib/jobs/encryption-key-rotate.ts index be6652341..a33588426 100644 --- a/src/lib/jobs/encryption-key-rotate.ts +++ b/src/lib/jobs/encryption-key-rotate.ts @@ -24,6 +24,7 @@ */ import { type Job } from "pg-boss"; import { withBackgroundEvent } from "@/lib/logging/background"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { auditLog } from "@/lib/auth/audit"; import { rotateCorpus, @@ -60,8 +61,8 @@ export async function runEncryptionKeyRotation(): Promise<{ export async function handleEncryptionKeyRotate( jobs: Job[], -) { - await withBackgroundEvent("job.encryption_key_rotate", async (evt) => { +): Promise { + return withBackgroundEvent("job.encryption_key_rotate", async (evt) => { const requestedBy = jobs[0]?.data?.requestedByUserId ?? null; try { const result = await runEncryptionKeyRotation(); @@ -78,6 +79,15 @@ export async function handleEncryptionKeyRotate( errors: result.totalErrors, }, }); + // Per-row errors are the fail-closed skip of a row under a key the + // deployment no longer configures. The pass still ran, so they ride out + // as a count rather than failing (and retrying) the whole corpus. + return jobDone({ + rotate_active_key_id: result.activeKeyId, + rotate_scanned: result.totalScanned, + rotate_rotated: result.totalRotated, + rotate_errors: result.totalErrors, + }); } catch (err) { evt.addWarning(`encryption-key-rotate failed: ${err}`); await auditLog("admin.encryption.rotate.failed", { diff --git a/src/lib/jobs/environment-fetch.ts b/src/lib/jobs/environment-fetch.ts index b21d8379d..84c8dd7b2 100644 --- a/src/lib/jobs/environment-fetch.ts +++ b/src/lib/jobs/environment-fetch.ts @@ -33,6 +33,7 @@ import { utcDayKey, } from "@/lib/environment/service"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { workerLog } from "./reminder/shared"; export const ENVIRONMENT_FETCH_QUEUE = "environment-fetch"; @@ -152,14 +153,14 @@ export async function enqueueEnvironmentFetchDiscovery( export async function handleEnvironmentFetch( boss: PgBoss, payload: EnvironmentFetchPayload, -): Promise { +): Promise { if (!payload.userId) { const { enqueued, skipped } = await enqueueEnvironmentFetchDiscovery(boss); workerLog( "info", `[environment-fetch] discovery enqueued=${enqueued} skipped=${skipped}`, ); - return; + return jobDone({ users_enqueued: enqueued, users_skipped: skipped }); } const userId = payload.userId; @@ -168,7 +169,7 @@ export async function handleEnvironmentFetch( "info", `[environment-fetch] user=${userId} module disabled — skipping`, ); - return; + return jobDone({ skipped: "environment module disabled" }); } const { startDate, endDate } = resolveRange(payload); @@ -179,7 +180,11 @@ export async function handleEnvironmentFetch( "error", `[environment-fetch] user=${userId} range ${startDate}..${endDate} (${span}d) exceeds cap — skipping`, ); - return; + // A refused range is not an opt-out: the job was asked to fill days and + // filled none, so it says so instead of completing quietly on a stderr line. + return jobFailed("environment backfill range exceeds cap", undefined, { + span_days: span, + }); } try { @@ -192,6 +197,11 @@ export async function handleEnvironmentFetch( "info", `[environment-fetch] user=${userId} ${startDate}..${endDate} stored=${result.stored} skipped=${result.skipped} fetches=${result.fetches}`, ); + return jobDone({ + days_stored: result.stored, + days_skipped: result.skipped, + fetches: result.fetches, + }); } catch (err) { recordError(); workerLog("error", `[environment-fetch] user=${userId} failed`, err); diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index d90b90e05..34b4c8b8e 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -17,6 +17,13 @@ * signal a failure) into commission (claiming a success that did not * happen), which is a smaller and much more reviewable class of defect. * + * A fan-out pass is judged on the pass, not on its legs. A nightly sweep + * over every connected account is `ok: true` when the sweep ran, even if one + * account's grant was revoked: that belongs on the account's own integration + * ledger, and failing the queue would retry the whole cohort over one user's + * expired token. The per-leg failures ride out as a count in `did`, which is + * what makes them visible at all — today they are visible nowhere. + * * `did` is the small pinned-shape bag of facts about the run — counts, * identifiers, booleans. It rides into the failure report and, once the * failure ledger lands, into the operator surface. Keep it to scalars that diff --git a/src/lib/jobs/medication-intake-import.ts b/src/lib/jobs/medication-intake-import.ts index 2d0a365e9..2bf370f41 100644 --- a/src/lib/jobs/medication-intake-import.ts +++ b/src/lib/jobs/medication-intake-import.ts @@ -3,6 +3,7 @@ import { Prisma, type PrismaClient } from "@/generated/prisma/client"; import { invalidateUserMedications } from "@/lib/cache/invalidate"; import { prisma, toJson } from "@/lib/db"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { consumeImportedIntakesBatch } from "@/lib/medications/inventory/consumption"; import { queueMedicationIntakeSync } from "@/lib/notifications/medication-intake-sync"; import { @@ -711,9 +712,19 @@ async function recordWorkerFailure( export async function handleMedicationIntakeImport( job: Job, -): Promise { +): Promise { try { - await processMedicationIntakeImportJob(job.data.jobId); + const result = await processMedicationIntakeImportJob(job.data.jobId); + // A null result is a delivery that found the row already terminal or gone. + // There was nothing left to import, which is an absence of work rather + // than a failure to do it. + return result === null + ? jobDone({ finalized: false }) + : jobDone({ + finalized: true, + imported: result.imported, + skipped: result.skipped, + }); } catch (error) { await recordWorkerFailure(job, error); throw error; diff --git a/src/lib/jobs/reaction-line.ts b/src/lib/jobs/reaction-line.ts index 413c8fcc0..db75ed241 100644 --- a/src/lib/jobs/reaction-line.ts +++ b/src/lib/jobs/reaction-line.ts @@ -29,6 +29,7 @@ import type { Job } from "pg-boss"; import { prisma } from "@/lib/db"; import { randomUUID } from "node:crypto"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { annotate } from "@/lib/logging/context"; import { withBackgroundEvent } from "@/lib/logging/background"; import { resolveProviderChain } from "@/lib/ai/provider"; @@ -677,13 +678,16 @@ export async function runReactionLine( export async function handleReactionLine( jobs: Job[], -): Promise { - await withBackgroundEvent("job.reaction_line", async (evt) => { +): Promise { + return withBackgroundEvent("job.reaction_line", async (evt) => { + let generated = 0; + let skipped = 0; for (const job of jobs) { try { const outcome = await runReactionLine(job.data); if (outcome.status === "skipped") { + skipped++; annotate({ action: { name: "arrival.reaction_line.skipped" }, meta: { reason: outcome.reason, kind: job.data.kind }, @@ -692,6 +696,7 @@ export async function handleReactionLine( continue; } + generated++; annotate({ action: { name: "arrival.reaction_line.generated" }, meta: { kind: job.data.kind, local_date: job.data.localDate }, @@ -705,5 +710,9 @@ export async function handleReactionLine( throw err; } } + // A skip is a refusal the surface is designed around, not a miss: the + // deterministic lead already stands without a line. The pass did its work + // whenever it ran to the end. + return jobDone({ total: jobs.length, generated, skipped }); }); } diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index 1106159ed..6d23f8a44 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -8,8 +8,12 @@ import { type Job } from "pg-boss"; import { recordError } from "@/lib/jobs/worker-status"; import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; import { encrypt } from "@/lib/crypto"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; -import { runOffhostBackup } from "@/lib/jobs/offhost-backup"; +import { + OffhostBackupNotConfiguredError, + runOffhostBackup, +} from "@/lib/jobs/offhost-backup"; import { getWorkerPrisma } from "./shared"; export interface DataBackupPayload { @@ -20,9 +24,11 @@ export interface OffhostBackupPayload { triggeredAt: string; } -export async function handleOffhostBackup(jobs: Job[]) { +export async function handleOffhostBackup( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.offhost_backup", async (evt) => { + return withBackgroundEvent("job.offhost_backup", async (evt) => { const p = getWorkerPrisma(); try { const report = await runOffhostBackup(p); @@ -39,16 +45,34 @@ export async function handleOffhostBackup(jobs: Job[]) { JSON.stringify(report.failures.slice(0, 10)), ); } + // Per-user upload failures ride out as `offhost_backup_failed`: the run + // itself uploaded what it could, and failing the queue over one user's + // object would re-upload the whole cohort on every retry. + return jobDone({ + offhost_backup_uploaded: report.uploaded, + offhost_backup_failed: report.failed, + offhost_backup_total_users: report.totalUsers, + offhost_backup_endpoint: report.config.endpoint, + offhost_backup_bucket: report.config.bucket, + }); } catch (err) { - // Not configured ⇒ skip silently with a warning, not an error. + // Not configured ⇒ skip silently with a warning, not an error: most + // self-hosts never set the S3 credentials, and a nightly failed job on + // every one of them buries the runs that genuinely could not upload. evt.addWarning(`offhost-backup skipped/failed: ${err}`); + if (err instanceof OffhostBackupNotConfiguredError) { + return jobDone({ offhost_backup_configured: false }); + } + return jobFailed("offhost backup failed", err); } }); } -export async function handleDataBackup(jobs: Job[]) { +export async function handleDataBackup( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.data_backup", async (evt) => { + return withBackgroundEvent("job.data_backup", async (evt) => { const prisma = getWorkerPrisma(); try { const users = await prisma.user.findMany({ @@ -56,6 +80,7 @@ export async function handleDataBackup(jobs: Job[]) { }); let backed = 0; + let usersFailed = 0; for (const user of users) { try { const { payload } = await buildFullBackupPayload(prisma, user.id, { @@ -82,6 +107,10 @@ export async function handleDataBackup(jobs: Job[]) { }); backed++; } catch (err) { + // One user's payload failing is that user's problem, not the pass's: + // it rides out as a count so the weekly run is not retried for the + // whole cohort. + usersFailed++; evt.addWarning(`Failed for user ${user.id}: ${err}`); } } @@ -90,6 +119,11 @@ export async function handleDataBackup(jobs: Job[]) { task_name: "job.data_backup", result: { backed, total: users.length }, }); + return jobDone({ + backed, + total: users.length, + users_failed: usersFailed, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/cleanup-handlers.ts b/src/lib/jobs/reminder/cleanup-handlers.ts index a4191a3c0..145ac3e30 100644 --- a/src/lib/jobs/reminder/cleanup-handlers.ts +++ b/src/lib/jobs/reminder/cleanup-handlers.ts @@ -2,10 +2,27 @@ * Retention and maintenance cleanups: rate-limit rows, idempotency keys, audit logs, OAuth states (Withings / WHOOP), mood-reminder events, push attempts, and measurement tombstones. * * Extracted from reminder-worker.ts, which owns the queue names, cron - * schedules, and boss.work registrations. + * schedules, and the queue bindings. + * + * Every pass in here used to catch its own failure, write a warning nobody + * reads, and return — so a retention purge that had not run for weeks was + * indistinguishable from one with nothing to delete. They now return + * `jobFailed`, which fails the pg-boss job and puts the queue on the operator's + * failing-jobs surface. + * + * That change has a price, and the registrars pay it: these are bulk DELETEs + * over a retention horizon, and their failure mode is deterministic. A + * statement that timed out against a large trailing edge times out again + * immediately. Under pg-boss's default policy (retryLimit 2, no delay, no + * backoff) each converted queue would run the same doomed statement three + * times in a row every night. So every cron-driven cleanup registered from + * this module is scheduled with `retryLimit: 0`: the failure is recorded once + * and the next night's tick is the retry, which is the right backoff for + * retention work that has a whole day of slack. */ import { type Job } from "pg-boss"; import { withBackgroundEvent } from "@/lib/logging/background"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { cleanupExpiredWhoopConnectTickets, cleanupExpiredWhoopOAuthStates, @@ -78,9 +95,9 @@ export interface MoodReminderCleanupPayload { export async function handleMoodReminderCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.mood_reminder_cleanup", async (evt) => { + return withBackgroundEvent("job.mood_reminder_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const cutoff = new Date(); @@ -90,8 +107,10 @@ export async function handleMoodReminderCleanup( where: { date: { lt: cutoffIso } }, }); evt.addMeta("mood_reminder_cleanup_deleted", deleted.count); + return jobDone({ deleted: deleted.count }); } catch (err) { evt.addWarning(`mood-reminder-cleanup failed: ${err}`); + return jobFailed("mood-reminder cleanup failed", err); } }); } @@ -118,9 +137,9 @@ export interface PushAttemptCleanupPayload { export async function handlePushAttemptCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.push_attempt_cleanup", async (evt) => { + return withBackgroundEvent("job.push_attempt_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const cutoff = new Date(); @@ -129,8 +148,10 @@ export async function handlePushAttemptCleanup( where: { createdAt: { lt: cutoff } }, }); evt.addMeta("push_attempt_cleanup_deleted", deleted.count); + return jobDone({ deleted: deleted.count }); } catch (err) { evt.addWarning(`push-attempt-cleanup failed: ${err}`); + return jobFailed("push-attempt cleanup failed", err); } }); } @@ -152,9 +173,9 @@ export interface ArrivalReactionCleanupPayload { export async function handleArrivalReactionCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.arrival_reaction_cleanup", async (evt) => { + return withBackgroundEvent("job.arrival_reaction_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const cutoff = new Date(); @@ -163,8 +184,10 @@ export async function handleArrivalReactionCleanup( where: { createdAt: { lt: cutoff } }, }); evt.addMeta("arrival_reaction_cleanup_deleted", deleted.count); + return jobDone({ deleted: deleted.count }); } catch (err) { evt.addWarning(`arrival-reaction-cleanup failed: ${err}`); + return jobFailed("arrival-reaction cleanup failed", err); } }); } @@ -175,9 +198,9 @@ export interface MeasurementTombstoneCleanupPayload { export async function handleMeasurementTombstoneCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent( + return withBackgroundEvent( "job.measurement_tombstone_cleanup", async (evt) => { const p = getWorkerPrisma(); @@ -208,6 +231,12 @@ export async function handleMeasurementTombstoneCleanup( "tombstone-cleanup stopped at the batch cap; a backlog remains for the next run", ); } + return jobDone({ + measurements_pruned: measurements.deleted, + mood_pruned: mood.deleted, + intakes_pruned: intakes.deleted, + drained, + }); } catch (err) { // Rethrow. A retention purge that fails silently every night looks // exactly like one that has nothing to do; pg-boss recording a failed @@ -221,32 +250,36 @@ export async function handleMeasurementTombstoneCleanup( export async function handleRateLimitCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.rate_limit_cleanup", async (evt) => { + return withBackgroundEvent("job.rate_limit_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const result = await p.$executeRaw` DELETE FROM rate_limits WHERE reset_at < NOW() `; evt.addMeta("rate_limit_cleanup_deleted", result); + return jobDone({ deleted: result }); } catch (err) { evt.addWarning(`rate-limit-cleanup failed: ${err}`); + return jobFailed("rate-limit cleanup failed", err); } }); } export async function handleIdempotencyCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.idempotency_cleanup", async (evt) => { + return withBackgroundEvent("job.idempotency_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredIdempotencyKeys(p); evt.addMeta("idempotency_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { evt.addWarning(`idempotency-cleanup failed: ${err}`); + return jobFailed("idempotency-key cleanup failed", err); } }); } @@ -266,26 +299,28 @@ export async function handleIdempotencyCleanup( */ export async function handleStepUpElevationCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.step_up_elevation_cleanup", async (evt) => { + return withBackgroundEvent("job.step_up_elevation_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const { count } = await p.stepUpElevation.deleteMany({ where: { expiresAt: { lt: new Date() } }, }); evt.addMeta("step_up_elevation_cleanup_deleted", count); + return jobDone({ deleted: count }); } catch (err) { evt.addWarning(`step-up-elevation-cleanup failed: ${err}`); + return jobFailed("step-up elevation cleanup failed", err); } }); } export async function handleAuditLogCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.audit_log_cleanup", async (evt) => { + return withBackgroundEvent("job.audit_log_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const outcome = await cleanupOldAuditLogs(p); @@ -296,6 +331,7 @@ export async function handleAuditLogCleanup( "audit-log-cleanup stopped at the batch cap; a backlog remains for the next run", ); } + return jobDone({ deleted: outcome.deleted, drained: outcome.drained }); } catch (err) { // Rethrow — see the tombstone handler. Storage-limitation retention that // fails quietly is retention that is not happening. @@ -307,15 +343,17 @@ export async function handleAuditLogCleanup( export async function handleCoachMessageCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.coach_message_cleanup", async (evt) => { + return withBackgroundEvent("job.coach_message_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupOldCoachMessages(p); evt.addMeta("coach_message_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { evt.addWarning(`coach-message-cleanup failed: ${err}`); + return jobFailed("coach-message cleanup failed", err); } }); } @@ -326,9 +364,9 @@ export interface McpTokenCleanupPayload { export async function handleMcpTokenCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.mcp_token_cleanup", async (evt) => { + return withBackgroundEvent("job.mcp_token_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const result = await cleanupExpiredMcpTokens(p); @@ -340,27 +378,40 @@ export async function handleMcpTokenCleanup( "mcp_token_cleanup_connections_deleted", result.connectionsDeleted, ); + return jobDone({ + access_tokens_deleted: result.accessTokensDeleted, + connections_deleted: result.connectionsDeleted, + }); } catch (err) { evt.addWarning(`mcp-token-cleanup failed: ${err}`); + return jobFailed("mcp-token cleanup failed", err); } }); } export async function handleWithingsOAuthStateCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.withings_oauth_state_cleanup", async (evt) => { - const p = getWorkerPrisma(); - try { - const deleted = await cleanupExpiredWithingsOAuthStates(p); - evt.addMeta("withings_oauth_state_cleanup_deleted", deleted); - } catch (err) { - // The OAuth flow tolerates a stale row sticking around for an - // extra day — log + carry on so the boss queue doesn't retry-loop. - evt.addWarning(`withings-oauth-state-cleanup failed: ${err}`); - } - }); + return withBackgroundEvent( + "job.withings_oauth_state_cleanup", + async (evt) => { + const p = getWorkerPrisma(); + try { + const deleted = await cleanupExpiredWithingsOAuthStates(p); + evt.addMeta("withings_oauth_state_cleanup_deleted", deleted); + return jobDone({ deleted }); + } catch (err) { + // The OAuth flow tolerates a stale row for an extra day, which used + // to be the argument for swallowing this so the queue would not + // retry-loop. Tolerable is not the same as invisible: the failure is + // now reported, and the retry-loop worry is answered where it belongs, + // by the `retryLimit: 0` this queue is scheduled with. + evt.addWarning(`withings-oauth-state-cleanup failed: ${err}`); + return jobFailed("withings oauth-state cleanup failed", err); + } + }, + ); } export interface OidcNativeHandoffCleanupPayload { @@ -369,17 +420,21 @@ export interface OidcNativeHandoffCleanupPayload { export async function handleOidcNativeHandoffCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.oidc_native_handoff_cleanup", async (evt) => { + return withBackgroundEvent("job.oidc_native_handoff_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredOidcNativeHandoffs(p); evt.addMeta("oidc_native_handoff_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { - // The handoff flow tolerates a stale row for an extra day (expiry is - // enforced at read) — log + carry on so the boss queue doesn't retry-loop. + // Expiry is enforced at read, so a stale row for an extra day is + // harmless — but see the withings twin above: harmless is not a + // reason to hide the failure. `retryLimit: 0` on the schedule is what + // keeps a doomed statement from running three times a night. evt.addWarning(`oidc-native-handoff-cleanup failed: ${err}`); + return jobFailed("oidc native-handoff cleanup failed", err); } }); } @@ -390,17 +445,22 @@ export interface WhoopOAuthStateCleanupPayload { export async function handleWhoopOAuthStateCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.whoop_oauth_state_cleanup", async (evt) => { + return withBackgroundEvent("job.whoop_oauth_state_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredWhoopOAuthStates(p); evt.addMeta("whoop_oauth_state_cleanup_deleted", deleted); const ticketsDeleted = await cleanupExpiredWhoopConnectTickets(p); evt.addMeta("whoop_connect_ticket_cleanup_deleted", ticketsDeleted); + return jobDone({ + states_deleted: deleted, + connect_tickets_deleted: ticketsDeleted, + }); } catch (err) { evt.addWarning(`whoop-oauth-state-cleanup failed: ${err}`); + return jobFailed("whoop oauth-state cleanup failed", err); } }); } diff --git a/src/lib/jobs/reminder/fitbit-sync.ts b/src/lib/jobs/reminder/fitbit-sync.ts index d9a45b895..e55886ff5 100644 --- a/src/lib/jobs/reminder/fitbit-sync.ts +++ b/src/lib/jobs/reminder/fitbit-sync.ts @@ -7,6 +7,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { runFitbitPollCohort } from "@/lib/fitbit/sync"; import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; @@ -23,8 +24,10 @@ export interface FitbitSyncPayload { userId?: string; } -export async function handleFitbitSync(jobs: Job[]) { - await withBackgroundEvent("job.fitbit_sync", async (evt) => { +export async function handleFitbitSync( + jobs: Job[], +): Promise { + return withBackgroundEvent("job.fitbit_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -37,16 +40,26 @@ export async function handleFitbitSync(jobs: Job[]) { }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } // Fan the cohort out with bounded concurrency + per-user error isolation: // one slow Google response can't stall the whole cohort, and a single // user's failure is warned without aborting the pass. + let usersFailed = 0; const { usersSynced, measurementsImported } = await runFitbitPollCohort( targets.map((t) => t.userId), { - onUserError: (userId, err) => - evt.addWarning(`job.fitbit_sync failed for user ${userId}: ${err}`), + onUserError: (userId, err) => { + usersFailed++; + evt.addWarning(`job.fitbit_sync failed for user ${userId}: ${err}`); + }, // v1.18.1 — a fresh reading landed; resolve the user's Vorsorge // reminders eventfully. Fire-and-forget; the cron is the net. onUserSynced: (userId, imported) => { @@ -67,6 +80,14 @@ export async function handleFitbitSync(jobs: Job[]) { measurements_imported: measurementsImported, }, }); + // The pass is the unit of success: a revoked grant belongs on that user's + // integration ledger, not in a queue failure that retries every account. + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -81,15 +102,17 @@ export interface FitbitOAuthStateCleanupPayload { export async function handleFitbitOAuthStateCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.fitbit_oauth_state_cleanup", async (evt) => { + return withBackgroundEvent("job.fitbit_oauth_state_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredFitbitOAuthStates(p); evt.addMeta("fitbit_oauth_state_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { evt.addWarning(`fitbit-oauth-state-cleanup failed: ${err}`); + return jobFailed("fitbit oauth state cleanup failed", err); } }); } diff --git a/src/lib/jobs/reminder/google-health-sync.ts b/src/lib/jobs/reminder/google-health-sync.ts index 0006f5ca8..d6791814b 100644 --- a/src/lib/jobs/reminder/google-health-sync.ts +++ b/src/lib/jobs/reminder/google-health-sync.ts @@ -7,6 +7,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { runGoogleHealthPollCohort } from "@/lib/google-health/sync"; import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; @@ -26,8 +27,8 @@ export interface GoogleHealthSyncPayload { export async function handleGoogleHealthSync( jobs: Job[], -) { - await withBackgroundEvent("job.google_health_sync", async (evt) => { +): Promise { + return withBackgroundEvent("job.google_health_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -40,19 +41,29 @@ export async function handleGoogleHealthSync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } // Fan the cohort out with bounded concurrency + per-user error isolation: // one slow Google response can't stall the whole cohort, and a single // user's failure is warned without aborting the pass. + let usersFailed = 0; const { usersSynced, measurementsImported } = await runGoogleHealthPollCohort( targets.map((t) => t.userId), { - onUserError: (userId, err) => + onUserError: (userId, err) => { + usersFailed++; evt.addWarning( `job.google_health_sync failed for user ${userId}: ${err}`, - ), + ); + }, // A fresh reading landed; resolve the user's Vorsorge reminders // eventfully. Fire-and-forget; the cron is the net. onUserSynced: (userId, imported) => { @@ -73,6 +84,14 @@ export async function handleGoogleHealthSync( measurements_imported: measurementsImported, }, }); + // The pass is the unit of success: a revoked grant belongs on that user's + // integration ledger, not in a queue failure that retries every account. + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -87,17 +106,19 @@ export interface GoogleHealthOAuthStateCleanupPayload { export async function handleGoogleHealthOAuthStateCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent( + return withBackgroundEvent( "job.google_health_oauth_state_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredGoogleHealthOAuthStates(p); evt.addMeta("google_health_oauth_state_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { evt.addWarning(`google-health-oauth-state-cleanup failed: ${err}`); + return jobFailed("google health oauth state cleanup failed", err); } }, ); diff --git a/src/lib/jobs/reminder/insights-handlers.ts b/src/lib/jobs/reminder/insights-handlers.ts index 2c7117714..fd275f43c 100644 --- a/src/lib/jobs/reminder/insights-handlers.ts +++ b/src/lib/jobs/reminder/insights-handlers.ts @@ -19,6 +19,7 @@ import { type InsightStatusGeneratePayload, runInsightStatusGenerate, } from "@/lib/jobs/insight-status-generate"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { generateBloodPressureStatusForUser } from "@/lib/insights/blood-pressure-status"; import { generateWeightStatusForUser } from "@/lib/insights/weight-status"; @@ -69,6 +70,10 @@ export interface MedicationComplianceStatusPayload { * `status-cron-candidates.ts` for the full division of nightly labour). * The generators normalise `locale` themselves (de stays de, everything * else gets English prose). + * + * The pass is what succeeds or fails. One user whose generation threw is that + * user's cold card, reported as the `failed` count; failing the job would + * re-run the whole cohort over it. */ export async function runStatusCronGenerate( taskName: string, @@ -76,13 +81,15 @@ export async function runStatusCronGenerate( userId: string, options: { locale: string | null; force: boolean }, ) => Promise, -): Promise { - await withBackgroundEvent(taskName, async (evt) => { +): Promise { + return withBackgroundEvent(taskName, async (evt) => { const prisma = getWorkerPrisma(); try { const users = await findStatusCronCandidates(prisma); - if (users.length === 0) return; + if (users.length === 0) { + return jobDone({ total: 0, generated: 0, failed: 0 }); + } let generated = 0; let failed = 0; @@ -103,6 +110,7 @@ export async function runStatusCronGenerate( task_name: taskName, result: { generated, failed, total: users.length }, }); + return jobDone({ total: users.length, generated, failed }); } catch (err) { evt.setError(err); recordError(); @@ -129,13 +137,15 @@ export async function runStatusCronGenerate( * inside the batch itself). Net effect: the nightly ladder pays one call per * user instead of seven, with no queue/registry churn. */ -async function runStatusBatchCron(taskName: string): Promise { - await withBackgroundEvent(taskName, async (evt) => { +async function runStatusBatchCron(taskName: string): Promise { + return withBackgroundEvent(taskName, async (evt) => { const prisma = getWorkerPrisma(); try { recordInsightsRun(); const users = await findStatusCronCandidates(prisma); - if (users.length === 0) return; + if (users.length === 0) { + return jobDone({ total: 0, generated: 0, served: 0, failed: 0 }); + } let generated = 0; let served = 0; @@ -167,6 +177,10 @@ async function runStatusBatchCron(taskName: string): Promise { task_name: taskName, result: { generated, served, failed, total: users.length }, }); + // A user whose batch call threw already fell back to the single-card + // path inside the batch, and the six later per-metric crons cover what + // is still cold. The batch pass ran, so it reports the count. + return jobDone({ total: users.length, generated, served, failed }); } catch (err) { evt.setError(err); recordError(); @@ -176,14 +190,16 @@ async function runStatusBatchCron(taskName: string): Promise { }); } -export function handleGeneralStatusGenerate(jobs: Job[]) { +export function handleGeneralStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusBatchCron("job.insights.batch"); } export function handleBloodPressureStatusGenerate( jobs: Job[], -) { +): Promise { void jobs; return runStatusCronGenerate( "job.insights.blood_pressure", @@ -191,7 +207,9 @@ export function handleBloodPressureStatusGenerate( ); } -export function handleWeightStatusGenerate(jobs: Job[]) { +export function handleWeightStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusCronGenerate( "job.insights.weight", @@ -199,7 +217,9 @@ export function handleWeightStatusGenerate(jobs: Job[]) { ); } -export function handlePulseStatusGenerate(jobs: Job[]) { +export function handlePulseStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusCronGenerate( "job.insights.pulse", @@ -207,19 +227,23 @@ export function handlePulseStatusGenerate(jobs: Job[]) { ); } -export function handleBmiStatusGenerate(jobs: Job[]) { +export function handleBmiStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusCronGenerate("job.insights.bmi", generateBmiStatusForUser); } -export function handleMoodStatusGenerate(jobs: Job[]) { +export function handleMoodStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusCronGenerate("job.insights.mood", generateMoodStatusForUser); } export function handleMedicationComplianceStatusGenerate( jobs: Job[], -) { +): Promise { void jobs; return runStatusCronGenerate( "job.insights.medication_compliance", @@ -229,8 +253,8 @@ export function handleMedicationComplianceStatusGenerate( export async function handleInsightPregenerateJob( jobs: Job[], -) { - await withBackgroundEvent("job.insight_pregenerate", async (evt) => { +): Promise { + return withBackgroundEvent("job.insight_pregenerate", async (evt) => { // v1.8.7.1 — a forced single-user warm carries `{ userId, force }`; // the scheduled tick carries neither. Route each job individually so a // batch that mixes a cron tick with on-demand warms (it never does in @@ -265,13 +289,30 @@ export async function handleInsightPregenerateJob( } } - if (scheduled.length === 0) return; + // A batch of forced warms only is a complete run: every warm either landed + // or threw out of the loop above. + if (scheduled.length === 0) { + return jobDone({ forced: forced.length, scheduled: 0 }); + } try { const summary = await runInsightPregenerate(getWorkerPrisma()); evt.setBackground({ task_name: "job.insight_pregenerate", result: { ...summary }, }); + return jobDone({ + forced: forced.length, + scheduled: scheduled.length, + total: summary.total, + generated: summary.generated, + cached: summary.cached, + unchanged: summary.unchanged, + skipped: summary.skipped, + failed: summary.failed, + budget_blocked: summary.budgetBlocked, + assessments_warmed: summary.assessmentsWarmed, + metric_assessments_warmed: summary.metricAssessmentsWarmed, + }); } catch (err) { evt.addWarning( `insight-pregenerate failed: ${ @@ -296,12 +337,18 @@ export async function handleInsightPregenerateJob( */ export async function handleInsightStatusGenerate( jobs: Job[], -) { - await withBackgroundEvent("job.insight_status_generate", async (evt) => { +): Promise { + return withBackgroundEvent("job.insight_status_generate", async (evt) => { + let generated = 0; + let malformed = 0; for (const job of jobs) { - if (!job.data?.userId || !job.data?.metric) continue; + if (!job.data?.userId || !job.data?.metric) { + malformed++; + continue; + } try { await runInsightStatusGenerate(job.data); + generated++; evt.addMeta( "status_generated", `${job.data.metric}:${job.data.locale}`, @@ -321,5 +368,8 @@ export async function handleInsightStatusGenerate( throw err; } } + // Every job in the batch either generated a card or carried no metric to + // generate; a generation that failed left through the rethrow above. + return jobDone({ generated, malformed }); }); } diff --git a/src/lib/jobs/reminder/medication-maintenance.ts b/src/lib/jobs/reminder/medication-maintenance.ts index 326cb8cd3..b906eb8f6 100644 --- a/src/lib/jobs/reminder/medication-maintenance.ts +++ b/src/lib/jobs/reminder/medication-maintenance.ts @@ -10,6 +10,7 @@ import { type IntakeAutoSkipPayload, runIntakeAutoSkipPass, } from "@/lib/jobs/intake-auto-skip"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { expireStaleInUseItems } from "@/lib/medications/inventory/service"; import { getWorkerPrisma } from "./shared"; @@ -21,18 +22,21 @@ import { getWorkerPrisma } from "./shared"; */ export async function handleMedicationInventoryExpire( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.medication_inventory_expire", async (evt) => { + return withBackgroundEvent("job.medication_inventory_expire", async (evt) => { try { const count = await expireStaleInUseItems({ nowMs: Date.now() }); evt.addMeta("inventory_expired_count", count); + // A pass that found no lapsed pen did its work; zero is a result. + return jobDone({ inventory_expired_count: count }); } catch (err) { evt.addWarning( `medication-inventory-expire failed: ${ err instanceof Error ? err.message : String(err) }`, ); + return jobFailed("inventory expire pass failed", err); } }); } @@ -46,9 +50,11 @@ export async function handleMedicationInventoryExpire( * unit test can drive it with an in-memory fake Prisma; this wrapper * threads the worker's pg-boss + background-event plumbing. */ -export async function handleIntakeAutoSkip(jobs: Job[]) { +export async function handleIntakeAutoSkip( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.intake_auto_skip", async (evt) => { + return withBackgroundEvent("job.intake_auto_skip", async (evt) => { const prisma = getWorkerPrisma(); try { const result = await runIntakeAutoSkipPass(prisma, { @@ -56,12 +62,17 @@ export async function handleIntakeAutoSkip(jobs: Job[]) { }); evt.addMeta("intake_auto_skip_count", result.skippedCount); evt.addMeta("intake_auto_skip_cutoff", result.cutoff.toISOString()); + return jobDone({ + intake_auto_skip_count: result.skippedCount, + intake_auto_skip_cutoff: result.cutoff.toISOString(), + }); } catch (err) { evt.addWarning( `intake-auto-skip failed: ${ err instanceof Error ? err.message : String(err) }`, ); + return jobFailed("intake auto-skip pass failed", err); } }); } diff --git a/src/lib/jobs/reminder/medication-reminder-check.ts b/src/lib/jobs/reminder/medication-reminder-check.ts index ff64ab72d..a21f07939 100644 --- a/src/lib/jobs/reminder/medication-reminder-check.ts +++ b/src/lib/jobs/reminder/medication-reminder-check.ts @@ -8,6 +8,7 @@ import { type Job } from "pg-boss"; import { recordError, recordReminderCheck } from "@/lib/jobs/worker-status"; import { recomputeMedicationComplianceForEvent } from "@/lib/rollups/medication-compliance-rollups"; import { decrypt } from "@/lib/crypto"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { dispatchNotification } from "@/lib/notifications/dispatcher"; import { medicationDoseTag } from "@/lib/notifications/dose-tag"; @@ -137,9 +138,11 @@ export async function cleanupScheduledTelegramDeletions(): Promise { * Uses phase-based logic (GREEN/YELLOW/ORANGE/RED) to send one notification * per phase transition rather than every 15 minutes. */ -export async function handleReminderCheck(jobs: Job[]) { +export async function handleReminderCheck( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.medication_reminder", async (evt) => { + return withBackgroundEvent("job.medication_reminder", async (evt) => { const prisma = getWorkerPrisma(); try { recordReminderCheck(); @@ -176,6 +179,12 @@ export async function handleReminderCheck(jobs: Job[]) { }, }); + // The per-slot facts the tick already emits as wide-event meta, counted + // so the run's outcome carries them as scalars. + let missedDosesMinted = 0; + let notificationsDispatched = 0; + let notificationsFailed = 0; + for (const med of medications) { const userTz = med.user.timezone || "Europe/Berlin"; const { start: todayStart, end: todayEnd } = getUserTodayBounds( @@ -440,6 +449,7 @@ export async function handleReminderCheck(jobs: Job[]) { }, }); + missedDosesMinted++; evt.addMeta("missed_dose", `${med.name}:${slotTime}`); // v1.4.39 W-MED — refresh the compliance rollup so the @@ -521,7 +531,12 @@ export async function handleReminderCheck(jobs: Job[]) { url: `/medications/${med.id}`, }, }); + notificationsDispatched++; } catch (notifErr) { + // One user's channel failing must not fail the tick — the + // dedup anchor is already stamped, so the count is the only + // trace this slot's notice never left the host. + notificationsFailed++; evt.addWarning( `Notification dispatch failed for ${currentPhase} phase ${med.name}: ${notifErr}`, ); @@ -530,6 +545,13 @@ export async function handleReminderCheck(jobs: Job[]) { } } } + + return jobDone({ + medications: medications.length, + missed_doses_minted: missedDosesMinted, + notifications_dispatched: notificationsDispatched, + notifications_failed: notificationsFailed, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/mood-cycle-checks.ts b/src/lib/jobs/reminder/mood-cycle-checks.ts index afa03a835..53b783c9d 100644 --- a/src/lib/jobs/reminder/mood-cycle-checks.ts +++ b/src/lib/jobs/reminder/mood-cycle-checks.ts @@ -6,6 +6,7 @@ */ import { type Job } from "pg-boss"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { runMoodReminderTick } from "@/lib/jobs/mood-reminder"; import { runCycleReminderTick } from "@/lib/jobs/cycle-reminder"; @@ -30,9 +31,9 @@ export interface MeasurementReminderPayload { export async function handleMoodReminderCheck( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.mood_reminder", async (evt) => { + return withBackgroundEvent("job.mood_reminder", async (evt) => { const prisma = getWorkerPrisma(); try { const summary = await runMoodReminderTick(prisma, new Date()); @@ -47,6 +48,16 @@ export async function handleMoodReminderCheck( skipped_outside_window: summary.skippedOutsideWindow, }, }); + // A tick that found nobody in their local reminder hour still ran the + // sweep, so it is a success with zero counts. + return jobDone({ + candidates_scanned: summary.candidatesScanned, + in_window: summary.inWindow, + dispatched: summary.dispatched, + skipped_already_logged: summary.skippedAlreadyLogged, + skipped_already_dispatched: summary.skippedAlreadyDispatched, + skipped_outside_window: summary.skippedOutsideWindow, + }); } catch (err) { evt.setError(err); recordError(); @@ -63,9 +74,9 @@ export async function handleMoodReminderCheck( */ export async function handleCycleReminderCheck( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.cycle_reminder", async (evt) => { + return withBackgroundEvent("job.cycle_reminder", async (evt) => { const prisma = getWorkerPrisma(); try { const summary = await runCycleReminderTick(prisma, new Date()); @@ -82,6 +93,16 @@ export async function handleCycleReminderCheck( skipped_outside_window: summary.skippedOutsideWindow, }, }); + return jobDone({ + candidates_scanned: summary.candidatesScanned, + in_window: summary.inWindow, + dispatched_period_soon: summary.dispatchedPeriodSoon, + dispatched_period_confirm: summary.dispatchedPeriodConfirm, + suppressed_client_managed: summary.suppressedClientManaged, + suppressed_discreet: summary.suppressedDiscreet, + skipped_already_notified: summary.skippedAlreadyNotified, + skipped_outside_window: summary.skippedOutsideWindow, + }); } catch (err) { evt.setError(err); recordError(); @@ -97,9 +118,9 @@ export async function handleCycleReminderCheck( */ export async function handleMeasurementReminderCheck( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.measurement_reminder", async (evt) => { + return withBackgroundEvent("job.measurement_reminder", async (evt) => { const prisma = getWorkerPrisma(); try { const summary = await runMeasurementReminderTick(prisma, new Date()); @@ -116,6 +137,19 @@ export async function handleMeasurementReminderCheck( failed: summary.failed, }, }); + // `failed` counts per-reminder dispatch failures the tick isolated. The + // sweep itself ran, so those ride out as a count rather than failing the + // job and retrying the whole cohort over one user's channel. + return jobDone({ + candidates_scanned: summary.candidatesScanned, + in_window: summary.inWindow, + dispatched: summary.dispatched, + auto_resolved: summary.autoResolved, + skipped_not_due: summary.skippedNotDue, + skipped_outside_window: summary.skippedOutsideWindow, + skipped_no_channel: summary.skippedNoChannel, + failed: summary.failed, + }); } catch (err) { evt.setError(err); recordError(); @@ -133,8 +167,8 @@ export async function handleMeasurementReminderCheck( */ export async function handleReminderSatisfy( jobs: Job[], -) { - await withBackgroundEvent("job.reminder_satisfy", async (evt) => { +): Promise { + return withBackgroundEvent("job.reminder_satisfy", async (evt) => { const prisma = getWorkerPrisma(); const now = new Date(); // Distinct users in this batch — a multi-source ingest spike can enqueue @@ -160,6 +194,13 @@ export async function handleReminderSatisfy( satisfied, }, }); + // A batch whose jobs carried no usable userId resolved nothing and is + // still a success with zero counts. + return jobDone({ + users: userIds.length, + candidates_scanned: candidates, + satisfied, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/ops-handlers.ts b/src/lib/jobs/reminder/ops-handlers.ts index a03154056..b483243c5 100644 --- a/src/lib/jobs/reminder/ops-handlers.ts +++ b/src/lib/jobs/reminder/ops-handlers.ts @@ -8,6 +8,7 @@ import { type Job } from "pg-boss"; import { runGeoBackfill } from "@/lib/jobs/geo-backfill"; import { runTlsPinMonitor } from "@/lib/jobs/tls-pin-monitor"; import { type PrDetectionPayload } from "@/lib/jobs/pr-detection"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { runHostMetricTick } from "@/lib/jobs/host-metric-sampler"; import { aggregateRecommendationFeedback } from "@/lib/jobs/feedback-aggregator"; @@ -34,40 +35,45 @@ export interface TlsPinMonitorPayload { export async function handleHostMetricSample( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.host_metric_sample", async (evt) => { + return withBackgroundEvent("job.host_metric_sample", async (evt) => { const p = getWorkerPrisma(); try { const { pruned } = await runHostMetricTick(p); evt.addMeta("host_metric_pruned", pruned); + return jobDone({ host_metric_pruned: pruned }); } catch (err) { - // The chart degrades gracefully when samples are missing — log - // and move on rather than poisoning the boss queue with retries. + // The chart degrades gracefully when samples are missing, so the + // warning stays — but a tick that took no sample did not do its work. evt.addWarning(`host-metric-sample failed: ${err}`); + return jobFailed("host metric sample failed", err); } }); } export async function handleFeedbackAggregator( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.feedback_aggregator", async (evt) => { + return withBackgroundEvent("job.feedback_aggregator", async (evt) => { const p = getWorkerPrisma(); try { const summary = await aggregateRecommendationFeedback(p); + const totalRows = summary.buckets.reduce((acc, b) => acc + b.total, 0); evt.addMeta("feedback_buckets", summary.buckets.length); - evt.addMeta( - "feedback_total_rows", - summary.buckets.reduce((acc, b) => acc + b.total, 0), - ); + evt.addMeta("feedback_total_rows", totalRows); evt.addMeta("feedback_window_days", summary.windowDays); + return jobDone({ + feedback_buckets: summary.buckets.length, + feedback_total_rows: totalRows, + feedback_window_days: summary.windowDays, + }); } catch (err) { - // The admin dashboard tolerates a stale summary — log and move - // on rather than poisoning the boss queue with retries that - // would block the next cleanup window. + // The admin dashboard tolerates a stale summary, so the warning stays — + // but a pass that aggregated nothing did not do its work. evt.addWarning(`feedback-aggregator failed: ${err}`); + return jobFailed("feedback aggregation failed", err); } }); } @@ -91,18 +97,21 @@ export async function handleFeedbackAggregator( */ let geoBackfillRunning = false; -export async function handleGeoBackfill(jobs: Job[]) { +export async function handleGeoBackfill( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.geo_backfill", async (evt) => { + return withBackgroundEvent("job.geo_backfill", async (evt) => { if (geoBackfillRunning) { // Earlier pass still in flight; skip this tick. Idempotent + the // next tick after the in-flight pass completes will pick up - // anything we miss here. + // anything we miss here. Deliberately ceding to the in-flight pass + // is not a failure, so the tick reports the skip and succeeds. evt.addWarning( "geo-backfill skipped — earlier pass still in flight inside the same worker process", ); evt.addMeta("geo_backfill_skipped", true); - return; + return jobDone({ geo_backfill_skipped: true }); } geoBackfillRunning = true; const p = getWorkerPrisma(); @@ -112,38 +121,60 @@ export async function handleGeoBackfill(jobs: Job[]) { evt.addMeta("geo_backfill_located", summary.located); evt.addMeta("geo_backfill_carrier_resolved", summary.carrierResolved); evt.addMeta("geo_backfill_still_unresolved", summary.stillUnresolved); + return jobDone({ + geo_backfill_scanned: summary.scanned, + geo_backfill_located: summary.located, + geo_backfill_carrier_resolved: summary.carrierResolved, + geo_backfill_still_unresolved: summary.stillUnresolved, + }); } catch (err) { - // The admin sign-in overview tolerates a stale Standort cell — - // log and move on so a one-off resolver hiccup does not poison - // the queue and block the next pass. + // The admin sign-in overview tolerates a stale Standort cell, so the + // warning stays — but a pass that resolved nothing did not do its work. evt.addWarning(`geo-backfill failed: ${err}`); + return jobFailed("geo backfill pass failed", err); } finally { geoBackfillRunning = false; } }); } -export async function handleTlsPinMonitor(jobs: Job[]) { +export async function handleTlsPinMonitor( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.tls_pin_monitor", async (evt) => { + return withBackgroundEvent("job.tls_pin_monitor", async (evt) => { const p = getWorkerPrisma(); try { const summary = await runTlsPinMonitor(p); evt.addMeta("tls_pin_monitor_outcome", summary.outcome); evt.addMeta("tls_pin_monitor_host", summary.host); evt.addMeta("tls_pin_monitor_known_count", summary.knownPinCount); + // No configured target leaves `host` null; the empty string keeps the + // fact shape scalar, and `outcome: "skipped"` already names that case. + return jobDone({ + tls_pin_monitor_outcome: summary.outcome, + tls_pin_monitor_host: summary.host ?? "", + tls_pin_monitor_known_count: summary.knownPinCount, + }); } catch (err) { // runTlsPinMonitor swallows probe failures internally; anything that - // escapes is unexpected. Log and move on so a one-off failure does not - // poison the queue and block the next tick. + // escapes is unexpected, and a tick that never compared a pin is + // exactly the silence this monitor exists to break. evt.addWarning(`tls-pin-monitor failed: ${err}`); + return jobFailed("tls pin probe failed", err); } }); } export async function handlePrDetection( jobs: Job[], -) { +): Promise { + // Each job in the batch opens its own wide event; the outcome is reported + // once for the batch, so the per-job facts accumulate out here. + let usersScanned = 0; + let insertedAcrossJobs = 0; + let tiesAcrossJobs = 0; + let usersFailed = 0; for (const job of jobs) { await withBackgroundEvent("job.pr_detection", async (evt) => { const p = getWorkerPrisma(); @@ -170,6 +201,10 @@ export async function handlePrDetection( insertedTotal += result.inserted; tiesTotal += result.ties; } catch (err) { + // One user's detection failing is that user's problem, not the + // pass's: it rides out as a count so the cron is not retried for + // the whole cohort. + usersFailed++; evt.addWarning(`pr-detection failed for user ${userId}: ${err}`); } } @@ -178,6 +213,18 @@ export async function handlePrDetection( evt.addMeta("pr_detection_ties", tiesTotal); evt.addMeta("pr_detection_silent", silent); evt.addMeta("pr_detection_mode", payloadUserId ? "ingest" : "cron"); + + usersScanned += userIds.length; + insertedAcrossJobs += insertedTotal; + tiesAcrossJobs += tiesTotal; }); } + + return jobDone({ + pr_detection_jobs: jobs.length, + pr_detection_users: usersScanned, + pr_detection_inserted: insertedAcrossJobs, + pr_detection_ties: tiesAcrossJobs, + pr_detection_users_failed: usersFailed, + }); } diff --git a/src/lib/jobs/reminder/poll-cohort.ts b/src/lib/jobs/reminder/poll-cohort.ts index a24a12ff3..154f61af0 100644 --- a/src/lib/jobs/reminder/poll-cohort.ts +++ b/src/lib/jobs/reminder/poll-cohort.ts @@ -21,6 +21,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; import { isSyncFailureRecorded } from "@/lib/integrations/status"; @@ -58,6 +59,13 @@ export interface PollCohortConfig { * handler resolves its targets from the batch payloads (single-user enqueue) or * the cohort query (cron tick), then re-syncs each sequentially with per-user * error isolation so one revoked grant / outage never starves the cohort. + * + * Outcome semantics for a fan-out pass: the JOB succeeded if the pass ran. + * One user's revoked grant is that user's problem — it belongs on their + * integration ledger, not in a queue-wide failure that retries the whole + * cohort — so per-user failures ride out as the `users_failed` count. Only a + * failure of the pass itself (cohort discovery, the wide-event scope) fails + * the job, and that path already throws. */ export function makePollCohortHandler({ taskName, @@ -65,8 +73,10 @@ export function makePollCohortHandler({ syncUser, recordFailure, }: PollCohortConfig) { - return async function handlePollCohort(jobs: Job[]) { - await withBackgroundEvent(taskName, async (evt) => { + return async function handlePollCohort( + jobs: Job[], + ): Promise { + return withBackgroundEvent(taskName, async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -77,9 +87,12 @@ export function makePollCohortHandler({ const ids = await findCohort(prisma); targets.push(...ids.map((userId) => ({ userId }))); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ total: 0, users_synced: 0, users_failed: 0 }); + } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const { userId } of targets) { try { @@ -106,6 +119,7 @@ export function makePollCohortHandler({ ); }); } + usersFailed++; evt.addWarning(`${taskName} failed for user ${userId}: ${err}`); } } @@ -114,10 +128,17 @@ export function makePollCohortHandler({ task_name: taskName, result: { users_synced: usersSynced, + users_failed: usersFailed, total: targets.length, measurements_imported: measurementsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/register-reminders.ts b/src/lib/jobs/reminder/register-reminders.ts index 66a42de2a..581a81358 100644 --- a/src/lib/jobs/reminder/register-reminders.ts +++ b/src/lib/jobs/reminder/register-reminders.ts @@ -7,9 +7,15 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The * `cycle-reminder-queue` and `measurement-reminder-queue` guards read THIS * module. + * + * Retry policy: every queue here keeps the pg-boss default. These ticks are + * fan-out passes whose per-user failures never fail the job, so a job-level + * failure means the pass itself could not run — a transient the default two + * retries are the right answer to. The satisfy queue carries its own + * `retryLimit` at send time (see `reminder-satisfy.ts`). */ import { PgBoss } from "pg-boss"; import { @@ -17,7 +23,11 @@ import { REMINDER_SATISFY_CONCURRENCY, type ReminderSatisfyPayload, } from "@/lib/jobs/reminder-satisfy"; -import { createAndSchedule, type ScheduleEntry } from "./registrar-shared"; +import { + createAndSchedule, + createAndWork, + type ScheduleEntry, +} from "./registrar-shared"; import { ReminderCheckPayload, handleReminderCheck, @@ -121,7 +131,8 @@ export async function registerReminderQueues( ): Promise { await createAndSchedule(boss, allQueues, schedules); - await boss.work( + await createAndWork( + boss, QUEUE_NAME, { localConcurrency: 1 }, handleReminderCheck, @@ -130,7 +141,8 @@ export async function registerReminderQueues( // two reminder ticks from interleaving against the same user row; // the dedup ledger would still save us, but skipping the race here // avoids spurious P2002 errors in the wide-event log. - await boss.work( + await createAndWork( + boss, MOOD_REMINDER_QUEUE, { localConcurrency: 1 }, handleMoodReminderCheck, @@ -138,7 +150,8 @@ export async function registerReminderQueues( // v1.15 — single-flight cycle-reminder worker. localConcurrency=1 keeps // two ticks from racing the fire-and-forget `push_attempts` ledger that // anchors the per-day idempotency, exactly like the mood-reminder worker. - await boss.work( + await createAndWork( + boss, CYCLE_REMINDER_QUEUE, { localConcurrency: 1 }, handleCycleReminderCheck, @@ -147,7 +160,8 @@ export async function registerReminderQueues( // localConcurrency=1 keeps two ticks from racing the `nextDueAt` // advance that anchors the per-cycle idempotency, exactly like the // mood / cycle reminder workers. - await boss.work( + await createAndWork( + boss, MEASUREMENT_REMINDER_QUEUE, { localConcurrency: 1 }, handleMeasurementReminderCheck, @@ -155,7 +169,8 @@ export async function registerReminderQueues( // v1.18.1 — eventful Vorsorge satisfaction. Resolves a user's reminders // against their just-landed measurement / lab. Read-heavy on the user's // own data; the same small concurrency budget as PR detection. - await boss.work( + await createAndWork( + boss, REMINDER_SATISFY_QUEUE, { localConcurrency: REMINDER_SATISFY_CONCURRENCY }, handleReminderSatisfy, diff --git a/src/lib/jobs/reminder/whoop-sync.ts b/src/lib/jobs/reminder/whoop-sync.ts index 10f07df56..830ab4ebc 100644 --- a/src/lib/jobs/reminder/whoop-sync.ts +++ b/src/lib/jobs/reminder/whoop-sync.ts @@ -7,6 +7,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { syncUserRecovery, @@ -66,6 +67,10 @@ type ByIdSync = (userId: string, resourceId: string) => Promise; * - A cron tick (no `userId` on any job) walks every connection's collection. * * One user's parked-at-reauth state never starves the rest of the cohort. + * + * The pass is what the outcome judges: a per-user failure is that user's + * problem, recorded on their integration ledger and counted as `users_failed`, + * not a queue-wide failure that would retry the whole cohort. */ export async function runWhoopResourceSync( taskName: string, @@ -73,8 +78,8 @@ export async function runWhoopResourceSync( jobs: Job[], syncFn: CollectionSync, byIdFn?: ByIdSync, -): Promise { - await withBackgroundEvent(taskName, async (evt) => { +): Promise { + return withBackgroundEvent(taskName, async (evt) => { const prisma = getWorkerPrisma(); try { // (userId, optional resourceId) work items from the webhook payloads. @@ -94,9 +99,17 @@ export async function runWhoopResourceSync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const { userId, resourceId } of targets) { try { @@ -123,6 +136,7 @@ export async function runWhoopResourceSync( if (!isSyncFailureRecorded(err)) { await recordWhoopSyncFailure(userId, err, leg); } + usersFailed++; evt.addWarning(`${taskName} failed for user ${userId}: ${err}`); } } @@ -135,6 +149,12 @@ export async function runWhoopResourceSync( measurements_imported: measurementsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -143,7 +163,9 @@ export async function runWhoopResourceSync( }); } -export function handleWhoopRecoverySync(jobs: Job[]) { +export function handleWhoopRecoverySync( + jobs: Job[], +): Promise { return runWhoopResourceSync( "job.whoop_recovery_sync", "recovery", @@ -153,7 +175,9 @@ export function handleWhoopRecoverySync(jobs: Job[]) { ); } -export function handleWhoopSleepSync(jobs: Job[]) { +export function handleWhoopSleepSync( + jobs: Job[], +): Promise { return runWhoopResourceSync( "job.whoop_sleep_sync", "sleep", @@ -163,7 +187,9 @@ export function handleWhoopSleepSync(jobs: Job[]) { ); } -export function handleWhoopWorkoutSync(jobs: Job[]) { +export function handleWhoopWorkoutSync( + jobs: Job[], +): Promise { return runWhoopResourceSync( "job.whoop_workout_sync", "workout", @@ -173,7 +199,9 @@ export function handleWhoopWorkoutSync(jobs: Job[]) { ); } -export function handleWhoopCycleSync(jobs: Job[]) { +export function handleWhoopCycleSync( + jobs: Job[], +): Promise { // Cycle has no webhook (poll-only), so no fetch-by-id path. return runWhoopResourceSync( "job.whoop_cycle_sync", diff --git a/src/lib/jobs/reminder/withings-sync.ts b/src/lib/jobs/reminder/withings-sync.ts index 96be0b354..153283b58 100644 --- a/src/lib/jobs/reminder/withings-sync.ts +++ b/src/lib/jobs/reminder/withings-sync.ts @@ -7,6 +7,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError, recordWithingsSync } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { retryDueWithingsWebhookSubscriptions, @@ -50,15 +51,20 @@ export interface WithingsSleepSyncPayload { */ export async function handleWithingsFallbackSync( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.withings_sync", async (evt) => { + return withBackgroundEvent("job.withings_sync", async (evt) => { const prisma = getWorkerPrisma(); try { recordWithingsSync(); + // The subscription repair is an auxiliary leg, not the pass this queue is + // named for. Failing the job on it would skip the fallback sync that every + // user relies on when a webhook goes missing, so it rides out as a fact. + let subscriptionRepairFailed = false; try { await retryDueWithingsWebhookSubscriptions(); } catch { + subscriptionRepairFailed = true; evt.addWarning( "Withings subscription repair failed; continuing fallback sync", ); @@ -68,10 +74,17 @@ export async function handleWithingsFallbackSync( }); if (connections.length === 0) { - return; + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + subscription_repair_failed: subscriptionRepairFailed, + }); } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const connection of connections) { @@ -87,6 +100,7 @@ export async function handleWithingsFallbackSync( }); } } catch { + usersFailed++; evt.addWarning(`Fallback sync failed for user ${connection.userId}`); } } @@ -99,6 +113,13 @@ export async function handleWithingsFallbackSync( measurements_imported: measurementsImported, }, }); + return jobDone({ + total: connections.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + subscription_repair_failed: subscriptionRepairFailed, + }); } catch (err) { evt.setError(err); recordError(); @@ -127,8 +148,8 @@ export async function handleWithingsFallbackSync( */ export async function handleWithingsEcgSync( jobs: Job[], -) { - await withBackgroundEvent("job.withings_ecg_sync", async (evt) => { +): Promise { + return withBackgroundEvent("job.withings_ecg_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ @@ -152,9 +173,17 @@ export async function handleWithingsEcgSync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + recordings_imported: 0, + }); + } let usersSynced = 0; + let usersFailed = 0; let recordingsImported = 0; for (const { userId, startdate, enddate } of targets) { try { @@ -165,6 +194,7 @@ export async function handleWithingsEcgSync( recordingsImported += await syncUserEcg(userId, options); usersSynced++; } catch (err) { + usersFailed++; evt.addWarning(`Withings ECG sync failed for user ${userId}: ${err}`); } } @@ -177,6 +207,12 @@ export async function handleWithingsEcgSync( recordings_imported: recordingsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + recordings_imported: recordingsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -203,8 +239,8 @@ export async function handleWithingsEcgSync( */ export async function handleWithingsActivitySync( jobs: Job[], -) { - await withBackgroundEvent("job.withings_activity_sync", async (evt) => { +): Promise { + return withBackgroundEvent("job.withings_activity_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -220,9 +256,17 @@ export async function handleWithingsActivitySync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const { userId } of targets) { try { @@ -235,6 +279,7 @@ export async function handleWithingsActivitySync( }); } } catch { + usersFailed++; evt.addWarning(`Withings activity sync failed for user ${userId}`); } } @@ -247,6 +292,12 @@ export async function handleWithingsActivitySync( measurements_imported: measurementsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -262,8 +313,8 @@ export async function handleWithingsActivitySync( */ export async function handleWithingsSleepSync( jobs: Job[], -) { - await withBackgroundEvent("job.withings_sleep_sync", async (evt) => { +): Promise { + return withBackgroundEvent("job.withings_sleep_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -278,9 +329,17 @@ export async function handleWithingsSleepSync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const { userId } of targets) { try { @@ -293,6 +352,7 @@ export async function handleWithingsSleepSync( }); } } catch { + usersFailed++; evt.addWarning(`Withings sleep sync failed for user ${userId}`); } } @@ -305,6 +365,12 @@ export async function handleWithingsSleepSync( measurements_imported: measurementsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index c37b6fc73..5497174ce 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -30,6 +30,7 @@ import { type S3Like, } from "@/lib/jobs/offhost-backup"; import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; export const RESTORE_DRILL_QUEUE = "data-restore-drill"; @@ -149,9 +150,11 @@ export async function runRestoreDrill( * page); every other failure goes through `reportWorkerError` so the * operator hears about a rotting backup chain. */ -export async function handleRestoreDrill(jobs: Job[]): Promise { +export async function handleRestoreDrill( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.restore_drill", async (evt) => { + return withBackgroundEvent("job.restore_drill", async (evt) => { try { const report = await runRestoreDrill(); evt.addMeta("restore_drill_object_key", report.objectKey); @@ -181,13 +184,31 @@ export async function handleRestoreDrill(jobs: Job[]): Promise { { objectKey: report.objectKey, ageDays: report.ageDays }, ); } + // A stale chain is a verdict about the uploader, not a failure of the + // drill: the artefact was fetched, decrypted and parsed, and the page + // above already reached the operator. Failing the job here would retry a + // reading that cannot change until the next upload lands. + return jobDone({ + restore_drill_object_key: report.objectKey, + restore_drill_age_days: report.ageDays, + restore_drill_ciphertext_bytes: report.ciphertextBytes, + restore_drill_plaintext_bytes: report.plaintextBytes, + restore_drill_measurements: report.recordCounts.measurements, + restore_drill_medications: report.recordCounts.medications, + restore_drill_intake_events: report.recordCounts.intakeEvents, + restore_drill_mood_entries: report.recordCounts.moodEntries, + restore_drill_stale: report.stale, + }); } catch (err) { if (err instanceof OffhostBackupNotConfiguredError) { evt.addWarning(`restore-drill skipped: ${err.message}`); - return; + // Most self-hosters never set the S3 vars. Nothing to verify is not a + // failed verification, and a monthly failed job would be noise. + return jobDone({ skipped: "offhost backup not configured" }); } evt.addWarning(`restore-drill failed: ${err}`); await reportWorkerError(RESTORE_DRILL_QUEUE, err); + return jobFailed("restore drill failed", err); } }); } diff --git a/src/lib/jobs/run-job.ts b/src/lib/jobs/run-job.ts index 27fcd268a..05a180084 100644 --- a/src/lib/jobs/run-job.ts +++ b/src/lib/jobs/run-job.ts @@ -45,6 +45,12 @@ export class JobFailure extends Error { this.name = "JobFailure"; this.queue = queue; this.reason = reason; + // Carry the thrown error's stack. Without this the only stack the + // operator ever sees points at `runJob`, which is the same for all 101 + // queues and says nothing about where the work actually broke. + if (cause instanceof Error && cause.stack) { + this.stack = `${this.name}: ${this.message}\ncaused by ${cause.stack}`; + } } } diff --git a/src/lib/jobs/workout-insight-generate.ts b/src/lib/jobs/workout-insight-generate.ts index 4a690d088..a5854f285 100644 --- a/src/lib/jobs/workout-insight-generate.ts +++ b/src/lib/jobs/workout-insight-generate.ts @@ -48,6 +48,7 @@ import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-work import { defaultLocale, locales, type Locale } from "@/lib/i18n/config"; import { finalizeStatusSummary } from "@/lib/insights/status-shared"; import { runStatusCompletion } from "@/lib/insights/status-provider"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { annotate } from "@/lib/logging/context"; import { withBackgroundEvent } from "@/lib/logging/background"; import { resolveModuleMap } from "@/lib/modules/gate"; @@ -524,10 +525,14 @@ export async function runWorkoutInsightGenerate( export async function handleWorkoutInsightGenerate( jobs: Job[], -): Promise { - await withBackgroundEvent("job.workout_insight_generate", async (evt) => { +): Promise { + return withBackgroundEvent("job.workout_insight_generate", async (evt) => { + let generated = 0; + let skipped = 0; for (const job of jobs) { const outcome = await runWorkoutInsightGenerate(job.data); + if (outcome.status === "skipped") skipped++; + else generated++; evt.addMeta( "workout_insight", outcome.status === "skipped" @@ -535,5 +540,9 @@ export async function handleWorkoutInsightGenerate( : `generated:${outcome.providerType}`, ); } + // Every gate in front of the provider returns `skipped`, so a pass that + // wrote no paragraph at all is still a pass that ran. Only a transient + // fault escapes `runWorkoutInsightGenerate`, and that one throws. + return jobDone({ total: jobs.length, generated, skipped }); }); } From 0aa5cec17d173b75465411f31fa45b6d7df935e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Mon, 27 Jul 2026 23:08:50 +0200 Subject: [PATCH 3/6] feat(jobs): bind every queue through createAndWork and ban the bare call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 104 bindings across the five registrars now go through createAndWork, whose handler type is (jobs) => Promise. The forty-two inline handlers return what they did. A handler that falls off the end no longer compiles, and the healthlog/job-handler-outcome rule refuses a bare boss.work( anywhere under src/lib/jobs except the one module that owns the wrapper. The rule's header states its own limit: it catches the omission, not a handler that claims ok:true after a swallow. Retry policy, reviewed queue by queue because the rethrow changes what runs, not only what is visible: - Twenty-one cron passes carry cronIsTheRetry (retryLimit 0). They are the retention DELETEs, the OAuth-state prunes, the hourly maintenance sweeps and the nightly per-sample drain — every one of them a statement that will fail identically a second later. pg-boss defaults to two retries with no delay, so converting them without this would have run the same doomed statement three times a night. The next tick is the retry and it arrives with the whole gap as backoff. - The policy rides on the schedule, not on createQueue: create_queue() ends in ON CONFLICT DO NOTHING, so a queue-level default would be inert on every already-running instance. - Everything else keeps its retries deliberately. A TLS probe, an off-host upload, a restore drill and a provider poll all fail transiently, and the reminder ticks only fail at job level when the pass itself could not run. --- eslint-plugins/healthlog/index.js | 4 + .../healthlog/job-handler-outcome.js | 107 +++++ eslint.config.mjs | 7 + .../reminder/register-integration-sync.ts | 157 ++++++-- src/lib/jobs/reminder/register-maintenance.ts | 346 +++++++++++++--- src/lib/jobs/reminder/register-rollup.ts | 185 ++++++++- src/lib/jobs/reminder/register-status.ts | 371 ++++++++++++------ src/lib/jobs/reminder/registrar-shared.ts | 26 ++ 8 files changed, 975 insertions(+), 228 deletions(-) create mode 100644 eslint-plugins/healthlog/job-handler-outcome.js diff --git a/eslint-plugins/healthlog/index.js b/eslint-plugins/healthlog/index.js index d1529a8bc..8147b1540 100644 --- a/eslint-plugins/healthlog/index.js +++ b/eslint-plugins/healthlog/index.js @@ -8,6 +8,8 @@ * - `no-raw-palette-color` — ban raw Tailwind palette utilities in app UI. * - `spacing-scale` — no pt-/pb- overrides on gap-based Card slots, * and no off-scale `5` step on a bg-card shell. + * - `job-handler-outcome` — pg-boss bindings route through createAndWork, + * whose handler returns a JobOutcome. */ "use strict"; @@ -17,6 +19,7 @@ const safeFetchRequired = require("./safe-fetch-required.js"); const apiFetchRequired = require("./api-fetch-required.js"); const noRawPaletteColor = require("./no-raw-palette-color.js"); const spacingScale = require("./spacing-scale.js"); +const jobHandlerOutcome = require("./job-handler-outcome.js"); module.exports = { rules: { @@ -30,5 +33,6 @@ module.exports = { // named rule (now error-level too; the staged warn phase ended with // the semantic sweep). See the rule header. "no-dracula-utility": noRawPaletteColor, + "job-handler-outcome": jobHandlerOutcome, }, }; diff --git a/eslint-plugins/healthlog/job-handler-outcome.js b/eslint-plugins/healthlog/job-handler-outcome.js new file mode 100644 index 000000000..9dfdbafc4 --- /dev/null +++ b/eslint-plugins/healthlog/job-handler-outcome.js @@ -0,0 +1,107 @@ +/** + * @fileoverview ESLint rule — a pg-boss queue binding must go through + * `createAndWork`, never a bare `boss.work(`. + * + * `boss.work(queue, options, handler)` completes a job whenever the handler's + * promise resolves, whatever it resolves to. Every binding in this tree used + * to be written that way, so a handler that caught an error, logged a warning + * and returned looked exactly like one that did its work. `createAndWork` + * (`src/lib/jobs/reminder/registrar-shared.ts`) types its handler as + * `(jobs) => Promise` and wraps it in `runJob`, which fails the + * pg-boss job on `ok: false`. Routing every binding through it is what makes + * "the handler returned nothing" a compile error rather than a silent pass. + * + * WHAT THIS RULE PROVES, AND WHAT IT DOES NOT + * + * It proves that no queue in this tree is bound by a path that accepts a + * void handler. That is all it proves. It does NOT detect a handler that + * returns `{ ok: true }` after swallowing an exception — that compiles, it + * passes this rule, and it is exactly as invisible as the defect the outcome + * type was written for. The rule converts an omission (forgetting to signal + * failure) into a commission (claiming a success that did not happen). The + * second is a smaller class and a reviewable one; it is not zero. + * + * Two further evasions are known and unhandled: aliasing the method + * (`const w = boss.work.bind(boss)`, `const { work } = boss`) and calling + * `work` on an object not named `boss`. Both are syntactic dodges a reviewer + * would see in the diff; neither is defended against here. + * + * Scope: `src/lib/jobs/`, excluding the registrar module that owns the one + * legal `boss.work` call and excluding test files, which mock `boss.work` + * and assert against it directly. + * + * @see src/lib/jobs/run-job.ts + * @see src/lib/jobs/job-outcome.ts + * @see src/lib/jobs/reminder/registrar-shared.ts + */ + +"use strict"; + +// The one module allowed to call `boss.work` — `createAndWork` lives here +// and is the wrapper every other binding has to use. +const EXEMPT_FILES = ["src/lib/jobs/reminder/registrar-shared.ts"]; + +// The worker tree. A `boss.work` outside it would not be a queue binding. +const ENFORCED_ROOTS = ["src/lib/jobs/"]; + +function toPosix(filename) { + return filename.replace(/\\/g, "/"); +} + +function isTestFile(posix) { + return ( + /\.test\.[cm]?[jt]sx?$/.test(posix) || + /\.spec\.[cm]?[jt]sx?$/.test(posix) || + posix.includes("/__tests__/") || + posix.includes("/__mocks__/") + ); +} + +function isEnforced(filename) { + const posix = toPosix(filename); + if (!ENFORCED_ROOTS.some((root) => posix.includes(root))) return false; + if (EXEMPT_FILES.some((f) => posix.includes(f))) return false; + if (isTestFile(posix)) return false; + return true; +} + +/** @type {import("eslint").Rule.RuleModule} */ +const jobHandlerOutcomeRule = { + meta: { + type: "problem", + docs: { + description: + "pg-boss queue bindings must go through createAndWork so the handler returns a JobOutcome.", + }, + schema: [], + messages: { + bareBossWork: + 'boss.work() completes a job whenever the handler resolves, so a swallowed failure reads as success. Bind the queue with createAndWork(boss, QUEUE, options, handler) from "./registrar-shared" — its handler type is (jobs) => Promise and runJob fails the job on ok:false.', + }, + }, + create(context) { + const filename = context.filename ?? context.getFilename?.(); + if (!filename || !isEnforced(filename)) { + return {}; + } + + return { + CallExpression(node) { + const callee = node.callee; + if ( + callee.type !== "MemberExpression" || + callee.computed || + callee.property.type !== "Identifier" || + callee.property.name !== "work" || + callee.object.type !== "Identifier" || + callee.object.name !== "boss" + ) { + return; + } + context.report({ node, messageId: "bareBossWork" }); + }, + }; + }, +}; + +module.exports = jobHandlerOutcomeRule; diff --git a/eslint.config.mjs b/eslint.config.mjs index 96ad3e1fd..82fa8cfae 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -75,6 +75,13 @@ const eslintConfig = defineConfig([ // trip it. Density lives on the Card itself (`gap-2 py-3 md:py-4`) or on // a swept `p-4 md:p-6` shell. See the rule header. "healthlog/spacing-scale": "error", + // Every pg-boss queue binding under src/lib/jobs goes through + // `createAndWork`, whose handler type is `(jobs) => Promise`. + // A bare `boss.work(` accepts a handler that resolves to nothing, which + // is how a swallowed failure came to read as a completed job. The rule + // catches the omission only — a handler that returns `{ ok: true }` + // after swallowing still passes, and its header says so. + "healthlog/job-handler-outcome": "error", }, }, // v1.28.17 — every recharts-rendering component funnels through the diff --git a/src/lib/jobs/reminder/register-integration-sync.ts b/src/lib/jobs/reminder/register-integration-sync.ts index 85a5e1bf2..54e753948 100644 --- a/src/lib/jobs/reminder/register-integration-sync.ts +++ b/src/lib/jobs/reminder/register-integration-sync.ts @@ -9,8 +9,8 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The queue-wiring - * guards (`withings-queues`, `whoop-queues`, `fitbit-queues`, + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The + * queue-wiring guards (`withings-queues`, `whoop-queues`, `fitbit-queues`, * `nightscout-queues`, `sleep-timeline-queues`) read THIS module. */ import { PgBoss } from "pg-boss"; @@ -78,8 +78,11 @@ import { type IntegrationBackfillAdmissionPayload, } from "@/lib/jobs/integration-backfill-admission"; import { workerLog } from "./shared"; +import { jobDone } from "@/lib/jobs/job-outcome"; import { createAndSchedule, + createAndWork, + cronIsTheRetry, type QueuePolicyTable, type ScheduleEntry, } from "./registrar-shared"; @@ -333,7 +336,16 @@ const schedules: ScheduleEntry[] = [ // v1.32.23 — hourly ECG catch-net (:41) so watch-only accounts pull their // strips without a webhook or a scale. [WITHINGS_ECG_SYNC_QUEUE, WITHINGS_ECG_SYNC_CRON], - [WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, WITHINGS_OAUTH_STATE_CLEANUP_CRON], + // The five OAuth-state / handoff prunes carry `cronIsTheRetry`: their + // handlers used to warn and carry on, they now fail the job, and a DELETE + // that could not run will not run any better a second later. The daily tick + // is the retry. The sync crons above keep the default retries — a provider + // outage is transient and worth retrying inside the hour. + [ + WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, + WITHINGS_OAUTH_STATE_CLEANUP_CRON, + cronIsTheRetry, + ], // v1.11.0 — WHOOP poll-fallback crons. Recovery/sleep/workout catch // dropped webhooks; cycle is the sole driver (no webhook). Staggered off // the Withings crons so the hourly ticks don't pile up on one boss poll. @@ -342,11 +354,19 @@ const schedules: ScheduleEntry[] = [ [WHOOP_WORKOUT_SYNC_QUEUE, WHOOP_WORKOUT_SYNC_CRON], [WHOOP_CYCLE_SYNC_QUEUE, WHOOP_CYCLE_SYNC_CRON], // v1.11.0 — daily 03:22 Europe/Berlin prune for expired WHOOP OAuth states. - [WHOOP_OAUTH_STATE_CLEANUP_QUEUE, WHOOP_OAUTH_STATE_CLEANUP_CRON], + [ + WHOOP_OAUTH_STATE_CLEANUP_QUEUE, + WHOOP_OAUTH_STATE_CLEANUP_CRON, + cronIsTheRetry, + ], // v1.12.0 — hourly Fitbit poll (:08, staggered off WHOOP/Withings) + the // daily 03:24 Europe/Berlin prune for expired Fitbit OAuth states. [FITBIT_SYNC_QUEUE, FITBIT_SYNC_CRON], - [FITBIT_OAUTH_STATE_CLEANUP_QUEUE, FITBIT_OAUTH_STATE_CLEANUP_CRON], + [ + FITBIT_OAUTH_STATE_CLEANUP_QUEUE, + FITBIT_OAUTH_STATE_CLEANUP_CRON, + cronIsTheRetry, + ], // v1.26.0 — hourly Google Health poll (:18, staggered off WHOOP/Fitbit/ // Nightscout/Polar/Oura) + the daily 03:26 Europe/Berlin prune for expired // Google Health OAuth states. @@ -354,6 +374,7 @@ const schedules: ScheduleEntry[] = [ [ GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, + cronIsTheRetry, ], // v1.17.0 — hourly Nightscout CGM poll (:11, staggered off the other sync // ticks). @@ -364,7 +385,11 @@ const schedules: ScheduleEntry[] = [ // v1.28.x — hourly Strava (:17) OAuth poll. [STRAVA_SYNC_QUEUE, STRAVA_SYNC_CRON], // v1.30.x — daily 03:28 Europe/Berlin prune for expired native OIDC handoffs. - [OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, OIDC_NATIVE_HANDOFF_CLEANUP_CRON], + [ + OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, + OIDC_NATIVE_HANDOFF_CLEANUP_CRON, + cronIsTheRetry, + ], ]; /** @@ -450,19 +475,30 @@ export async function registerIntegrationSyncQueues( // reaches this shared durable queue before doing full-history work. The // common group is enforced in PostgreSQL across worker processes/nodes. - await boss.work( + await createAndWork( + boss, INTEGRATION_BACKFILL_ADMISSION_QUEUE, { localConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, groupConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, }, async (jobs) => { + // The drain has no per-job error isolation: a runner that throws fails + // the job and the admission retry policy applies. So reaching the end + // means every admitted job did its work, and the counts each runner + // already returns are what the pass reports. + let importedTotal = 0; + let removedTotal = 0; + let deletedTotal = 0; + let markersTotal = 0; + let linkedTotal = 0; for (const job of jobs) { const { kind, data } = job.data; const { userId } = data; switch (kind) { case "whoop-backfill": { const { imported } = await runWhoopBackfillForUser(userId); + importedTotal += imported; workerLog( "info", `[whoop-backfill] user=${userId} imported=${imported}`, @@ -471,6 +507,7 @@ export async function registerIntegrationSyncQueues( } case "fitbit-backfill": { const { imported } = await runFitbitBackfillForUser(userId); + importedTotal += imported; workerLog( "info", `[fitbit-backfill] user=${userId} imported=${imported}`, @@ -479,6 +516,7 @@ export async function registerIntegrationSyncQueues( } case "google-health-backfill": { const { imported } = await runGoogleHealthBackfillForUser(userId); + importedTotal += imported; workerLog( "info", `[google-health-backfill] user=${userId} imported=${imported}`, @@ -488,6 +526,7 @@ export async function registerIntegrationSyncQueues( case "google-health-sleep-repair": { const { imported } = await runGoogleHealthSleepRepairForUser(userId); + importedTotal += imported; workerLog( "info", `[google-health-sleep-repair] user=${userId} imported=${imported}`, @@ -497,6 +536,8 @@ export async function registerIntegrationSyncQueues( case "fitbit-sleep-repair": { const { imported, removed } = await runFitbitSleepRepairForUser(userId); + importedTotal += imported; + removedTotal += removed; workerLog( "info", `[fitbit-sleep-repair] user=${userId} imported=${imported} removed=${removed}`, @@ -513,6 +554,8 @@ export async function registerIntegrationSyncQueues( userId, data.provider, ); + deletedTotal += deleted; + importedTotal += imported; workerLog( "info", `[sleep-timeline-backfill] user=${userId} provider=${data.provider} deleted=${deleted} imported=${imported}`, @@ -522,6 +565,8 @@ export async function registerIntegrationSyncQueues( case "lab-biomarker-backfill": { const { markers, linked } = await runLabBiomarkerBackfillForUser(userId); + markersTotal += markers; + linkedTotal += linked; workerLog( "info", `[lab-biomarker-backfill] user=${userId} markers=${markers} linked=${linked}`, @@ -530,6 +575,7 @@ export async function registerIntegrationSyncQueues( } case "strava-backfill": { const { imported } = await runStravaBackfillForUser(userId); + importedTotal += imported; workerLog( "info", `[strava-backfill] user=${userId} imported=${imported}`, @@ -538,30 +584,43 @@ export async function registerIntegrationSyncQueues( } } } + return jobDone({ + jobs: jobs.length, + imported: importedTotal, + removed: removedTotal, + deleted: deletedTotal, + markers: markersTotal, + linked: linkedTotal, + }); }, ); - await boss.work( + await createAndWork( + boss, WITHINGS_SYNC_QUEUE, { localConcurrency: 1 }, handleWithingsFallbackSync, ); - await boss.work( + await createAndWork( + boss, WITHINGS_ACTIVITY_QUEUE, { localConcurrency: 1 }, handleWithingsActivitySync, ); - await boss.work( + await createAndWork( + boss, WITHINGS_SLEEP_QUEUE, { localConcurrency: 1 }, handleWithingsSleepSync, ); - await boss.work( + await createAndWork( + boss, WITHINGS_ECG_SYNC_QUEUE, { localConcurrency: 1 }, handleWithingsEcgSync, ); - await boss.work( + await createAndWork( + boss, WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleWithingsOAuthStateCleanup, @@ -569,29 +628,34 @@ export async function registerIntegrationSyncQueues( // v1.11.0 — WHOOP per-resource sync handlers. Webhook-driven per-user + // cron full-iteration. Serial concurrency so a backfill-heavy tick never // crowds the request pool and stays inside WHOOP's 100 req/min app cap. - await boss.work( + await createAndWork( + boss, WHOOP_RECOVERY_SYNC_QUEUE, { localConcurrency: 1 }, handleWhoopRecoverySync, ); - await boss.work( + await createAndWork( + boss, WHOOP_SLEEP_SYNC_QUEUE, { localConcurrency: 1 }, handleWhoopSleepSync, ); - await boss.work( + await createAndWork( + boss, WHOOP_WORKOUT_SYNC_QUEUE, { localConcurrency: 1 }, handleWhoopWorkoutSync, ); - await boss.work( + await createAndWork( + boss, WHOOP_CYCLE_SYNC_QUEUE, { localConcurrency: 1 }, handleWhoopCycleSync, ); // v1.11.0 — self-converging WHOOP backfill. This source queue retains the // per-connection singleton and forwards admitted work to the shared drain. - await boss.work( + await createAndWork( + boss, WHOOP_BACKFILL_QUEUE, { localConcurrency: WHOOP_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -601,29 +665,34 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); - await boss.work( + await createAndWork( + boss, WHOOP_OAUTH_STATE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleWhoopOAuthStateCleanup, ); // v1.30.x — daily sweep for the native OIDC handoff ledger. - await boss.work( + await createAndWork( + boss, OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, { localConcurrency: 1 }, handleOidcNativeHandoffCleanup, ); // v1.12.0 — Fitbit poll-sync (cron full-iteration; no webhook). Serial // concurrency so a backfill-heavy tick never crowds the request pool. - await boss.work( + await createAndWork( + boss, FITBIT_SYNC_QUEUE, { localConcurrency: 1 }, handleFitbitSync, ); // v1.12.0 — self-converging Fitbit backfill. This source queue retains the // per-connection singleton and forwards admitted work to the shared drain. - await boss.work( + await createAndWork( + boss, FITBIT_BACKFILL_QUEUE, { localConcurrency: FITBIT_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -633,12 +702,14 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); // One-shot Fitbit sleep duplicate repair. This source queue retains the // per-connection singleton and forwards the watermark-safe bounded sleep // re-read to the shared drain. - await boss.work( + await createAndWork( + boss, FITBIT_SLEEP_REPAIR_QUEUE, { localConcurrency: FITBIT_SLEEP_REPAIR_CONCURRENCY }, async (jobs) => { @@ -648,16 +719,19 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); - await boss.work( + await createAndWork( + boss, FITBIT_OAUTH_STATE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleFitbitOAuthStateCleanup, ); // v1.26.0 — Google Health poll-sync (cron full-iteration; no webhook). Serial // concurrency so a backfill-heavy tick never crowds the request pool. - await boss.work( + await createAndWork( + boss, GOOGLE_HEALTH_SYNC_QUEUE, { localConcurrency: 1 }, handleGoogleHealthSync, @@ -665,7 +739,8 @@ export async function registerIntegrationSyncQueues( // v1.26.0 — self-converging Google Health backfill. This source queue // retains the per-connection singleton and forwards admitted work to the // shared drain. - await boss.work( + await createAndWork( + boss, GOOGLE_HEALTH_BACKFILL_QUEUE, { localConcurrency: GOOGLE_HEALTH_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -675,12 +750,14 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); // v1.28.x — one-shot Google Health sleep duplicate repair. This source // queue retains the per-connection singleton and forwards the watermark-safe // full sleep-history re-read to the shared drain. - await boss.work( + await createAndWork( + boss, GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, { localConcurrency: GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY }, async (jobs) => { @@ -690,9 +767,11 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); - await boss.work( + await createAndWork( + boss, GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleGoogleHealthOAuthStateCleanup, @@ -700,7 +779,8 @@ export async function registerIntegrationSyncQueues( // v1.17.1 — one-shot sleep-timeline backfill. This source queue retains the // per-(user, provider) singleton and forwards admitted work to the shared // drain. - await boss.work( + await createAndWork( + boss, SLEEP_TIMELINE_BACKFILL_QUEUE, { localConcurrency: SLEEP_TIMELINE_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -710,11 +790,13 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); // v1.18.1 — one-shot lab-biomarker backfill. This source queue retains the // per-user singleton and forwards admitted work to the shared drain. - await boss.work( + await createAndWork( + boss, LAB_BIOMARKER_BACKFILL_QUEUE, { localConcurrency: LAB_BIOMARKER_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -724,12 +806,14 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); // v1.17.0 — Nightscout CGM poll-cohort sync. The hourly cron tick (no // `userId`) walks every configured instance; one user's unreachable host is // warned, not fatal. - await boss.work( + await createAndWork( + boss, NIGHTSCOUT_SYNC_QUEUE, { localConcurrency: 1 }, handleNightscoutSync, @@ -737,12 +821,14 @@ export async function registerIntegrationSyncQueues( // v1.17.0 (F4) — Polar + Oura OAuth poll-cohort sync. The hourly cron tick // (no `userId`) re-walks every connected user; one user's revoked grant is // warned, not fatal. - await boss.work( + await createAndWork( + boss, POLAR_SYNC_QUEUE, { localConcurrency: 1 }, handlePolarSync, ); - await boss.work( + await createAndWork( + boss, OURA_SYNC_QUEUE, { localConcurrency: 1 }, handleOuraSync, @@ -750,14 +836,16 @@ export async function registerIntegrationSyncQueues( // v1.28.x — Strava OAuth poll-cohort sync (poll-only; webhook deferred). The // hourly cron tick (no `userId`) re-walks every connected user; one user's // revoked grant is warned, not fatal. - await boss.work( + await createAndWork( + boss, STRAVA_SYNC_QUEUE, { localConcurrency: 1 }, handleStravaSync, ); // v1.28.x — self-converging Strava backfill. This source queue retains the // per-connection singleton and forwards admitted work to the shared drain. - await boss.work( + await createAndWork( + boss, STRAVA_BACKFILL_QUEUE, { localConcurrency: STRAVA_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -767,6 +855,7 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); return allQueues; diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 921f1595e..1f4f327ad 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -10,7 +10,7 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The * `measurement-tombstone-cleanup`, `withings-oauth-state-cleanup`, * `intake-slot-dedup-queue`, `tls-pin-monitor-queue`, and `geo-backfill` * guards read THIS module. @@ -131,9 +131,12 @@ import { type EnvironmentFetchPayload, } from "@/lib/jobs/environment-fetch"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared"; import { createAndSchedule, + createAndWork, + cronIsTheRetry, type QueuePolicyTable, type ScheduleEntry, } from "./registrar-shared"; @@ -411,24 +414,35 @@ const allQueues = [ DOCUMENT_SUMMARY_CATCHUP_QUEUE, ]; +// `cronIsTheRetry` (retryLimit 0) marks the passes whose handler now returns +// `jobFailed` where it used to warn and carry on, AND whose failure repeats: +// a bulk DELETE or UPDATE that timed out will time out again on an immediate +// retry, so pg-boss's default two would run the same doomed statement three +// times a night. The queues without it are the ones whose failure is +// transient — the TLS probe, the off-host upload, the restore drill, the +// per-user environment fetch — and those still want the default retries. const schedules: ScheduleEntry[] = [ // v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload) // that fans out one per-user environment fetch per opted-in account. [ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON], [DATA_BACKUP_QUEUE, DATA_BACKUP_CRON], - [RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON], - [IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON], + [RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry], + [IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry], [AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON], - [STEP_UP_ELEVATION_CLEANUP_QUEUE, STEP_UP_ELEVATION_CLEANUP_CRON], + [ + STEP_UP_ELEVATION_CLEANUP_QUEUE, + STEP_UP_ELEVATION_CLEANUP_CRON, + cronIsTheRetry, + ], [OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON], [RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON], - [HOST_METRIC_QUEUE, HOST_METRIC_CRON], - [FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON], + [HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry], + [FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry], // v1.4.37 — hourly geo backfill. The helper is idempotent + capped // at 5 000 rows per pass; running it at :40 every hour catches the // long tail of audit rows that landed with the offline MMDB // missing or the online provider unreachable. - [GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON], + [GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON, cronIsTheRetry], // v1.12.2 ios-coord — every-6-hour TLS leaf SPKI probe (:07 off the // hourly sync crons). Surfaces a pinned-leaf rotation well inside the // ≥11-day re-pin window the iOS release owner needs. @@ -443,14 +457,18 @@ const schedules: ScheduleEntry[] = [ // entities. Flips IN_USE rows whose 30-day clock has blown to // EXPIRED at 03:30 Europe/Berlin (in the existing 02:xx–03:xx // maintenance window, right after idempotency-cleanup). - [MEDICATION_INVENTORY_EXPIRE_QUEUE, MEDICATION_INVENTORY_EXPIRE_CRON], + [ + MEDICATION_INVENTORY_EXPIRE_QUEUE, + MEDICATION_INVENTORY_EXPIRE_CRON, + cronIsTheRetry, + ], // v1.4.46 — hourly auto-skip for medication intakes the user never // marked. Cron `5 * * * *` slots off the top-of-the-hour // reminder-check (:00) so the hourly ticks don't pile up on the same // boss poll. - [INTAKE_AUTO_SKIP_QUEUE, INTAKE_AUTO_SKIP_CRON], + [INTAKE_AUTO_SKIP_QUEUE, INTAKE_AUTO_SKIP_CRON, cronIsTheRetry], // v0.5.4 ios-coord — daily mood-reminder dispatch-ledger retention sweep. - [MOOD_REMINDER_CLEANUP_QUEUE, MOOD_REMINDER_CLEANUP_CRON], + [MOOD_REMINDER_CLEANUP_QUEUE, MOOD_REMINDER_CLEANUP_CRON, cronIsTheRetry], // v1.15.19 — daily 03:28 Europe/Berlin duplicate dose-slot dedup // discovery. The empty cron payload (no `userId`) is the handler's // signal to run the discovery fan-out instead of a per-user pass, so @@ -458,20 +476,24 @@ const schedules: ScheduleEntry[] = [ // a day instead of waiting for the next worker reboot. [INTAKE_SLOT_DEDUP_QUEUE, INTAKE_SLOT_DEDUP_CRON], // v1.4.49 — daily 03:35 Europe/Berlin prune for push_attempts. - [PUSH_ATTEMPT_CLEANUP_QUEUE, PUSH_ATTEMPT_CLEANUP_CRON], + [PUSH_ATTEMPT_CLEANUP_QUEUE, PUSH_ATTEMPT_CLEANUP_CRON, cronIsTheRetry], // v1.31.0 — daily 04:10 Europe/Berlin prune for arrival_reactions. - [ARRIVAL_REACTION_CLEANUP_QUEUE, ARRIVAL_REACTION_CLEANUP_CRON], + [ + ARRIVAL_REACTION_CLEANUP_QUEUE, + ARRIVAL_REACTION_CLEANUP_CRON, + cronIsTheRetry, + ], // v1.7.0 — daily 03:40 Europe/Berlin prune for expired measurement // tombstones. [MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, MEASUREMENT_TOMBSTONE_CLEANUP_CRON], // v1.18.7 — daily 03:50 Europe/Berlin prune for stale Coach history. - [COACH_MESSAGE_CLEANUP_QUEUE, COACH_MESSAGE_CLEANUP_CRON], + [COACH_MESSAGE_CLEANUP_QUEUE, COACH_MESSAGE_CLEANUP_CRON, cronIsTheRetry], // v1.23 — daily 03:55 Europe/Berlin note-encryption backfill discovery. // The empty cron payload (no `userId`) is the handler's signal to fan out // one per-user job per account still holding a plaintext note. [NOTE_ENCRYPTION_BACKFILL_QUEUE, NOTE_ENCRYPTION_BACKFILL_CRON], // v1.24 — daily 04:00 Europe/Berlin prune for dead MCP connector tokens. - [MCP_TOKEN_CLEANUP_QUEUE, MCP_TOKEN_CLEANUP_CRON], + [MCP_TOKEN_CLEANUP_QUEUE, MCP_TOKEN_CLEANUP_CRON, cronIsTheRetry], // v1.25 — daily 04:05 Europe/Berlin medication-note encryption backfill // discovery. The empty cron payload (no `userId`) is the handler's signal to // fan out one per-user job per account still holding a plaintext medication @@ -479,14 +501,14 @@ const schedules: ScheduleEntry[] = [ [MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, MED_NOTES_ENCRYPTION_BACKFILL_CRON], // Document vault — daily 04:10 Europe/Berlin purge for tombstoned // documents past the 30-day undo grace. - [DOCUMENT_PURGE_QUEUE, DOCUMENT_PURGE_CRON], + [DOCUMENT_PURGE_QUEUE, DOCUMENT_PURGE_CRON, cronIsTheRetry], // v1.32.1 (issue #588) — every-15-minute orphan-ImportJob sweep. Re-runs // the same reconcile the boot path uses, so a stuck "unpacking" row // whose worker crashed/restarted without the boot-time pass catching it // (heartbeat not yet stale, pg-boss job not yet expired) still converges // to a visible `failed` state within two ticks instead of staying stuck // forever on an otherwise-healthy worker. - [IMPORT_JOB_RECONCILE_QUEUE, IMPORT_JOB_RECONCILE_CRON], + [IMPORT_JOB_RECONCILE_QUEUE, IMPORT_JOB_RECONCILE_CRON, cronIsTheRetry], ]; /** @@ -584,61 +606,76 @@ export async function registerMaintenanceQueues( ): Promise { await createAndSchedule(boss, allQueues, schedules, queuePolicies); - await boss.work( + await createAndWork( + boss, DATA_BACKUP_QUEUE, { localConcurrency: 1 }, handleDataBackup, ); - await boss.work( + await createAndWork( + boss, RATE_LIMIT_CLEANUP_QUEUE, { localConcurrency: 1 }, handleRateLimitCleanup, ); - await boss.work( + await createAndWork( + boss, IDEMPOTENCY_CLEANUP_QUEUE, { localConcurrency: 1 }, handleIdempotencyCleanup, ); - await boss.work( + await createAndWork( + boss, AUDIT_LOG_CLEANUP_QUEUE, { localConcurrency: 1 }, handleAuditLogCleanup, ); - await boss.work( + await createAndWork( + boss, STEP_UP_ELEVATION_CLEANUP_QUEUE, { localConcurrency: 1 }, handleStepUpElevationCleanup, ); - await boss.work( + await createAndWork( + boss, OFFHOST_BACKUP_QUEUE, { localConcurrency: 1 }, handleOffhostBackup, ); - // prettier-ignore - await boss.work(RESTORE_DRILL_QUEUE, { localConcurrency: 1 }, handleRestoreDrill); - await boss.work( + await createAndWork( + boss, + RESTORE_DRILL_QUEUE, + { localConcurrency: 1 }, + handleRestoreDrill, + ); + await createAndWork( + boss, HOST_METRIC_QUEUE, { localConcurrency: 1 }, handleHostMetricSample, ); - await boss.work( + await createAndWork( + boss, FEEDBACK_AGGREGATOR_QUEUE, { localConcurrency: 1 }, handleFeedbackAggregator, ); - await boss.work( + await createAndWork( + boss, GEO_BACKFILL_QUEUE, { localConcurrency: 1 }, handleGeoBackfill, ); // v1.12.2 ios-coord — TLS leaf SPKI-change monitor. Single-flight: one // short outbound TLS handshake per tick, no benefit to overlapping ticks. - await boss.work( + await createAndWork( + boss, TLS_PIN_MONITOR_QUEUE, { localConcurrency: 1 }, handleTlsPinMonitor, ); - await boss.work( + await createAndWork( + boss, MOOD_REMINDER_CLEANUP_QUEUE, { localConcurrency: 1 }, handleMoodReminderCleanup, @@ -647,7 +684,8 @@ export async function registerMaintenanceQueues( // matches every other cleanup queue; two ticks racing on the same // DELETE statement is wasted work and the second tick's payload // would be a no-op anyway. - await boss.work( + await createAndWork( + boss, PUSH_ATTEMPT_CLEANUP_QUEUE, { localConcurrency: 1 }, handlePushAttemptCleanup, @@ -655,14 +693,16 @@ export async function registerMaintenanceQueues( // v1.31.0 — daily prune of the data-arrival reaction markers. Single-flight // like every other cleanup queue: two ticks racing the same DELETE is // wasted work. - await boss.work( + await createAndWork( + boss, ARRIVAL_REACTION_CLEANUP_QUEUE, { localConcurrency: 1 }, handleArrivalReactionCleanup, ); // v1.7.0 — daily prune of expired measurement tombstones. Single-flight // like every other cleanup queue. - await boss.work( + await createAndWork( + boss, MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleMeasurementTombstoneCleanup, @@ -670,14 +710,16 @@ export async function registerMaintenanceQueues( // v1.18.7 — daily prune of the encrypted Coach conversation history. // Single-flight like every other cleanup queue; two ticks racing on the // same DELETE is wasted work and the second is a no-op. - await boss.work( + await createAndWork( + boss, COACH_MESSAGE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleCoachMessageCleanup, ); // MCP Phase 3 (M2) — daily prune of dead MCP connector access tokens + // long-revoked connection anchors. Single-flight like every other cleanup. - await boss.work( + await createAndWork( + boss, MCP_TOKEN_CLEANUP_QUEUE, { localConcurrency: 1 }, handleMcpTokenCleanup, @@ -685,17 +727,20 @@ export async function registerMaintenanceQueues( // Document vault — daily purge of tombstoned documents past the 30-day // undo grace. Single-flight like every other cleanup queue; the underlying // deleteMany is idempotent so a duplicate tick is a no-op. - await boss.work( + await createAndWork( + boss, DOCUMENT_PURGE_QUEUE, { localConcurrency: 1 }, handleDocumentPurge, ); - await boss.work( + await createAndWork( + boss, PR_DETECTION_QUEUE, { localConcurrency: PR_DETECTION_CONCURRENCY }, handlePrDetection, ); - await boss.work( + await createAndWork( + boss, MEDICATION_INVENTORY_EXPIRE_QUEUE, { localConcurrency: 1 }, handleMedicationInventoryExpire, @@ -704,7 +749,8 @@ export async function registerMaintenanceQueues( // Single-flight: two ticks racing against the same row pile is wasted // work, and the underlying `updateMany` is the canonical idempotent // shape anyway. - await boss.work( + await createAndWork( + boss, INTAKE_AUTO_SKIP_QUEUE, { localConcurrency: 1 }, handleIntakeAutoSkip, @@ -712,48 +758,64 @@ export async function registerMaintenanceQueues( // v1.4.34 — Apple Health export.zip ingest worker. localConcurrency // caps at 1 because the parse loop is CPU-bound and a concurrent // second 1 GB import would race the first for RSS. - await boss.work( + await createAndWork( + boss, APPLE_HEALTH_IMPORT_V2_QUEUE, { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, async (jobs) => { // pg-boss v12 work callbacks always receive an array (batched // worker mode); for our concurrency-1 + batchSize-1 case we just // process each job sequentially. + // + // Each import is independent work, so a failed one does not abort the + // rest of the batch; the first failure is what the job reports. + let failure: JobOutcome | null = null; for (const job of jobs) { - await handleAppleHealthImport(job); + const outcome = await handleAppleHealthImport(job); + if (!outcome.ok && failure === null) failure = outcome; } + return failure ?? jobDone({ jobs: jobs.length }); }, ); // Drain pre-revision-2 backlog by moving each claimed legacy job onto the // isolated v2 queue. This preserves the ImportJob status id without letting // the current parser execute under a revision-1 mirror. - await boss.work( + await createAndWork( + boss, APPLE_HEALTH_IMPORT_LEGACY_QUEUE, { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, async (jobs) => { for (const job of jobs) { await migrateLegacyAppleHealthImport(job); } + return jobDone({ jobs: jobs.length }); }, ); // v1.32.1 (issue #588) — periodic orphan-ImportJob sweep. Single-flight: // the underlying `updateMany` is idempotent and two ticks racing the same // read-then-write pass would just be wasted work. - await boss.work( + await createAndWork( + boss, IMPORT_JOB_RECONCILE_QUEUE, { localConcurrency: 1 }, handleImportJobReconcileTick, ); - await boss.work( + await createAndWork( + boss, MEDICATION_INTAKE_IMPORT_QUEUE, { localConcurrency: MEDICATION_INTAKE_IMPORT_CONCURRENCY, includeMetadata: true, }, async (jobs) => { + // Independent per-delivery imports: run them all, report the first + // failure rather than losing it behind a later success. + let failure: JobOutcome | null = null; for (const job of jobs) { - await handleMedicationIntakeImport(job); + const outcome = await handleMedicationIntakeImport(job); + if (!outcome.ok && failure === null) failure = outcome; } + return failure ?? jobDone({ jobs: jobs.length }); }, ); @@ -768,10 +830,21 @@ export async function registerMaintenanceQueues( // above) whose payload omits `userId`. That tick runs the SAME discovery // fan-out the boot pass uses, so duplicate slots created between deploys // collapse within a day instead of waiting for the next worker reboot. - await boss.work>( + await createAndWork>( + boss, INTAKE_SLOT_DEDUP_QUEUE, { localConcurrency: INTAKE_SLOT_DEDUP_CONCURRENCY }, async (jobs) => { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a + // failed fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let slotsCollapsed = 0; + let rowsSoftDeleted = 0; + let rowsNormalised = 0; + let daysRecomputed = 0; for (const job of jobs) { const { userId } = job.data; if (!userId) { @@ -779,6 +852,9 @@ export async function registerMaintenanceQueues( // still holding duplicate-slot candidates (singletonKey-coalesced, // identical to the boot pass). const result = await enqueueBootTimeIntakeSlotDedup(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; workerLog( "info", `[intake-slot-dedup] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, @@ -787,6 +863,11 @@ export async function registerMaintenanceQueues( } try { const summary = await dedupeUserIntakeSlots(userId); + users++; + slotsCollapsed += summary.slotsCollapsed; + rowsSoftDeleted += summary.rowsSoftDeleted; + rowsNormalised += summary.rowsNormalised; + daysRecomputed += summary.daysRecomputed; workerLog( "info", `[intake-slot-dedup] user=${userId} slotsCollapsed=${summary.slotsCollapsed} rowsSoftDeleted=${summary.rowsSoftDeleted} rowsNormalised=${summary.rowsNormalised} daysRecomputed=${summary.daysRecomputed}`, @@ -797,6 +878,17 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + slots_collapsed: slotsCollapsed, + rows_soft_deleted: rowsSoftDeleted, + rows_normalised: rowsNormalised, + days_recomputed: daysRecomputed, + }); }, ); @@ -806,14 +898,26 @@ export async function registerMaintenanceQueues( // empty-userId payload is the daily discovery tick and re-fans the per-user // jobs (singletonKey-coalesced, identical to the boot pass). Serial // concurrency so the migration never crowds the request pool. - await boss.work>( + await createAndWork>( + boss, NOTE_ENCRYPTION_BACKFILL_QUEUE, { localConcurrency: NOTE_ENCRYPTION_BACKFILL_CONCURRENCY }, async (jobs) => { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a + // failed fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let measurements = 0; + let moodEntries = 0; for (const job of jobs) { const { userId } = job.data; if (!userId) { const result = await enqueueBootTimeNoteEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; workerLog( "info", `[note-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, @@ -823,6 +927,9 @@ export async function registerMaintenanceQueues( try { const { measurementsMigrated, moodEntriesMigrated } = await runNoteEncryptionBackfillForUser(userId); + users++; + measurements += measurementsMigrated; + moodEntries += moodEntriesMigrated; workerLog( "info", `[note-encryption-backfill] user=${userId} measurements=${measurementsMigrated} moodEntries=${moodEntriesMigrated}`, @@ -837,11 +944,21 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + measurements_migrated: measurements, + mood_entries_migrated: moodEntries, + }); }, ); // v1.23 — admin-triggered encryption-key rotation (on-demand, singleton). - await boss.work( + await createAndWork( + boss, ENCRYPTION_KEY_ROTATE_QUEUE, { localConcurrency: ENCRYPTION_KEY_ROTATE_CONCURRENCY }, handleEncryptionKeyRotate, @@ -854,14 +971,27 @@ export async function registerMaintenanceQueues( // payload is the daily discovery tick and re-fans the per-user jobs // (singletonKey-coalesced, identical to the boot pass). Serial concurrency // so the migration never crowds the request pool. - await boss.work>( + await createAndWork>( + boss, MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, { localConcurrency: MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY }, async (jobs) => { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a + // failed fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let sideEffects = 0; + let doseChanges = 0; + let inventoryItems = 0; for (const job of jobs) { const { userId } = job.data; if (!userId) { const result = await enqueueBootTimeMedNotesEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; workerLog( "info", `[med-notes-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, @@ -874,6 +1004,10 @@ export async function registerMaintenanceQueues( doseChangesMigrated, inventoryItemsMigrated, } = await runMedNotesEncryptionBackfillForUser(userId); + users++; + sideEffects += sideEffectsMigrated; + doseChanges += doseChangesMigrated; + inventoryItems += inventoryItemsMigrated; workerLog( "info", `[med-notes-encryption-backfill] user=${userId} sideEffects=${sideEffectsMigrated} doseChanges=${doseChangesMigrated} inventoryItems=${inventoryItemsMigrated}`, @@ -888,6 +1022,16 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + side_effects_migrated: sideEffects, + dose_changes_migrated: doseChanges, + inventory_items_migrated: inventoryItems, + }); }, ); @@ -895,16 +1039,25 @@ export async function registerMaintenanceQueues( // trigger endpoint sends one per-user job; this handler indexes that user's // not-yet-indexed documents (consent + budget gated, bounded + resumable). // Serial concurrency so the provider calls never crowd the request pool. - await boss.work( + await createAndWork( + boss, CONTENT_INDEX_BACKFILL_QUEUE, { localConcurrency: CONTENT_INDEX_BACKFILL_CONCURRENCY }, async (jobs) => { + let users = 0; + let skipped = 0; + let documentsIndexed = 0; for (const job of jobs) { const { userId } = job.data; - if (!userId) continue; + if (!userId) { + skipped++; + continue; + } try { const { indexed, reason } = await runContentIndexBackfillForUser(userId); + users++; + documentsIndexed += indexed; workerLog( "info", `[document-content-index-backfill] user=${userId} indexed=${indexed} reason=${reason}`, @@ -919,6 +1072,12 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + skipped, + documents_indexed: documentsIndexed, + }); }, ); @@ -926,15 +1085,22 @@ export async function registerMaintenanceQueues( // (one job per stored document); provider-first (vision) when configured + // consented, else local text-layer extraction. Serial concurrency so the // provider calls + PDF parse never crowd the request pool. - await boss.work( + await createAndWork( + boss, DOCUMENT_INDEX_QUEUE, { localConcurrency: DOCUMENT_INDEX_CONCURRENCY }, async (jobs) => { + let documents = 0; + let skipped = 0; for (const job of jobs) { const { userId, documentId } = job.data; - if (!userId || !documentId) continue; + if (!userId || !documentId) { + skipped++; + continue; + } try { await runDocumentIndex(job.data); + documents++; } catch (err) { recordError(); workerLog( @@ -945,6 +1111,7 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ jobs: jobs.length, documents, skipped }); }, ); @@ -952,15 +1119,22 @@ export async function registerMaintenanceQueues( // upload (one job per stored document); decodes + downscales the original to // a small encrypted JPEG. Pure local compute; serial concurrency so the // canvas/pdfjs decode never crowds the request pool. - await boss.work( + await createAndWork( + boss, DOCUMENT_THUMBNAIL_QUEUE, { localConcurrency: DOCUMENT_THUMBNAIL_CONCURRENCY }, async (jobs) => { + let documents = 0; + let skipped = 0; for (const job of jobs) { const { userId, documentId } = job.data; - if (!userId || !documentId) continue; + if (!userId || !documentId) { + skipped++; + continue; + } try { await runDocumentThumbnail(job.data); + documents++; } catch (err) { recordError(); workerLog( @@ -971,6 +1145,7 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ jobs: jobs.length, documents, skipped }); }, ); @@ -979,15 +1154,22 @@ export async function registerMaintenanceQueues( // the `documentsAutoAiRead` opt-in is ON (egress consent + budget gated) and // persists the summary encrypted. Serial concurrency so the provider call // never crowds the request pool. - await boss.work( + await createAndWork( + boss, DOCUMENT_SUMMARY_QUEUE, { localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY }, async (jobs) => { + let documents = 0; + let skipped = 0; for (const job of jobs) { const { userId, documentId } = job.data; - if (!userId || !documentId) continue; + if (!userId || !documentId) { + skipped++; + continue; + } try { await runDocumentSummaryJob(job.data); + documents++; } catch (err) { recordError(); workerLog( @@ -998,6 +1180,7 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ jobs: jobs.length, documents, skipped }); }, ); @@ -1007,15 +1190,24 @@ export async function registerMaintenanceQueues( // documents (the upload-time enqueue no-opped while the flag was OFF). It // only enqueues — the opt-in, egress consent and budget gates all still run // per document inside the summary job. Serial concurrency. - await boss.work( + await createAndWork( + boss, DOCUMENT_SUMMARY_CATCHUP_QUEUE, { localConcurrency: DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY }, async (jobs) => { + let users = 0; + let skipped = 0; + let summariesEnqueued = 0; for (const job of jobs) { const { userId } = job.data; - if (!userId) continue; + if (!userId) { + skipped++; + continue; + } try { - await runSummaryCatchUpForUser(userId); + const { enqueued } = await runSummaryCatchUpForUser(userId); + users++; + summariesEnqueued += enqueued; } catch (err) { recordError(); workerLog( @@ -1026,6 +1218,12 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + skipped, + summaries_enqueued: summariesEnqueued, + }); }, ); @@ -1033,15 +1231,24 @@ export async function registerMaintenanceQueues( // discovery sends one per-user job; this handler fans out per-document // thumbnail jobs for that user's not-yet-thumbnailed documents. Serial // concurrency; cheap discovery + enqueue only. - await boss.work( + await createAndWork( + boss, DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, { localConcurrency: DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY }, async (jobs) => { + let users = 0; + let skipped = 0; + let thumbnailsEnqueued = 0; for (const job of jobs) { const { userId } = job.data; - if (!userId) continue; + if (!userId) { + skipped++; + continue; + } try { const { enqueued } = await runThumbnailBackfillForUser(userId); + users++; + thumbnailsEnqueued += enqueued; workerLog( "info", `[document-thumbnail-backfill] user=${userId} enqueued=${enqueued}`, @@ -1056,6 +1263,12 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + skipped, + thumbnails_enqueued: thumbnailsEnqueued, + }); }, ); @@ -1063,13 +1276,20 @@ export async function registerMaintenanceQueues( // payload) fans out one per-user job per opted-in account; the queue also // serves the on-demand backfill payloads from the settings surface. Serial // concurrency so the staggered outbound fetches never crowd the request pool. - await boss.work( + await createAndWork( + boss, ENVIRONMENT_FETCH_QUEUE, { localConcurrency: ENVIRONMENT_FETCH_CONCURRENCY }, async (jobs) => { + // A discovery tick and a per-user backfill share this queue and are + // independent of each other, so the pass runs to the end and reports + // the first failure it saw. + let failure: JobOutcome | null = null; for (const job of jobs) { - await handleEnvironmentFetch(boss, job.data); + const outcome = await handleEnvironmentFetch(boss, job.data); + if (!outcome.ok && failure === null) failure = outcome; } + return failure ?? jobDone({ jobs: jobs.length }); }, ); diff --git a/src/lib/jobs/reminder/register-rollup.ts b/src/lib/jobs/reminder/register-rollup.ts index 1b9a0ae01..eca1bf4ff 100644 --- a/src/lib/jobs/reminder/register-rollup.ts +++ b/src/lib/jobs/reminder/register-rollup.ts @@ -10,7 +10,7 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The * `drain-cumulative-queue`, `mean-consolidation-queue`, and * `stress-strain-retention-queue` (dense-retention facet) guards read THIS * module. @@ -104,9 +104,12 @@ import { } from "./shared"; import { createAndSchedule, + createAndWork, + cronIsTheRetry, type QueuePolicyTable, type ScheduleEntry, } from "./registrar-shared"; +import { jobDone, jobFailed } from "@/lib/jobs/job-outcome"; // v1.4.37 W7c — nightly drain of per-sample APPLE_HEALTH cumulative rows. // Collapses each user × cumulative-type × calendar-day bucket into one // `stats:…` row so the list view stops painting hundreds of step chunks per @@ -170,7 +173,7 @@ const schedules: ScheduleEntry[] = [ // v1.4.37 W7c — nightly fold of per-sample APPLE_HEALTH cumulative // rows into one row per day per type. Slots between the // audit-log cleanup (03:15) and the feedback aggregator (04:00). - [DRAIN_CUMULATIVE_QUEUE, DRAIN_CUMULATIVE_CRON], + [DRAIN_CUMULATIVE_QUEUE, DRAIN_CUMULATIVE_CRON, cronIsTheRetry], ]; /** @@ -274,7 +277,8 @@ export async function registerRollupQueues( // the DAY bucket is already recomputed synchronously by the hook // itself. Concurrency-2 keeps two recomputes in flight without // crowding the dashboard request pool. - await boss.work( + await createAndWork( + boss, ROLLUP_RECOMPUTE_QUEUE, { localConcurrency: ROLLUP_RECOMPUTE_CONCURRENCY }, async (jobs) => { @@ -287,6 +291,10 @@ export async function registerRollupQueues( to: new Date(payload.to), }); } + // No per-bucket isolation here on purpose: a recompute that throws + // fails the job, so reaching this line means every bucket in the batch + // was folded. + return jobDone({ buckets_recomputed: jobs.length }); }, ); @@ -295,18 +303,25 @@ export async function registerRollupQueues( // full `recomputeUserRollups` against the default 5-year window // across every granularity. Serial concurrency so the populator // never crowds the dashboard request pool. - await boss.work( + await createAndWork( + boss, ROLLUP_FULL_BACKFILL_QUEUE, { localConcurrency: ROLLUP_FULL_BACKFILL_CONCURRENCY }, async (jobs) => { + let rowsUpsertedTotal = 0; for (const job of jobs) { const { userId } = job.data; const { rowsUpserted, durationMs } = await recomputeUserRollups(userId); + rowsUpsertedTotal += rowsUpserted; workerLog( "info", `[rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, ); } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); }, ); @@ -315,19 +330,29 @@ export async function registerRollupQueues( // step rows; this handler collapses them into one daily-total row per // calendar day and soft-deletes the originals. Serial concurrency so // the populator never crowds the dashboard request pool. - await boss.work( + await createAndWork( + boss, STEP_CONSOLIDATION_QUEUE, { localConcurrency: STEP_CONSOLIDATION_CONCURRENCY }, async (jobs) => { + let daysTotal = 0; + let legacyRowsTotal = 0; for (const job of jobs) { const { userId } = job.data; const { daysConsolidated, legacyRowsSoftDeleted } = await runStepConsolidationForUser(userId); + daysTotal += daysConsolidated; + legacyRowsTotal += legacyRowsSoftDeleted; workerLog( "info", `[step-consolidation] user=${userId} days=${daysConsolidated} legacyRowsSoftDeleted=${legacyRowsSoftDeleted}`, ); } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + legacy_rows_soft_deleted: legacyRowsTotal, + }); }, ); @@ -335,18 +360,39 @@ export async function registerRollupQueues( // GOOGLE_HEALTH / FITBIT daily-total step rows the pre-fix consolidation // swept and removes the shadow MANUAL totals it minted. Serial // concurrency so the repair never crowds the dashboard request pool. - await boss.work( + await createAndWork( + boss, STEP_CONSOLIDATION_REPAIR_QUEUE, { localConcurrency: STEP_CONSOLIDATION_REPAIR_CONCURRENCY }, async (jobs) => { + let rowsResurrected = 0; + let wedgeSkipped = 0; + let manualMintsRemoved = 0; + let daysRecomputed = 0; + let failures = 0; for (const job of jobs) { const { userId } = job.data; const summary = await runStepConsolidationRepairForUser(userId); + rowsResurrected += summary.rowsResurrected; + wedgeSkipped += summary.wedgeSkipped; + manualMintsRemoved += summary.manualMintsRemoved; + daysRecomputed += summary.daysRecomputed; + // The repair isolates its own per-day failures and reports them as a + // count, so they ride out as a fact rather than failing the job. + failures += summary.failures; workerLog( "info", `[step-consolidation-repair] user=${userId} resurrected=${summary.rowsResurrected} wedgeSkipped=${summary.wedgeSkipped} manualMintsRemoved=${summary.manualMintsRemoved} daysRecomputed=${summary.daysRecomputed} failures=${summary.failures}`, ); } + return jobDone({ + users: jobs.length, + rows_resurrected: rowsResurrected, + wedge_skipped: wedgeSkipped, + manual_mints_removed: manualMintsRemoved, + days_recomputed: daysRecomputed, + failures, + }); }, ); @@ -356,18 +402,28 @@ export async function registerRollupQueues( // it and re-runs detection silently so the honest re-derived best takes // its place. Serial concurrency so the repair never crowds the // dashboard request pool. - await boss.work( + await createAndWork( + boss, CUMULATIVE_PR_REDERIVE_QUEUE, { localConcurrency: CUMULATIVE_PR_REDERIVE_CONCURRENCY }, async (jobs) => { + let rowsDeleted = 0; + let rowsReinserted = 0; for (const job of jobs) { const { userId } = job.data; const summary = await runCumulativePrRederivationForUser(userId); + rowsDeleted += summary.rowsDeleted; + rowsReinserted += summary.rowsReinserted; workerLog( "info", `[cumulative-pr-rederive] user=${userId} rowsDeleted=${summary.rowsDeleted} rowsReinserted=${summary.rowsReinserted}`, ); } + return jobDone({ + users: jobs.length, + rows_deleted: rowsDeleted, + rows_reinserted: rowsReinserted, + }); }, ); @@ -376,19 +432,29 @@ export async function registerRollupQueues( // mean-type rows; this handler collapses each completed day to its // mean and soft-deletes the originals. Serial concurrency so the // populator never crowds the dashboard request pool. - await boss.work( + await createAndWork( + boss, MEAN_CONSOLIDATION_QUEUE, { localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY }, async (jobs) => { + let daysTotal = 0; + let perSampleRowsTotal = 0; for (const job of jobs) { const { userId } = job.data; const { daysConsolidated, perSampleRowsSoftDeleted } = await runMeanConsolidationForUser(userId); + daysTotal += daysConsolidated; + perSampleRowsTotal += perSampleRowsSoftDeleted; workerLog( "info", `[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`, ); } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + per_sample_rows_soft_deleted: perSampleRowsTotal, + }); }, ); @@ -399,10 +465,14 @@ export async function registerRollupQueues( // soft-deletes the originals, keeping the in-window intra-day shape intact // for the Stress engine. Serial concurrency so the backfill never crowds // the request pool. - await boss.work( + await createAndWork( + boss, DENSE_INTRADAY_RETENTION_QUEUE, { localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY }, async (jobs) => { + let daysTotal = 0; + let perSampleRowsTotal = 0; + let derivedRestingRowsTotal = 0; for (const job of jobs) { const { userId } = job.data; try { @@ -411,6 +481,9 @@ export async function registerRollupQueues( perSampleRowsSoftDeleted, derivedRestingRowsUpserted, } = await runDenseIntradayRetentionForUser(userId); + daysTotal += daysConsolidated; + perSampleRowsTotal += perSampleRowsSoftDeleted; + derivedRestingRowsTotal += derivedRestingRowsUpserted; workerLog( "info", `[dense-intraday-retention] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted} derivedRestingRowsUpserted=${derivedRestingRowsUpserted}`, @@ -425,6 +498,12 @@ export async function registerRollupQueues( throw err; } } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + per_sample_rows_soft_deleted: perSampleRowsTotal, + derived_resting_rows_upserted: derivedRestingRowsTotal, + }); }, ); @@ -435,10 +514,15 @@ export async function registerRollupQueues( // Self-converging: a rebuilt day drops out of the discovery pairing, so // the pass runs once per install. Serial concurrency so the rebuild never // crowds the request pool. - await boss.work( + await createAndWork( + boss, DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, { localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY }, async (jobs) => { + let daysRebuiltTotal = 0; + let hourlyRowsTotal = 0; + let dailyRowsRetiredTotal = 0; + let daysSkippedTotal = 0; for (const job of jobs) { const { userId } = job.data; try { @@ -448,6 +532,10 @@ export async function registerRollupQueues( dailyRowsRetired, daysSkippedNoTombstones, } = await runDenseIntradayHourlyRebuildForUser(userId); + daysRebuiltTotal += daysRebuilt; + hourlyRowsTotal += hourlyRowsUpserted; + dailyRowsRetiredTotal += dailyRowsRetired; + daysSkippedTotal += daysSkippedNoTombstones; workerLog( "info", `[dense-intraday-hourly-rebuild] user=${userId} daysRebuilt=${daysRebuilt} hourlyRowsUpserted=${hourlyRowsUpserted} dailyRowsRetired=${dailyRowsRetired} daysSkippedNoTombstones=${daysSkippedNoTombstones}`, @@ -462,6 +550,13 @@ export async function registerRollupQueues( throw err; } } + return jobDone({ + users: jobs.length, + days_rebuilt: daysRebuiltTotal, + hourly_rows_upserted: hourlyRowsTotal, + daily_rows_retired: dailyRowsRetiredTotal, + days_skipped_no_tombstones: daysSkippedTotal, + }); }, ); @@ -471,7 +566,8 @@ export async function registerRollupQueues( // No current read path consumes these buckets — they exist so a // future cross-granularity reader can ship without a backfill // step. Concurrency-2 mirrors the measurement-rollup worker. - await boss.work( + await createAndWork( + boss, MOOD_ROLLUP_RECOMPUTE_QUEUE, { localConcurrency: MOOD_ROLLUP_RECOMPUTE_CONCURRENCY }, async (jobs) => { @@ -483,6 +579,7 @@ export async function registerRollupQueues( to: new Date(payload.to), }); } + return jobDone({ buckets_recomputed: jobs.length }); }, ); @@ -491,10 +588,12 @@ export async function registerRollupQueues( // but zero rollup rows; this handler folds the full 5-year window // across every granularity. Concurrency-1 so the populator never // crowds the request pool. - await boss.work( + await createAndWork( + boss, MOOD_ROLLUP_FULL_BACKFILL_QUEUE, { localConcurrency: MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY }, async (jobs) => { + let rowsUpsertedTotal = 0; for (const job of jobs) { const { userId } = job.data; try { @@ -508,6 +607,7 @@ export async function registerRollupQueues( userId, { replace: true }, ); + rowsUpsertedTotal += rowsUpserted; workerLog( "info", `[mood-rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, @@ -522,6 +622,10 @@ export async function registerRollupQueues( throw err; } } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); }, ); @@ -530,15 +634,18 @@ export async function registerRollupQueues( // but zero rollup coverage; this handler folds the trailing 90-day // window per account. Concurrency-1 so the populator never crowds // the request pool. - await boss.work( + await createAndWork( + boss, MEDICATION_COMPLIANCE_BACKFILL_QUEUE, { localConcurrency: MEDICATION_COMPLIANCE_BACKFILL_CONCURRENCY }, async (jobs) => { + let rowsUpsertedTotal = 0; for (const job of jobs) { const { userId } = job.data; try { const { rowsUpserted, durationMs } = await recomputeUserMedicationCompliance(userId); + rowsUpsertedTotal += rowsUpserted; workerLog( "info", `[medication-compliance-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, @@ -553,6 +660,10 @@ export async function registerRollupQueues( throw err; } } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); }, ); @@ -563,10 +674,26 @@ export async function registerRollupQueues( // Concurrency-1 so the drain never crowds the dashboard request pool // and a long backfill on the maintainer's account (300 k+ measurement rows) // stays a single sequential walk. - await boss.work( + await createAndWork( + boss, DRAIN_CUMULATIVE_QUEUE, { localConcurrency: 1 }, async (jobs) => { + // The three drains below are independent passes sharing one nightly + // tick, so one failing pass must not stop the other two from running. + // The first failure is held back and reported once the tick is over — + // that keeps the existing order and still fails the job, which is what + // the swallowed errors never did. + let firstFailure: { reason: string; cause: unknown } | undefined; + let passesFailed = 0; + let bucketsCollapsed = 0; + let perSampleRowsDeleted = 0; + let meanDaysConsolidated = 0; + let meanRowsSoftDeleted = 0; + let denseDaysConsolidated = 0; + let denseRowsSoftDeleted = 0; + let denseSkipped = false; + for (const job of jobs) { try { const summary = await drainPerSampleCumulative(getWorkerPrisma(), { @@ -574,6 +701,8 @@ export async function registerRollupQueues( cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, log: (line) => workerLog("info", line), }); + bucketsCollapsed += summary.totals.bucketsCollapsed; + perSampleRowsDeleted += summary.totals.perSampleRowsDeleted; workerLog( "info", `[drain-cumulative] triggeredAt=${job.data.triggeredAt} usersScanned=${summary.totals.usersScanned} bucketsCollapsed=${summary.totals.bucketsCollapsed} perSampleRowsDeleted=${summary.totals.perSampleRowsDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted}`, @@ -581,6 +710,8 @@ export async function registerRollupQueues( } catch (err) { recordError(); await reportWorkerError("drain-cumulative", err); + passesFailed++; + firstFailure ??= { reason: "cumulative drain failed", cause: err }; } // v1.8.5 — fold the daily-MEAN drain onto the same nightly tick as @@ -600,6 +731,8 @@ export async function registerRollupQueues( cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, log: (line) => workerLog("info", line), }); + meanDaysConsolidated += meanSummary.totals.daysConsolidated; + meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted; workerLog( "info", `[mean-consolidation] triggeredAt=${job.data.triggeredAt} usersScanned=${meanSummary.totals.usersScanned} daysConsolidated=${meanSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${meanSummary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${meanSummary.totals.dailyRowsUpserted} daysFailed=${meanSummary.totals.daysFailed}`, @@ -609,6 +742,8 @@ export async function registerRollupQueues( await reportWorkerError("mean-consolidation", err, { tick: "nightly", }); + passesFailed++; + firstFailure ??= { reason: "mean consolidation failed", cause: err }; } // v1.10.0 WX-E — fold the dense intra-day retention drain onto the @@ -624,6 +759,7 @@ export async function registerRollupQueues( // shipped. try { if (!DENSE_INTRADAY_RETENTION_ENABLED) { + denseSkipped = true; workerLog( "info", "[dense-intraday-retention] disabled via DENSE_INTRADAY_RETENTION_ENABLED — skipping the nightly walk (operator kill-switch)", @@ -636,6 +772,9 @@ export async function registerRollupQueues( log: (line) => workerLog("info", line), }, ); + denseDaysConsolidated += denseSummary.totals.daysConsolidated; + denseRowsSoftDeleted += + denseSummary.totals.perSampleRowsSoftDeleted; workerLog( "info", `[dense-intraday-retention] triggeredAt=${job.data.triggeredAt} usersScanned=${denseSummary.totals.usersScanned} daysConsolidated=${denseSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${denseSummary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${denseSummary.totals.hourlyRowsUpserted} dailyRowsRetired=${denseSummary.totals.dailyRowsRetired}`, @@ -646,8 +785,26 @@ export async function registerRollupQueues( await reportWorkerError("dense-intraday-retention", err, { tick: "nightly", }); + passesFailed++; + firstFailure ??= { reason: "dense retention failed", cause: err }; } } + + const did = { + passes_failed: passesFailed, + buckets_collapsed: bucketsCollapsed, + per_sample_rows_deleted: perSampleRowsDeleted, + mean_days_consolidated: meanDaysConsolidated, + mean_rows_soft_deleted: meanRowsSoftDeleted, + dense_days_consolidated: denseDaysConsolidated, + dense_rows_soft_deleted: denseRowsSoftDeleted, + dense_skipped: denseSkipped, + }; + + if (firstFailure) { + return jobFailed(firstFailure.reason, firstFailure.cause, did); + } + return jobDone(did); }, ); diff --git a/src/lib/jobs/reminder/register-status.ts b/src/lib/jobs/reminder/register-status.ts index 5e2efa814..9ebca0bc2 100644 --- a/src/lib/jobs/reminder/register-status.ts +++ b/src/lib/jobs/reminder/register-status.ts @@ -10,8 +10,8 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The status-wiring - * guards (`status-cron-candidates`, `insight-pregenerate`, + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The + * status-wiring guards (`status-cron-candidates`, `insight-pregenerate`, * `insight-status-generate`, `recovery-score-queue`, * `stress-strain-retention-queue`, `period-narrative-warm`, * `coach-memory-refresh-queue`) read THIS module. @@ -101,9 +101,11 @@ import { type InsightStatusGeneratePayload, } from "@/lib/jobs/insight-status-generate"; import { withBackgroundEvent } from "@/lib/logging/background"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { getWorkerPrisma, workerLog } from "./shared"; import { createAndSchedule, + createAndWork, insightRetryOptions, type QueuePolicyTable, type ScheduleEntry, @@ -363,37 +365,44 @@ export async function registerStatusQueues( ): Promise { await createAndSchedule(boss, allQueues, schedules, queuePolicies); - await boss.work( + await createAndWork( + boss, GENERAL_STATUS_QUEUE, { localConcurrency: 1 }, handleGeneralStatusGenerate, ); - await boss.work( + await createAndWork( + boss, BLOOD_PRESSURE_STATUS_QUEUE, { localConcurrency: 1 }, handleBloodPressureStatusGenerate, ); - await boss.work( + await createAndWork( + boss, WEIGHT_STATUS_QUEUE, { localConcurrency: 1 }, handleWeightStatusGenerate, ); - await boss.work( + await createAndWork( + boss, PULSE_STATUS_QUEUE, { localConcurrency: 1 }, handlePulseStatusGenerate, ); - await boss.work( + await createAndWork( + boss, BMI_STATUS_QUEUE, { localConcurrency: 1 }, handleBmiStatusGenerate, ); - await boss.work( + await createAndWork( + boss, MOOD_STATUS_QUEUE, { localConcurrency: 1 }, handleMoodStatusGenerate, ); - await boss.work( + await createAndWork( + boss, MEDICATION_COMPLIANCE_STATUS_QUEUE, { localConcurrency: 1 }, handleMedicationComplianceStatusGenerate, @@ -409,7 +418,8 @@ export async function registerStatusQueues( // (the cohort walk is itself sequential per user, and the content-hash // gate + per-user budget gate inside the handler cover the rare // double-tick overlap the old single slot serialised away). - await boss.work( + await createAndWork( + boss, INSIGHT_PREGENERATE_QUEUE, { localConcurrency: 2 }, handleInsightPregenerateJob, @@ -420,17 +430,23 @@ export async function registerStatusQueues( // night's samples into one job, and the handler's marker re-check makes a // rare double-fire a no-op, so one slot is enough and bounds the provider // concurrency the same way the nightly path does. - await boss.work( + await createAndWork( + boss, MORNING_DIGEST_REFRESH_QUEUE, { localConcurrency: 1 }, - async (jobs) => { - await withBackgroundEvent("job.morning_digest_refresh", async (evt) => { + async (jobs): Promise => { + return withBackgroundEvent("job.morning_digest_refresh", async (evt) => { + // The counts the batch already decides on: one refresh per job, and + // the finalised subset that reaches the morning push seam. + let refreshed = 0; + let finalised = 0; for (const job of jobs) { try { const result = await runMorningDigestRefresh( getWorkerPrisma(), job.data, ); + refreshed++; evt.addMeta( "morning_refresh", `${result.status}:${result.comprehensive ?? "none"}`, @@ -442,6 +458,7 @@ export async function registerStatusQueues( // there, so a user who never opted in or is outside the window is a // clean no-op. The fallback cron covers the no-sleep-arrived case. if (result.status === "finalised") { + finalised++; const pushResult = await maybeDispatchDailyBriefing( getWorkerPrisma(), job.data.userId, @@ -457,6 +474,7 @@ export async function registerStatusQueues( throw err; } } + return jobDone({ refreshed, finalised }); }); }, ); @@ -464,7 +482,8 @@ export async function registerStatusQueues( // read-only status route on a cold card. Low concurrency so a first // visit that cold-misses several cards can't saturate the Prisma pool // or fan out an unbounded number of concurrent provider calls. - await boss.work( + await createAndWork( + boss, INSIGHT_STATUS_GENERATE_QUEUE, { localConcurrency: INSIGHT_STATUS_GENERATE_CONCURRENCY }, handleInsightStatusGenerate, @@ -474,56 +493,84 @@ export async function registerStatusQueues( // eligible user and upserts one `COMPUTED RECOVERY_SCORE` row per scored // day (idempotent — a re-fire overwrites in place). Single-flight so two // ticks never double-walk the cohort. - await boss.work(RECOVERY_SCORE_QUEUE, { localConcurrency: 1 }, async () => { - try { - const summary = await runRecoveryScore(getWorkerPrisma()); - workerLog( - "info", - `[recovery-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - } catch (err) { - recordError(); - workerLog("error", "[recovery-score] pass failed", err); - throw err; - } - }); + await createAndWork( + boss, + RECOVERY_SCORE_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + try { + const summary = await runRecoveryScore(getWorkerPrisma()); + workerLog( + "info", + `[recovery-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, + ); + return jobDone({ + considered: summary.considered, + stored: summary.stored, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + recordError(); + workerLog("error", "[recovery-score] pass failed", err); + throw err; + } + }, + ); // v1.15.20 — proactive Coach nudge. Single-flight; the push-attempts // ledger caps a user at one nudge per rolling week, so an overlapping // tick would only waste reads. Deterministic triggers, no AI call. - await boss.work(COACH_NUDGE_QUEUE, { localConcurrency: 1 }, async () => { - await withBackgroundEvent("job.coach_nudge", async (evt) => { - try { - const summary = await runCoachNudgeTick(getWorkerPrisma(), new Date()); - evt.setBackground({ - task_name: "job.coach_nudge", - result: { + await createAndWork( + boss, + COACH_NUDGE_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + return withBackgroundEvent("job.coach_nudge", async (evt) => { + try { + const summary = await runCoachNudgeTick( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.coach_nudge", + result: { + candidates_scanned: summary.candidatesScanned, + dispatched: summary.dispatched, + persisted: summary.persisted, + skipped_opted_out: summary.skippedOptedOut, + skipped_no_provider: summary.skippedNoProvider, + skipped_recent_nudge: summary.skippedRecentNudge, + skipped_recent_engagement: summary.skippedRecentEngagement, + skipped_no_trigger: summary.skippedNoTrigger, + skipped_no_channel: summary.skippedNoChannel, + failed: summary.failed, + }, + }); + // A per-user failure never fails this pass — it rides out as the + // `failed` count, exactly as it does in the wide event. + return jobDone({ candidates_scanned: summary.candidatesScanned, dispatched: summary.dispatched, persisted: summary.persisted, - skipped_opted_out: summary.skippedOptedOut, - skipped_no_provider: summary.skippedNoProvider, - skipped_recent_nudge: summary.skippedRecentNudge, - skipped_recent_engagement: summary.skippedRecentEngagement, - skipped_no_trigger: summary.skippedNoTrigger, - skipped_no_channel: summary.skippedNoChannel, failed: summary.failed, - }, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }); + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); + }, + ); // v1.16.11 — medication low-stock pass. Single-flight; the // per-medication stamp makes a re-fire idempotent (already-notified // crossings skip), so an overlapping tick would only waste reads. - await boss.work( + await createAndWork( + boss, MEDICATION_LOW_STOCK_QUEUE, { localConcurrency: 1 }, - async () => { - await withBackgroundEvent("job.medication_low_stock", async (evt) => { + async (): Promise => { + return withBackgroundEvent("job.medication_low_stock", async (evt) => { try { const summary = await runMedicationLowStockTick( getWorkerPrisma(), @@ -544,6 +591,15 @@ export async function registerStatusQueues( failed: summary.failed, }, }); + // A per-medication failure never fails this pass — it rides out as + // the `failed` count. + return jobDone({ + users_scanned: summary.usersScanned, + medications_evaluated: summary.medicationsEvaluated, + notified: summary.notified, + rearmed: summary.rearmed, + failed: summary.failed, + }); } catch (err) { evt.setError(err); recordError(); @@ -556,7 +612,8 @@ export async function registerStatusQueues( // to claim a durable unique row, so two slots would only race each other for // a claim exactly one of them can win. It makes no provider call — a // module-graph test pins that — so one slot costs nothing in throughput. - await boss.work( + await createAndWork( + boss, DATA_ARRIVAL_QUEUE, { localConcurrency: 1 }, handleDataArrival, @@ -564,7 +621,8 @@ export async function registerStatusQueues( // v1.31.0 — the arrival reaction line. Single-flight: this is the spine's // ONE provider call, and a second concurrent slot would only race the same // durable `generatedAt` claim exactly one of them can win. - await boss.work( + await createAndWork( + boss, REACTION_LINE_QUEUE, { localConcurrency: 1 }, handleReactionLine, @@ -573,7 +631,8 @@ export async function registerStatusQueues( // path the spine deliberately does not walk, so it stays off the request pool // and cannot fan out concurrent completions when several workouts land in one // sync. - await boss.work( + await createAndWork( + boss, WORKOUT_INSIGHT_GENERATE_QUEUE, { localConcurrency: WORKOUT_INSIGHT_GENERATE_CONCURRENCY }, handleWorkoutInsightGenerate, @@ -583,84 +642,131 @@ export async function registerStatusQueues( // is suppressed), and the tick only loads a digest for opted-in users at // their exact local fallback hour. No provider call — it reads the cached // digest and dispatches the deterministic line + cached lead. - await boss.work(DAILY_BRIEFING_QUEUE, { localConcurrency: 1 }, async () => { - await withBackgroundEvent("job.daily_briefing", async (evt) => { - try { - const summary = await runDailyBriefingTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.daily_briefing", - result: { + await createAndWork( + boss, + DAILY_BRIEFING_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + return withBackgroundEvent("job.daily_briefing", async (evt) => { + try { + const summary = await runDailyBriefingTick( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.daily_briefing", + result: { + candidates_scanned: summary.candidatesScanned, + in_slot: summary.inSlot, + sent: summary.sent, + suppressed_frequency: summary.suppressedFrequency, + no_digest: summary.noDigest, + opted_out: summary.optedOut, + module_off: summary.moduleOff, + no_channel: summary.noChannel, + outside_window: summary.outsideWindow, + failed: summary.failed, + }, + }); + // Most ticks land outside every user's slot; that is the tick doing + // its job, so a zero `sent` is a success with zero counts. + return jobDone({ candidates_scanned: summary.candidatesScanned, in_slot: summary.inSlot, sent: summary.sent, - suppressed_frequency: summary.suppressedFrequency, - no_digest: summary.noDigest, - opted_out: summary.optedOut, - module_off: summary.moduleOff, - no_channel: summary.noChannel, - outside_window: summary.outsideWindow, failed: summary.failed, - }, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }); + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); + }, + ); // v1.10.0 — computed scores (WX-E). Nightly Stress-score (HRV-derived // proxy) compute + store. Single-flight so two ticks never double-walk // the cohort. The runner iterates every eligible user and upserts one // `COMPUTED STRESS_SCORE` row per scored day (idempotent — a re-fire // overwrites in place). - await boss.work(STRESS_SCORE_QUEUE, { localConcurrency: 1 }, async () => { - try { - const summary = await runStressScore(getWorkerPrisma()); - workerLog( - "info", - `[stress-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - } catch (err) { - recordError(); - workerLog("error", "[stress-score] pass failed", err); - throw err; - } - }); + await createAndWork( + boss, + STRESS_SCORE_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + try { + const summary = await runStressScore(getWorkerPrisma()); + workerLog( + "info", + `[stress-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, + ); + return jobDone({ + considered: summary.considered, + stored: summary.stored, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + recordError(); + workerLog("error", "[stress-score] pass failed", err); + throw err; + } + }, + ); // v1.10.0 — computed scores (WX-E). Nightly Strain-score (Banister TRIMP // cardio-load) compute + store. Single-flight; upserts one `COMPUTED // STRAIN_SCORE` row per scored day (idempotent). - await boss.work(STRAIN_SCORE_QUEUE, { localConcurrency: 1 }, async () => { - try { - const summary = await runStrainScore(getWorkerPrisma()); - workerLog( - "info", - `[strain-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - } catch (err) { - recordError(); - workerLog("error", "[strain-score] pass failed", err); - throw err; - } - }); + await createAndWork( + boss, + STRAIN_SCORE_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + try { + const summary = await runStrainScore(getWorkerPrisma()); + workerLog( + "info", + `[strain-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, + ); + return jobDone({ + considered: summary.considered, + stored: summary.stored, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + recordError(); + workerLog("error", "[strain-score] pass failed", err); + throw err; + } + }, + ); // v1.11.0 — period-narrative warm. A scheduled tick (no `userId`) runs the // boundary-gated nightly fan-out; a `userId` payload runs a single-user warm // enqueued by the read-only GET on a cold/stale read. Single-flight so two // ticks never double-walk the cohort; the per-user budget gate covers the // fan-out and the enqueue `singletonKey` covers the single-user path. - await boss.work( + await createAndWork( + boss, PERIOD_NARRATIVE_QUEUE, { localConcurrency: 1 }, - async (jobs) => { + async (jobs): Promise => { + // Two modes on one queue, so the outcome counts them separately: a + // non-boundary night is a scheduled pass that generated nothing, which + // is a success with zero counts, not an idle queue. + let singleUser = 0; + let scheduled = 0; + let generated = 0; + let failed = 0; for (const job of jobs) { try { if (job.data?.userId) { await warmOneNarrative(job.data); + singleUser++; } else { const summary = await runPeriodNarrativeWarm(getWorkerPrisma()); + scheduled++; + generated += summary.generated; + failed += summary.failed; workerLog( "info", `[period-narrative] periods=${summary.periods.join(",") || "none"} total=${summary.total} generated=${summary.generated} cached=${summary.cached} skipped=${summary.skipped} insufficient=${summary.insufficient} failed=${summary.failed} budget=${summary.budgetBlocked}`, @@ -674,26 +780,45 @@ export async function registerStatusQueues( throw err; } } + return jobDone({ + single_user: singleUser, + scheduled, + generated, + failed, + }); }, ); // v1.11.1 — combined Coach memory refresh: rolling conversation summary + // durable fact extraction for one long conversation. localConcurrency 1 so a // burst of long-conversation turns can't fan out concurrent provider calls; // each step is budget-gated inside runStatusCompletion and fault-isolated. - await boss.work( + await createAndWork( + boss, COACH_MEMORY_REFRESH_QUEUE, { localConcurrency: 1 }, - async (jobs) => { + async (jobs): Promise => { + // An incomplete payload is skipped rather than failed, so it has to be + // countable — otherwise a queue full of them reads as a clean run. + let refreshed = 0; + let skippedIncomplete = 0; for (const job of jobs) { - if (!job.data?.conversationId || !job.data?.userId) continue; + if (!job.data?.conversationId || !job.data?.userId) { + skippedIncomplete++; + continue; + } try { await runCoachMemoryRefresh(job.data); + refreshed++; } catch (err) { recordError(); workerLog("error", "[coach-memory-refresh] failed", err); throw err; } } + return jobDone({ + refreshed, + skipped_incomplete: skippedIncomplete, + }); }, ); @@ -701,11 +826,12 @@ export async function registerStatusQueues( // reminder to `due` and minting plan-review reminders are both idempotent // across re-fires (an already-due reminder is not re-flipped; a plan whose // reviewDate was cleared is not re-minted). No provider call, no push. - await boss.work( + await createAndWork( + boss, COACH_REMINDER_SWEEP_QUEUE, { localConcurrency: 1 }, - async () => { - await withBackgroundEvent("job.coach_reminder_sweep", async (evt) => { + async (): Promise => { + return withBackgroundEvent("job.coach_reminder_sweep", async (evt) => { try { const summary = await runCoachReminderSweep( getWorkerPrisma(), @@ -719,6 +845,11 @@ export async function registerStatusQueues( errored: summary.errored, }, }); + return jobDone({ + reminders_due: summary.remindersDue, + plan_reviews_minted: summary.planReviewsMinted, + errored: summary.errored, + }); } catch (err) { evt.setError(err); recordError(); @@ -732,11 +863,12 @@ export async function registerStatusQueues( // reminders the sweep minted and writes the grounded before/after read-back // into the plan's encrypted outcome column, flipping it to `reviewed`. // Idempotent (a reviewed plan is not re-processed). No provider call, no push. - await boss.work( + await createAndWork( + boss, COACH_PLAN_REVIEW_QUEUE, { localConcurrency: 1 }, - async () => { - await withBackgroundEvent("job.coach_plan_review", async (evt) => { + async (): Promise => { + return withBackgroundEvent("job.coach_plan_review", async (evt) => { try { const summary = await runCoachPlanReviewTick( getWorkerPrisma(), @@ -750,6 +882,11 @@ export async function registerStatusQueues( errored: summary.errored, }, }); + return jobDone({ + reviewed: summary.reviewed, + insufficient: summary.insufficient, + errored: summary.errored, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/registrar-shared.ts b/src/lib/jobs/reminder/registrar-shared.ts index 178c580e6..c05f1ea87 100644 --- a/src/lib/jobs/reminder/registrar-shared.ts +++ b/src/lib/jobs/reminder/registrar-shared.ts @@ -167,6 +167,32 @@ export const insightRetryOptions = { retryBackoff: true, } as const; +/** + * Retry policy for a cron-driven pass whose failure mode is deterministic. + * + * Handlers now return a `JobOutcome`, and the retention cleanups that used to + * warn-and-return fail their job instead. That is the point — but it walks + * into pg-boss's queue default of `retryLimit: 2` with no delay and no + * backoff, so a bulk DELETE that timed out against a large trailing edge + * would run twice more, immediately, against the same rows. Three doomed + * sequential scans a night is a worse outcome than the silence it replaced. + * + * For this class the next cron tick IS the retry, and it arrives with the + * whole gap as backoff. So the schedule declares zero retries: the failure is + * recorded once, reaches the operator once, and the pass tries again on its + * own cadence. + * + * It rides on the SCHEDULE rather than on `createQueue`, deliberately. + * pg-boss's `create_queue()` ends in ON CONFLICT DO NOTHING, so a queue-level + * default would be inert on every already-running instance — the same trap + * `reconcileQueuePolicies` above exists to work around. Send options apply to + * every job the cron mints, new database or old. + * + * Use it only where the failure genuinely repeats. A transient failure (a TLS + * probe, an S3 fetch, a provider call) wants the default retries, not this. + */ +export const cronIsTheRetry = { retryLimit: 0 } as const; + /** * Create every queue in `queues`, then schedule every cron in `schedules`. * Centralised so each registrar provisions before it schedules in the exact From f0a804a812ac4ab8cc9730dfb1e5156a56900edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Mon, 27 Jul 2026 23:11:38 +0200 Subject: [PATCH 4/6] test(jobs): pin the outcome spine and the binding rule runJob's claim is behavioural, not typed: an ok:false outcome has to reach pg-boss as a failed job, reported once, with the handler's own facts and the thrown error's stack rather than runJob's frame. That is what the retention cleanups now rely on, so it is pinned. The rule's tests carry its blind spots in the valid list, where a reader will trip over them: an aliased receiver is not detected, and neither is a handler that returns ok:true after swallowing. --- .../__tests__/job-handler-outcome.test.mjs | 67 +++++++++++ src/lib/jobs/__tests__/run-job.test.ts | 112 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 eslint-plugins/healthlog/__tests__/job-handler-outcome.test.mjs create mode 100644 src/lib/jobs/__tests__/run-job.test.ts diff --git a/eslint-plugins/healthlog/__tests__/job-handler-outcome.test.mjs b/eslint-plugins/healthlog/__tests__/job-handler-outcome.test.mjs new file mode 100644 index 000000000..7d608ec10 --- /dev/null +++ b/eslint-plugins/healthlog/__tests__/job-handler-outcome.test.mjs @@ -0,0 +1,67 @@ +/** + * Unit tests for the `healthlog/job-handler-outcome` rule — the bare + * `boss.work(` ban, the one exempt module that owns `createAndWork`, the + * test-file exemption, and the scope limit to `src/lib/jobs/`. + * + * The `invalid` cases below also stand as the rule's own documented blind + * spots: an aliased or renamed receiver is not detected, and neither is a + * handler that returns `{ ok: true }` after swallowing. Those sit in `valid` + * on purpose, so a future reader can see what this rule does not claim. + */ +import { describe, it } from "vitest"; +import { RuleTester } from "eslint"; +import rule from "../job-handler-outcome.js"; + +RuleTester.describe = describe; +RuleTester.it = it; +RuleTester.itOnly = it.only; + +const ruleTester = new RuleTester({ + languageOptions: { ecmaVersion: 2023, sourceType: "module" }, +}); + +ruleTester.run("job-handler-outcome", rule, { + valid: [ + { + code: "await createAndWork(boss, QUEUE, { localConcurrency: 1 }, handleX);", + filename: "/repo/src/lib/jobs/reminder/register-status.ts", + }, + { + // The wrapper's own call — this module is the one legal caller. + code: "await boss.work(queue, options, runJob(queue, handler));", + filename: "/repo/src/lib/jobs/reminder/registrar-shared.ts", + }, + { + // Tests mock `boss.work` and assert against it directly. + code: 'boss.work("q", {}, async () => {});', + filename: "/repo/src/lib/jobs/__tests__/whoop-queues.test.ts", + }, + { + // Out of scope: not the worker tree. + code: 'boss.work("q", {}, async () => {});', + filename: "/repo/src/lib/integrations/status.ts", + }, + { + // Documented blind spot: an aliased receiver is not detected. + code: 'const b = boss; b.work("q", {}, async () => {});', + filename: "/repo/src/lib/jobs/reminder/register-rollup.ts", + }, + { + // Documented blind spot: this rule cannot see a swallowed failure. + code: "await createAndWork(boss, QUEUE, {}, async () => ({ ok: true, did: {} }));", + filename: "/repo/src/lib/jobs/reminder/register-rollup.ts", + }, + ], + invalid: [ + { + code: 'await boss.work("scratch", async () => {});', + filename: "/repo/src/lib/jobs/reminder/register-maintenance.ts", + errors: [{ messageId: "bareBossWork" }], + }, + { + code: "await boss.work(QUEUE, { localConcurrency: 1 }, handleX);", + filename: "/repo/src/lib/jobs/worker-status.ts", + errors: [{ messageId: "bareBossWork" }], + }, + ], +}); diff --git a/src/lib/jobs/__tests__/run-job.test.ts b/src/lib/jobs/__tests__/run-job.test.ts new file mode 100644 index 000000000..6973cb89a --- /dev/null +++ b/src/lib/jobs/__tests__/run-job.test.ts @@ -0,0 +1,112 @@ +/** + * Unit tests for the job-outcome spine. + * + * The whole point of `runJob` is that an `ok: false` outcome reaches pg-boss + * as a failed job. That is a behaviour claim, not a typing one, and it is the + * claim the retention cleanups now depend on — so it is pinned here: the + * failure is reported once with the handler's own facts, it rethrows, and a + * handler that throws on its own is passed through untouched rather than + * rewrapped into a stack that points at this file. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import type { Job } from "pg-boss"; + +const reportWorkerError = vi.fn(); + +vi.mock("../report-worker-error", () => ({ + reportWorkerError: (...a: unknown[]) => reportWorkerError(...a), +})); + +import { jobDone, jobFailed } from "../job-outcome"; +import { JobFailure, runJob } from "../run-job"; + +const noJobs: Job[] = []; + +beforeEach(() => { + vi.clearAllMocks(); + reportWorkerError.mockResolvedValue(undefined); +}); + +describe("runJob", () => { + it("resolves and reports nothing when the handler says it did the work", async () => { + const wrapped = runJob("rate-limit-cleanup", async () => + jobDone({ deleted: 12 }), + ); + + await expect(wrapped(noJobs)).resolves.toBeUndefined(); + expect(reportWorkerError).not.toHaveBeenCalled(); + }); + + it("treats a run with nothing to do as a success", async () => { + const wrapped = runJob("mood-reminder-cleanup", async () => + jobDone({ deleted: 0 }), + ); + + await expect(wrapped(noJobs)).resolves.toBeUndefined(); + expect(reportWorkerError).not.toHaveBeenCalled(); + }); + + it("rethrows a failed outcome so pg-boss records a failed job", async () => { + const cause = new Error("statement timeout"); + const wrapped = runJob("push-attempt-cleanup", async () => + jobFailed("push-attempt cleanup failed", cause, { deleted: 0 }), + ); + + await expect(wrapped(noJobs)).rejects.toBeInstanceOf(JobFailure); + await expect(wrapped(noJobs)).rejects.toThrow( + "push-attempt-cleanup: push-attempt cleanup failed", + ); + }); + + it("reports the failure with the handler's own facts and the cause", async () => { + const cause = new Error("statement timeout"); + const wrapped = runJob("push-attempt-cleanup", async () => + jobFailed("push-attempt cleanup failed", cause, { deleted: 0 }), + ); + + await expect(wrapped(noJobs)).rejects.toThrow(); + + expect(reportWorkerError).toHaveBeenCalledTimes(1); + const [queue, error, meta] = reportWorkerError.mock.calls[0]; + expect(queue).toBe("push-attempt-cleanup"); + expect(error).toBeInstanceOf(JobFailure); + expect(meta).toEqual({ deleted: 0, cause: "statement timeout" }); + }); + + it("carries the thrown error's stack instead of its own frame", async () => { + const cause = new Error("statement timeout"); + cause.stack = "Error: statement timeout\n at deleteMany (prisma.ts:1:1)"; + const wrapped = runJob("mcp-token-cleanup", async () => + jobFailed("mcp-token cleanup failed", cause), + ); + + const failure = await wrapped(noJobs).catch((err: unknown) => err); + expect(failure).toBeInstanceOf(JobFailure); + expect((failure as JobFailure).stack).toContain("at deleteMany"); + expect((failure as JobFailure).cause).toBe(cause); + }); + + it("passes a throwing handler through untouched", async () => { + const thrown = new Error("cohort discovery failed"); + const wrapped = runJob("withings-sync", async () => { + throw thrown; + }); + + await expect(wrapped(noJobs)).rejects.toBe(thrown); + // The throw already fails the job. Reporting it here would double every + // handler that reports for itself, and rewrapping would bury the stack. + expect(reportWorkerError).not.toHaveBeenCalled(); + }); + + it("hands the batch to the handler unchanged", async () => { + const handler = vi.fn(async () => jobDone({ jobs: 2 })); + const jobs = [ + { id: "a", data: { userId: "u1" } }, + { id: "b", data: { userId: "u2" } }, + ] as unknown as Job<{ userId: string }>[]; + + await runJob("data-arrival", handler)(jobs); + + expect(handler).toHaveBeenCalledWith(jobs); + }); +}); From ae294573ef8787e228ac4dbc7b0d9e3055b8f155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Mon, 27 Jul 2026 23:23:53 +0200 Subject: [PATCH 5/6] test(jobs): follow the queue guards onto the new binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thirty-three suites read a registrar as source text and matched `boss.work` next to a queue constant. They now match `createAndWork`, and the seven schedule tuples that grew a `cronIsTheRetry` third element assert that element rather than accepting any arity — a guard that accepts more than it did before is a guard that notices less. Two negative assertions were tightened while they were open: the "has no cron" checks on the arrival and workout-insight queues ended at a literal `]`, so a cron added WITH send options would have slipped past. The behavioural repairs carry the real change. Handlers that resolved to undefined now resolve to their outcome, so the assertions name the counts instead of asserting nothing. The reconcile tick's test said it swallows a failure instead of throwing; it reports a failed outcome now, and the test says that, with the comment pointing at the retryLimit that carries the old anti-retry-loop concern. --- .../apple-health-import-reconcile.test.ts | 40 ++++++++++++------- .../apple-health-import-send-policy.test.ts | 7 +++- .../coach-memory-refresh-queue.test.ts | 6 +-- .../coach-message-cleanup-queue.test.ts | 8 ++-- .../cumulative-pr-rederive-queue.test.ts | 6 +-- .../__tests__/cycle-reminder-queue.test.ts | 6 +-- src/lib/jobs/__tests__/daily-briefing.test.ts | 4 +- .../jobs/__tests__/data-arrival-queue.test.ts | 14 ++++--- .../__tests__/drain-cumulative-queue.test.ts | 6 +-- src/lib/jobs/__tests__/fitbit-queues.test.ts | 12 +++--- .../__tests__/google-health-queues.test.ts | 10 ++--- .../__tests__/insight-pregenerate.test.ts | 4 +- .../__tests__/insight-status-generate.test.ts | 6 +-- .../__tests__/intake-slot-dedup-queue.test.ts | 6 +-- .../__tests__/mcp-token-cleanup-queue.test.ts | 8 ++-- .../mean-consolidation-queue.test.ts | 6 +-- .../measurement-reminder-queue.test.ts | 12 +++--- .../measurement-tombstone-cleanup.test.ts | 6 +-- .../jobs/__tests__/nightscout-queues.test.ts | 10 ++--- .../__tests__/period-narrative-warm.test.ts | 6 ++- .../__tests__/recovery-score-queue.test.ts | 6 +-- .../__tests__/sleep-timeline-queues.test.ts | 6 +-- .../__tests__/status-cron-candidates.test.ts | 4 +- .../step-consolidation-repair-queue.test.ts | 6 +-- .../stress-strain-retention-queue.test.ts | 18 ++++----- .../__tests__/tls-pin-monitor-queue.test.ts | 6 +-- src/lib/jobs/__tests__/whoop-queues.test.ts | 14 +++---- .../__tests__/whoop-resource-sync.test.ts | 13 +++++- .../__tests__/withings-ecg-worker.test.ts | 28 ++++++++++--- .../jobs/__tests__/withings-queues.test.ts | 6 +-- .../withings-subscription-retry.test.ts | 25 +++++++++++- .../__tests__/workout-insight-queue.test.ts | 8 ++-- .../reminder/__tests__/poll-cohort.test.ts | 12 +++++- 33 files changed, 208 insertions(+), 127 deletions(-) diff --git a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts index 13e240628..219978aae 100644 --- a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts @@ -73,12 +73,16 @@ describe("periodic reconcile — wiring", () => { /const schedules[\s\S]*?=\s*\[([\s\S]*?)\];/, ); expect(schedules).not.toBeNull(); + // The third element pins the retry policy. It is load-bearing now that + // the handler reports a failed outcome instead of swallowing: without + // `cronIsTheRetry` a database outage would run the same doomed sweep + // three times per tick, and the next tick is already the retry. expect(schedules![1]).toMatch( - /\[IMPORT_JOB_RECONCILE_QUEUE,\s*IMPORT_JOB_RECONCILE_CRON\]/, + /\[IMPORT_JOB_RECONCILE_QUEUE,\s*IMPORT_JOB_RECONCILE_CRON,\s*cronIsTheRetry\]/, ); expect(maintenanceSource).toMatch( - /boss\.work\(\s*IMPORT_JOB_RECONCILE_QUEUE[\s\S]{0,120}handleImportJobReconcileTick/, + /createAndWork\(\s*boss,\s*IMPORT_JOB_RECONCILE_QUEUE[\s\S]{0,120}handleImportJobReconcileTick/, ); }); }); @@ -163,24 +167,30 @@ describe("reconcileOrphanImportJobs", () => { }); describe("handleImportJobReconcileTick", () => { - it("delegates to reconcileOrphanImportJobs and resolves on success", async () => { + it("delegates to reconcileOrphanImportJobs and reports a done outcome", async () => { mocks.findMany.mockResolvedValue([]); - await expect( - handleImportJobReconcileTick([] as never), - ).resolves.toBeUndefined(); + await expect(handleImportJobReconcileTick([] as never)).resolves.toEqual({ + ok: true, + did: {}, + }); expect(mocks.findMany).toHaveBeenCalledTimes(1); }); - it("swallows a reconcile failure instead of throwing", async () => { - mocks.findMany.mockRejectedValue(new Error("db unavailable")); + it("reports a failed outcome when the reconcile pass throws", async () => { + const cause = new Error("db unavailable"); + mocks.findMany.mockRejectedValue(cause); - // A background sweep this cheap must never spam pg-boss's - // retry/backoff machinery — it just re-runs on the next 15-minute - // tick. The handler must resolve even when the underlying reconcile - // pass throws. - await expect( - handleImportJobReconcileTick([] as never), - ).resolves.toBeUndefined(); + // The handler used to swallow this and resolve, which left a sweep + // that had not run for hours indistinguishable from one that found + // nothing to do. It now names the failure, `runJob` rethrows it, and + // pg-boss records a failed job. The queue carries `cronIsTheRetry` + // (retryLimit 0) because the next 15-minute tick is the retry — a + // database that is down stays down for an immediate re-run. + await expect(handleImportJobReconcileTick([] as never)).resolves.toEqual({ + ok: false, + reason: "apple health import reconcile failed", + cause, + }); }); }); diff --git a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts index 5e5de5c6c..d28594276 100644 --- a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts @@ -87,11 +87,14 @@ describe("apple health import — parser revision boundary", () => { expect(allQueues).not.toBeNull(); expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_V2_QUEUE\b/); expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_LEGACY_QUEUE\b/); + // The v2 window is the wider of the two: its binding wraps the batch in + // a loop that keeps the first failed outcome, so the handler call sits + // further from the queue name than the legacy bridge's does. expect(maintenanceSource).toMatch( - /boss\.work[\s\S]{0,200}APPLE_HEALTH_IMPORT_V2_QUEUE[\s\S]{0,400}handleAppleHealthImport/, + /createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_V2_QUEUE[\s\S]{0,600}handleAppleHealthImport/, ); expect(maintenanceSource).toMatch( - /boss\.work[\s\S]{0,200}APPLE_HEALTH_IMPORT_LEGACY_QUEUE[\s\S]{0,400}migrateLegacyAppleHealthImport/, + /createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_LEGACY_QUEUE[\s\S]{0,400}migrateLegacyAppleHealthImport/, ); }); diff --git a/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts b/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts index a8ad973da..bb6147eab 100644 --- a/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts +++ b/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts @@ -4,7 +4,7 @@ * * Same source-text-grep approach as the period-narrative / stress-strain * guards: assert the queue is imported, registered in `allQueues`, and wired - * to a `boss.work` handler that runs the refresh pipeline — without booting + * to a `createAndWork` handler that runs the refresh pipeline — without booting * pg-boss + Prisma. An unregistered queue silently never runs (the recurring * past bug this guards against; v1.4.37 W10 caught exactly this class). */ @@ -41,9 +41,9 @@ describe("reminder-worker — coach-memory-refresh wiring", () => { expect(allQueuesBlock()).toMatch(/\bCOACH_MEMORY_REFRESH_QUEUE\b/); }); - it("registers a boss.work handler that runs the refresh pipeline", () => { + it("registers a createAndWork handler that runs the refresh pipeline", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}COACH_MEMORY_REFRESH_QUEUE[\s\S]{0,400}runCoachMemoryRefresh/, + /createAndWork[\s\S]{0,200}COACH_MEMORY_REFRESH_QUEUE[\s\S]{0,600}runCoachMemoryRefresh/, ); }); }); diff --git a/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts index e00d24206..0567b4e40 100644 --- a/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts +++ b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts @@ -2,7 +2,7 @@ * v1.18.7 — coach-message-cleanup queue registration guard. * * Same source-text-grep approach as the other queue-wiring guards: assert the - * queue is registered in `allQueues`, scheduled, and wired to a `boss.work` + * queue is registered in `allQueues`, scheduled, and wired to a `createAndWork` * handler — without booting pg-boss + Prisma. An unregistered queue silently * never drains (the recurring v1.4.37 dead-queue bug), which would leave the * encrypted Coach history growing unbounded with the retention job dark. @@ -41,13 +41,13 @@ describe("reminder-worker — coach-message-cleanup wiring", () => { it("schedules the coach-message-cleanup cron", () => { expect(workerSource).toMatch( - /\[COACH_MESSAGE_CLEANUP_QUEUE,\s*COACH_MESSAGE_CLEANUP_CRON\]/, + /\[COACH_MESSAGE_CLEANUP_QUEUE,\s*COACH_MESSAGE_CLEANUP_CRON,\s*cronIsTheRetry\]/, ); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSource).toMatch( - /boss\.work[\s\S]{0,200}COACH_MESSAGE_CLEANUP_QUEUE[\s\S]{0,200}handleCoachMessageCleanup/, + /createAndWork[\s\S]{0,200}COACH_MESSAGE_CLEANUP_QUEUE[\s\S]{0,200}handleCoachMessageCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts b/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts index 61aa8c699..18a28299b 100644 --- a/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts +++ b/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts @@ -3,7 +3,7 @@ * registration guard. * * Same source-text-grep approach as the step-consolidation-repair guard: - * assert the queue is registered in `allQueues`, a `boss.work` handler is + * assert the queue is registered in `allQueues`, a `createAndWork` handler is * wired against it, and the boot-discovery enqueue helper fires — without * booting pg-boss + Prisma. An unregistered queue silently never drains * (the v1.4.37 dead-queue class). @@ -34,9 +34,9 @@ describe("reminder-worker — cumulative-pr-rederive wiring", () => { expect(allQueuesMatch![1]).toMatch(/\bCUMULATIVE_PR_REDERIVE_QUEUE\b/); }); - it("registers a boss.work handler against the queue", () => { + it("registers a createAndWork handler against the queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}CUMULATIVE_PR_REDERIVE_QUEUE[\s\S]{0,400}runCumulativePrRederivationForUser/, + /createAndWork[\s\S]{0,200}CUMULATIVE_PR_REDERIVE_QUEUE[\s\S]{0,400}runCumulativePrRederivationForUser/, ); }); diff --git a/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts b/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts index e060711e6..8c9158af2 100644 --- a/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts +++ b/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts @@ -5,7 +5,7 @@ * consolidation registration guards: assert the cycle-reminder queue is * declared, listed in `allQueues` (the v1.4.37 dead-queue lesson — an * unregistered queue is never provisioned and the schedule silently - * no-ops), scheduled on its cron, and wired to a `boss.work` handler that + * no-ops), scheduled on its cron, and wired to a `createAndWork` handler that * delegates to `runCycleReminderTick`. No pg-boss / Prisma boot. */ import { readFileSync } from "node:fs"; @@ -57,9 +57,9 @@ describe("reminder-worker — cycle-reminder wiring", () => { expect(source).toMatch(/\[CYCLE_REMINDER_QUEUE,\s*CYCLE_REMINDER_CRON\]/); }); - it("registers a boss.work handler that runs the cycle-reminder tick", () => { + it("registers a createAndWork handler that runs the cycle-reminder tick", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}CYCLE_REMINDER_QUEUE[\s\S]{0,200}handleCycleReminderCheck/, + /createAndWork[\s\S]{0,200}CYCLE_REMINDER_QUEUE[\s\S]{0,200}handleCycleReminderCheck/, ); expect(source).toMatch( /handleCycleReminderCheck[\s\S]{0,400}runCycleReminderTick/, diff --git a/src/lib/jobs/__tests__/daily-briefing.test.ts b/src/lib/jobs/__tests__/daily-briefing.test.ts index 8ae057493..ed34641db 100644 --- a/src/lib/jobs/__tests__/daily-briefing.test.ts +++ b/src/lib/jobs/__tests__/daily-briefing.test.ts @@ -98,9 +98,9 @@ describe("reminder-worker — daily-briefing wiring", () => { ); }); - it("wires a boss.work handler that runs the tick", () => { + it("wires a createAndWork handler that runs the tick", () => { expect(registrar).toMatch( - /boss\.work[\s\S]{0,200}DAILY_BRIEFING_QUEUE[\s\S]{0,400}runDailyBriefingTick/, + /createAndWork[\s\S]{0,200}DAILY_BRIEFING_QUEUE[\s\S]{0,400}runDailyBriefingTick/, ); }); diff --git a/src/lib/jobs/__tests__/data-arrival-queue.test.ts b/src/lib/jobs/__tests__/data-arrival-queue.test.ts index 9deeb634e..a86f8057f 100644 --- a/src/lib/jobs/__tests__/data-arrival-queue.test.ts +++ b/src/lib/jobs/__tests__/data-arrival-queue.test.ts @@ -39,9 +39,9 @@ describe("data-arrival queue wiring", () => { expect(allQueues![1]).toMatch(/\bDATA_ARRIVAL_QUEUE\b/); }); - it("binds a boss.work handler that drains it", () => { + it("binds a createAndWork handler that drains it", () => { expect(statusRegistrar).toMatch( - /boss\.work[\s\S]{0,200}DATA_ARRIVAL_QUEUE[\s\S]{0,200}handleDataArrival/, + /createAndWork[\s\S]{0,200}DATA_ARRIVAL_QUEUE[\s\S]{0,200}handleDataArrival/, ); }); @@ -61,7 +61,11 @@ describe("data-arrival queue wiring", () => { }); it("has no cron schedule — it is send-only, driven by ingest", () => { - expect(statusRegistrar).not.toMatch(/\[DATA_ARRIVAL_QUEUE,\s*\w+_CRON\]/); + // Tolerates the optional third schedule element, so a cron sneaking in + // with send options still trips this. + expect(statusRegistrar).not.toMatch( + /\[\s*DATA_ARRIVAL_QUEUE,\s*\w+_CRON\s*[,\]]/, + ); }); it("registers the retention job that prunes the markers", () => { @@ -71,10 +75,10 @@ describe("data-arrival queue wiring", () => { expect(allQueues).not.toBeNull(); expect(allQueues![1]).toMatch(/\bARRIVAL_REACTION_CLEANUP_QUEUE\b/); expect(maintenanceSource).toMatch( - /\[ARRIVAL_REACTION_CLEANUP_QUEUE,\s*ARRIVAL_REACTION_CLEANUP_CRON\]/, + /\[\s*ARRIVAL_REACTION_CLEANUP_QUEUE,\s*ARRIVAL_REACTION_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, ); expect(maintenanceSource).toMatch( - /boss\.work[\s\S]{0,200}ARRIVAL_REACTION_CLEANUP_QUEUE[\s\S]{0,200}handleArrivalReactionCleanup/, + /createAndWork[\s\S]{0,200}ARRIVAL_REACTION_CLEANUP_QUEUE[\s\S]{0,200}handleArrivalReactionCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts index 4bfc499cb..b904a30dd 100644 --- a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts +++ b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts @@ -50,15 +50,15 @@ describe("reminder-worker — drainPerSampleCumulative nightly schedule", () => expect(source).toMatch(/\bDRAIN_CUMULATIVE_CUTOFF_HOURS\b/); }); - it("registers a boss.work handler against the drain queue", () => { + it("registers a createAndWork handler against the drain queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,400}drainPerSampleCumulative/, + /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,900}drainPerSampleCumulative/, ); }); it("schedules the drain cron via boss.schedule (allQueues + schedules)", () => { expect(source).toMatch( - /\[DRAIN_CUMULATIVE_QUEUE,\s*DRAIN_CUMULATIVE_CRON\]/, + /\[DRAIN_CUMULATIVE_QUEUE,\s*DRAIN_CUMULATIVE_CRON,\s*cronIsTheRetry\]/, ); }); diff --git a/src/lib/jobs/__tests__/fitbit-queues.test.ts b/src/lib/jobs/__tests__/fitbit-queues.test.ts index 3e6145540..a9e9fbbc6 100644 --- a/src/lib/jobs/__tests__/fitbit-queues.test.ts +++ b/src/lib/jobs/__tests__/fitbit-queues.test.ts @@ -58,22 +58,22 @@ describe("reminder-worker — Fitbit sync queues", () => { it("schedules the hourly poll cron + the daily oauth-state sweep", () => { expect(source).toMatch(/\[FITBIT_SYNC_QUEUE,\s*FITBIT_SYNC_CRON\]/); expect(source).toMatch( - /\[FITBIT_OAUTH_STATE_CLEANUP_QUEUE,\s*FITBIT_OAUTH_STATE_CLEANUP_CRON\]/, + /\[\s*FITBIT_OAUTH_STATE_CLEANUP_QUEUE,\s*FITBIT_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, ); }); - it("registers a boss.work handler for every Fitbit queue", () => { + it("registers a createAndWork handler for every Fitbit queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,160}FITBIT_SYNC_QUEUE[\s\S]{0,160}handleFitbitSync/, + /createAndWork[\s\S]{0,160}FITBIT_SYNC_QUEUE[\s\S]{0,160}handleFitbitSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,200}FITBIT_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,200}FITBIT_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,200}FITBIT_SLEEP_REPAIR_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,200}FITBIT_SLEEP_REPAIR_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}FITBIT_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,160}handleFitbitOAuthStateCleanup/, + /createAndWork[\s\S]{0,160}FITBIT_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,160}handleFitbitOAuthStateCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/google-health-queues.test.ts b/src/lib/jobs/__tests__/google-health-queues.test.ts index 179091be9..783593da4 100644 --- a/src/lib/jobs/__tests__/google-health-queues.test.ts +++ b/src/lib/jobs/__tests__/google-health-queues.test.ts @@ -61,19 +61,19 @@ describe("reminder-worker — Google Health sync queues", () => { /\[GOOGLE_HEALTH_SYNC_QUEUE,\s*GOOGLE_HEALTH_SYNC_CRON\]/, ); expect(source).toMatch( - /\[\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON,?\s*\]/, + /\[\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, ); }); - it("registers a boss.work handler for every Google Health queue", () => { + it("registers a createAndWork handler for every Google Health queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}GOOGLE_HEALTH_SYNC_QUEUE[\s\S]{0,200}handleGoogleHealthSync/, + /createAndWork[\s\S]{0,200}GOOGLE_HEALTH_SYNC_QUEUE[\s\S]{0,200}handleGoogleHealthSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,320}GOOGLE_HEALTH_BACKFILL_QUEUE[\s\S]{0,320}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,320}GOOGLE_HEALTH_BACKFILL_QUEUE[\s\S]{0,320}enqueueIntegrationBackfillAdmission/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,200}GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,200}handleGoogleHealthOAuthStateCleanup/, + /createAndWork[\s\S]{0,200}GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,200}handleGoogleHealthOAuthStateCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/insight-pregenerate.test.ts b/src/lib/jobs/__tests__/insight-pregenerate.test.ts index e8bb06131..38c92003e 100644 --- a/src/lib/jobs/__tests__/insight-pregenerate.test.ts +++ b/src/lib/jobs/__tests__/insight-pregenerate.test.ts @@ -1181,9 +1181,9 @@ describe("queue registration", () => { ); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSrc).toMatch( - /boss\.work[\s\S]{0,80}INSIGHT_PREGENERATE_QUEUE/, + /createAndWork[\s\S]{0,80}INSIGHT_PREGENERATE_QUEUE/, ); }); diff --git a/src/lib/jobs/__tests__/insight-status-generate.test.ts b/src/lib/jobs/__tests__/insight-status-generate.test.ts index fe4a9b116..06ab5ed4e 100644 --- a/src/lib/jobs/__tests__/insight-status-generate.test.ts +++ b/src/lib/jobs/__tests__/insight-status-generate.test.ts @@ -6,7 +6,7 @@ * - an unknown metric is annotated and skipped, not thrown; * - the enqueue helper no-ops without a global boss instance and de-dupes * with a singletonKey when one is present; - * - the queue is registered in `allQueues` AND wired to a `boss.work` + * - the queue is registered in `allQueues` AND wired to a `createAndWork` * handler in reminder-worker.ts (the v1.4.37 W10 unregistered-queue * catch — a queue built but never registered silently never drains). */ @@ -171,9 +171,9 @@ describe("queue registration", () => { expect(match![1]).toMatch(/\bINSIGHT_STATUS_GENERATE_QUEUE\b/); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSrc).toMatch( - /boss\.work[\s\S]{0,120}INSIGHT_STATUS_GENERATE_QUEUE/, + /createAndWork[\s\S]{0,120}INSIGHT_STATUS_GENERATE_QUEUE/, ); }); diff --git a/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts b/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts index 16e1ae0ad..a99672474 100644 --- a/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts +++ b/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts @@ -3,7 +3,7 @@ * * Same source-text-grep approach as the drain + step-consolidation + * mean-consolidation regression guards: assert the queue is registered - * in `allQueues`, a `boss.work` handler is wired against it, and the + * in `allQueues`, a `createAndWork` handler is wired against it, and the * boot-discovery enqueue helper is called — without booting pg-boss + * Prisma. An unregistered queue silently never drains (the past incident * this guard exists to prevent). @@ -42,12 +42,12 @@ describe("reminder-worker — intake-slot-dedup wiring", () => { expect(allQueuesMatch![1]).toMatch(/\bINTAKE_SLOT_DEDUP_QUEUE\b/); }); - it("registers a boss.work handler against the intake-slot-dedup queue", () => { + it("registers a createAndWork handler against the intake-slot-dedup queue", () => { // v1.15.19 — span widened from {0,400}: the handler gained the daily // cron-tick dispatch branch (no `userId` → discovery fan-out) between // the queue name and the per-user dedup call. expect(source).toMatch( - /boss\.work[\s\S]{0,200}INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,1200}dedupeUserIntakeSlots/, + /createAndWork[\s\S]{0,200}INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,1400}dedupeUserIntakeSlots/, ); }); diff --git a/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts b/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts index 6215ec22f..7f42958a7 100644 --- a/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts +++ b/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts @@ -2,7 +2,7 @@ * MCP Phase 3 (M2) — mcp-token-cleanup queue registration guard. * * Source-text-grep guard like the other queue-wiring checks: assert the queue - * is in `allQueues`, scheduled, and wired to a `boss.work` handler. An + * is in `allQueues`, scheduled, and wired to a `createAndWork` handler. An * unregistered queue silently never drains (the recurring v1.4.37 dead-queue * bug), which would leave expired MCP connector access tokens growing unbounded. */ @@ -40,13 +40,13 @@ describe("reminder-worker — mcp-token-cleanup wiring", () => { it("schedules the mcp-token-cleanup cron", () => { expect(workerSource).toMatch( - /\[MCP_TOKEN_CLEANUP_QUEUE,\s*MCP_TOKEN_CLEANUP_CRON\]/, + /\[MCP_TOKEN_CLEANUP_QUEUE,\s*MCP_TOKEN_CLEANUP_CRON,\s*cronIsTheRetry\]/, ); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSource).toMatch( - /boss\.work[\s\S]{0,200}MCP_TOKEN_CLEANUP_QUEUE[\s\S]{0,200}handleMcpTokenCleanup/, + /createAndWork[\s\S]{0,200}MCP_TOKEN_CLEANUP_QUEUE[\s\S]{0,200}handleMcpTokenCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts b/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts index 3b5956ec6..4dbfd6d11 100644 --- a/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts +++ b/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts @@ -3,7 +3,7 @@ * * Same source-text-grep approach as the drain + step-consolidation * regression guards: assert the queue is registered in `allQueues`, a - * `boss.work` handler is wired against it, and the boot-discovery + * `createAndWork` handler is wired against it, and the boot-discovery * enqueue helper is called — without booting pg-boss + Prisma. An * unregistered queue silently never drains. */ @@ -34,9 +34,9 @@ describe("reminder-worker — mean-consolidation wiring", () => { expect(allQueuesMatch![1]).toMatch(/\bMEAN_CONSOLIDATION_QUEUE\b/); }); - it("registers a boss.work handler against the mean-consolidation queue", () => { + it("registers a createAndWork handler against the mean-consolidation queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}MEAN_CONSOLIDATION_QUEUE[\s\S]{0,400}runMeanConsolidationForUser/, + /createAndWork[\s\S]{0,200}MEAN_CONSOLIDATION_QUEUE[\s\S]{0,400}runMeanConsolidationForUser/, ); }); diff --git a/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts b/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts index 9c36703d4..564b9cb95 100644 --- a/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts +++ b/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts @@ -5,7 +5,7 @@ * guards: assert the Vorsorge-reminder queue is declared, listed in * `allQueues` (the v1.4.37 dead-queue lesson — an unregistered queue is * never provisioned and the schedule silently no-ops), scheduled on its - * cron, and wired to a `boss.work` handler that delegates to + * cron, and wired to a `createAndWork` handler that delegates to * `runMeasurementReminderTick`. No pg-boss / Prisma boot. */ import { readFileSync } from "node:fs"; @@ -56,9 +56,9 @@ describe("reminder-worker — measurement-reminder wiring", () => { ); }); - it("registers a boss.work handler that runs the tick", () => { + it("registers a createAndWork handler that runs the tick", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}MEASUREMENT_REMINDER_QUEUE[\s\S]{0,200}handleMeasurementReminderCheck/, + /createAndWork[\s\S]{0,200}MEASUREMENT_REMINDER_QUEUE[\s\S]{0,200}handleMeasurementReminderCheck/, ); expect(source).toMatch( /handleMeasurementReminderCheck[\s\S]{0,400}runMeasurementReminderTick/, @@ -80,12 +80,12 @@ describe("reminder-worker — eventful reminder-satisfy wiring (v1.18.1)", () => expect(allQueuesMatch![1]).toMatch(/\bREMINDER_SATISFY_QUEUE\b/); }); - it("binds a boss.work handler for the satisfy queue", () => { + it("binds a createAndWork handler for the satisfy queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}REMINDER_SATISFY_QUEUE[\s\S]{0,200}handleReminderSatisfy/, + /createAndWork[\s\S]{0,200}REMINDER_SATISFY_QUEUE[\s\S]{0,200}handleReminderSatisfy/, ); expect(source).toMatch( - /handleReminderSatisfy[\s\S]{0,600}runReminderSatisfyForUser/, + /handleReminderSatisfy[\s\S]{0,800}runReminderSatisfyForUser/, ); }); }); diff --git a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts index ed96efe98..fa68a9100 100644 --- a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts +++ b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts @@ -7,7 +7,7 @@ * recently-tombstoned rows untouched. * 2. Source-grep wiring (same approach as the drain-cumulative guard): * the queue is registered in `allQueues`, scheduled, and bound to a - * `boss.work` handler — a missing `allQueues` entry silently no-ops + * `createAndWork` handler — a missing `allQueues` entry silently no-ops * the schedule under pg-boss v12. */ import { readFileSync } from "node:fs"; @@ -114,9 +114,9 @@ describe("reminder-worker — measurement-tombstone-cleanup schedule", () => { ); }); - it("binds a boss.work handler to the queue", () => { + it("binds a createAndWork handler to the queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE[\s\S]{0,200}handleMeasurementTombstoneCleanup/, + /createAndWork[\s\S]{0,200}MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE[\s\S]{0,200}handleMeasurementTombstoneCleanup/, ); }); }); diff --git a/src/lib/jobs/__tests__/nightscout-queues.test.ts b/src/lib/jobs/__tests__/nightscout-queues.test.ts index 5873b8382..6c994d55a 100644 --- a/src/lib/jobs/__tests__/nightscout-queues.test.ts +++ b/src/lib/jobs/__tests__/nightscout-queues.test.ts @@ -50,9 +50,9 @@ describe("reminder-worker — Nightscout sync queue", () => { expect(source).toMatch(/\[NIGHTSCOUT_SYNC_QUEUE,\s*NIGHTSCOUT_SYNC_CRON\]/); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,160}NIGHTSCOUT_SYNC_QUEUE[\s\S]{0,160}handleNightscoutSync/, + /createAndWork[\s\S]{0,160}NIGHTSCOUT_SYNC_QUEUE[\s\S]{0,160}handleNightscoutSync/, ); }); }); @@ -85,12 +85,12 @@ describe("reminder-worker — Polar / Oura sync queues", () => { expect(source).toMatch(/\[OURA_SYNC_QUEUE,\s*OURA_SYNC_CRON\]/); }); - it("registers a boss.work handler for each queue", () => { + it("registers a createAndWork handler for each queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,160}POLAR_SYNC_QUEUE[\s\S]{0,160}handlePolarSync/, + /createAndWork[\s\S]{0,160}POLAR_SYNC_QUEUE[\s\S]{0,160}handlePolarSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}OURA_SYNC_QUEUE[\s\S]{0,160}handleOuraSync/, + /createAndWork[\s\S]{0,160}OURA_SYNC_QUEUE[\s\S]{0,160}handleOuraSync/, ); }); }); diff --git a/src/lib/jobs/__tests__/period-narrative-warm.test.ts b/src/lib/jobs/__tests__/period-narrative-warm.test.ts index 45838182e..5c9e6ea83 100644 --- a/src/lib/jobs/__tests__/period-narrative-warm.test.ts +++ b/src/lib/jobs/__tests__/period-narrative-warm.test.ts @@ -196,8 +196,10 @@ describe("queue registration", () => { ); }); - it("registers a boss.work handler for the queue", () => { - expect(workerSrc).toMatch(/boss\.work[\s\S]{0,120}PERIOD_NARRATIVE_QUEUE/); + it("registers a createAndWork handler for the queue", () => { + expect(workerSrc).toMatch( + /createAndWork[\s\S]{0,120}PERIOD_NARRATIVE_QUEUE/, + ); }); it("exposes a sane queue name + nightly cron", () => { diff --git a/src/lib/jobs/__tests__/recovery-score-queue.test.ts b/src/lib/jobs/__tests__/recovery-score-queue.test.ts index 5e7a15866..65628b9e9 100644 --- a/src/lib/jobs/__tests__/recovery-score-queue.test.ts +++ b/src/lib/jobs/__tests__/recovery-score-queue.test.ts @@ -3,7 +3,7 @@ * * Same source-text-grep approach as the other queue-wiring guards: assert * the queue is imported, registered in `allQueues`, scheduled, and wired to - * a `boss.work` handler — without booting pg-boss + Prisma. An unregistered + * a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered * queue silently never drains (the recurring past bug this guards against). * * Also pins the iOS-write rejection of the server-owned COMPUTED source on @@ -57,9 +57,9 @@ describe("reminder-worker — recovery-score wiring", () => { ); }); - it("registers a boss.work handler that runs the recovery-score pass", () => { + it("registers a createAndWork handler that runs the recovery-score pass", () => { expect(workerSource).toMatch( - /boss\.work[\s\S]{0,200}RECOVERY_SCORE_QUEUE[\s\S]{0,400}runRecoveryScore/, + /createAndWork[\s\S]{0,200}RECOVERY_SCORE_QUEUE[\s\S]{0,400}runRecoveryScore/, ); }); }); diff --git a/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts index c6375c871..cc803ecc5 100644 --- a/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts +++ b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts @@ -3,7 +3,7 @@ * * The reminder-worker imports heavy infrastructure, so we read the source as * text and assert the queue constant is declared, registered in `allQueues`, - * wired to a `boss.work` handler, and enqueued at boot — the fast guard that + * wired to a `createAndWork` handler, and enqueued at boot — the fast guard that * catches the v1.4.37 dead-queue class (a queue declared but never registered, * which silently never drains). */ @@ -30,9 +30,9 @@ describe("reminder-worker — sleep-timeline backfill queue", () => { expect(block![1]!).toContain("SLEEP_TIMELINE_BACKFILL_QUEUE"); }); - it("wires a boss.work handler through the shared admission queue", () => { + it("wires a createAndWork handler through the shared admission queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,260}SLEEP_TIMELINE_BACKFILL_QUEUE[\s\S]{0,300}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,260}SLEEP_TIMELINE_BACKFILL_QUEUE[\s\S]{0,300}enqueueIntegrationBackfillAdmission/, ); }); diff --git a/src/lib/jobs/__tests__/status-cron-candidates.test.ts b/src/lib/jobs/__tests__/status-cron-candidates.test.ts index 15c13baa8..4f2690a1f 100644 --- a/src/lib/jobs/__tests__/status-cron-candidates.test.ts +++ b/src/lib/jobs/__tests__/status-cron-candidates.test.ts @@ -196,8 +196,8 @@ describe("mood-status queue registration (dead-queue guard)", () => { expect(workerSrc).toMatch(/MOOD_STATUS_CRON\s*=\s*"30 2 \* \* \*"/); }); - it("registers a boss.work handler for the queue", () => { - expect(workerSrc).toMatch(/boss\.work[\s\S]{0,120}MOOD_STATUS_QUEUE/); + it("registers a createAndWork handler for the queue", () => { + expect(workerSrc).toMatch(/createAndWork[\s\S]{0,120}MOOD_STATUS_QUEUE/); }); it("drives all seven status crons through the shared discovery", () => { diff --git a/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts b/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts index 8e0eeab75..e34c295cd 100644 --- a/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts +++ b/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts @@ -4,7 +4,7 @@ * * Same source-text-grep approach as the drain + step-consolidation + * mean-consolidation guards: assert the repair queue is registered in - * `allQueues`, a `boss.work` handler is wired against it, and the + * `allQueues`, a `createAndWork` handler is wired against it, and the * boot-discovery enqueue helper fires — without booting pg-boss + Prisma. * An unregistered queue silently never drains (the dead-queue class). */ @@ -36,9 +36,9 @@ describe("reminder-worker — step-consolidation-repair wiring", () => { expect(allQueuesMatch![1]).toMatch(/\bSTEP_CONSOLIDATION_REPAIR_QUEUE\b/); }); - it("registers a boss.work handler against the repair queue", () => { + it("registers a createAndWork handler against the repair queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}STEP_CONSOLIDATION_REPAIR_QUEUE[\s\S]{0,400}runStepConsolidationRepairForUser/, + /createAndWork[\s\S]{0,200}STEP_CONSOLIDATION_REPAIR_QUEUE[\s\S]{0,400}runStepConsolidationRepairForUser/, ); }); diff --git a/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts b/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts index 839c2b575..969d80fba 100644 --- a/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts +++ b/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts @@ -5,7 +5,7 @@ * * Same source-text-grep approach as the WX-C recovery-score guard: assert * each queue is imported, registered in `allQueues`, scheduled (the two - * cron-driven score jobs), and wired to a `boss.work` handler — without + * cron-driven score jobs), and wired to a `createAndWork` handler — without * booting pg-boss + Prisma. An unregistered queue silently never drains * (the recurring past bug this guards against). */ @@ -67,9 +67,9 @@ describe("reminder-worker — stress-score wiring", () => { expect(source).toMatch(/\[STRESS_SCORE_QUEUE,\s*STRESS_SCORE_CRON\]/); }); - it("registers a boss.work handler that runs the stress-score pass", () => { + it("registers a createAndWork handler that runs the stress-score pass", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}STRESS_SCORE_QUEUE[\s\S]{0,400}runStressScore/, + /createAndWork[\s\S]{0,200}STRESS_SCORE_QUEUE[\s\S]{0,400}runStressScore/, ); }); }); @@ -90,9 +90,9 @@ describe("reminder-worker — strain-score wiring", () => { expect(source).toMatch(/\[STRAIN_SCORE_QUEUE,\s*STRAIN_SCORE_CRON\]/); }); - it("registers a boss.work handler that runs the strain-score pass", () => { + it("registers a createAndWork handler that runs the strain-score pass", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}STRAIN_SCORE_QUEUE[\s\S]{0,400}runStrainScore/, + /createAndWork[\s\S]{0,200}STRAIN_SCORE_QUEUE[\s\S]{0,400}runStrainScore/, ); }); }); @@ -111,9 +111,9 @@ describe("reminder-worker — dense intra-day retention wiring", () => { expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_RETENTION_QUEUE\b/); }); - it("registers a boss.work handler against the dense-intraday-retention queue", () => { + it("registers a createAndWork handler against the dense-intraday-retention queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,250}DENSE_INTRADAY_RETENTION_QUEUE[\s\S]{0,500}runDenseIntradayRetentionForUser/, + /createAndWork[\s\S]{0,250}DENSE_INTRADAY_RETENTION_QUEUE[\s\S]{0,500}runDenseIntradayRetentionForUser/, ); }); @@ -163,9 +163,9 @@ describe("reminder-worker — dense hourly-rebuild wiring", () => { expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_HOURLY_REBUILD_QUEUE\b/); }); - it("registers a boss.work handler against the hourly-rebuild queue", () => { + it("registers a createAndWork handler against the hourly-rebuild queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,250}DENSE_INTRADAY_HOURLY_REBUILD_QUEUE[\s\S]{0,500}runDenseIntradayHourlyRebuildForUser/, + /createAndWork[\s\S]{0,250}DENSE_INTRADAY_HOURLY_REBUILD_QUEUE[\s\S]{0,500}runDenseIntradayHourlyRebuildForUser/, ); }); diff --git a/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts b/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts index 68764d932..9fe67d0b0 100644 --- a/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts +++ b/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts @@ -3,7 +3,7 @@ * * Same source-text-grep approach as the other queue-wiring guards: assert * the queue is imported, registered in `allQueues`, scheduled, and wired to - * a `boss.work` handler — without booting pg-boss + Prisma. An unregistered + * a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered * queue silently never drains (the recurring v1.4.37 dead-queue bug), which * would leave the pinned-leaf-rotation alarm dark. */ @@ -50,9 +50,9 @@ describe("reminder-worker — tls-pin-monitor wiring", () => { ); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSource).toMatch( - /boss\.work[\s\S]{0,200}TLS_PIN_MONITOR_QUEUE[\s\S]{0,200}handleTlsPinMonitor/, + /createAndWork[\s\S]{0,200}TLS_PIN_MONITOR_QUEUE[\s\S]{0,200}handleTlsPinMonitor/, ); }); diff --git a/src/lib/jobs/__tests__/whoop-queues.test.ts b/src/lib/jobs/__tests__/whoop-queues.test.ts index 7d2c514c1..a1ad468b4 100644 --- a/src/lib/jobs/__tests__/whoop-queues.test.ts +++ b/src/lib/jobs/__tests__/whoop-queues.test.ts @@ -77,25 +77,25 @@ describe("reminder-worker — WHOOP sync queues", () => { /\[WHOOP_CYCLE_SYNC_QUEUE,\s*WHOOP_CYCLE_SYNC_CRON\]/, ); expect(source).toMatch( - /\[WHOOP_OAUTH_STATE_CLEANUP_QUEUE,\s*WHOOP_OAUTH_STATE_CLEANUP_CRON\]/, + /\[\s*WHOOP_OAUTH_STATE_CLEANUP_QUEUE,\s*WHOOP_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, ); }); - it("registers a boss.work handler for every WHOOP sync queue", () => { + it("registers a createAndWork handler for every WHOOP sync queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,160}WHOOP_RECOVERY_SYNC_QUEUE[\s\S]{0,160}handleWhoopRecoverySync/, + /createAndWork[\s\S]{0,160}WHOOP_RECOVERY_SYNC_QUEUE[\s\S]{0,160}handleWhoopRecoverySync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}WHOOP_SLEEP_SYNC_QUEUE[\s\S]{0,160}handleWhoopSleepSync/, + /createAndWork[\s\S]{0,160}WHOOP_SLEEP_SYNC_QUEUE[\s\S]{0,160}handleWhoopSleepSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}WHOOP_WORKOUT_SYNC_QUEUE[\s\S]{0,160}handleWhoopWorkoutSync/, + /createAndWork[\s\S]{0,160}WHOOP_WORKOUT_SYNC_QUEUE[\s\S]{0,160}handleWhoopWorkoutSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}WHOOP_CYCLE_SYNC_QUEUE[\s\S]{0,160}handleWhoopCycleSync/, + /createAndWork[\s\S]{0,160}WHOOP_CYCLE_SYNC_QUEUE[\s\S]{0,160}handleWhoopCycleSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,200}WHOOP_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,200}WHOOP_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, ); }); diff --git a/src/lib/jobs/__tests__/whoop-resource-sync.test.ts b/src/lib/jobs/__tests__/whoop-resource-sync.test.ts index 6359d2ecb..cbe181b6d 100644 --- a/src/lib/jobs/__tests__/whoop-resource-sync.test.ts +++ b/src/lib/jobs/__tests__/whoop-resource-sync.test.ts @@ -128,7 +128,18 @@ describe("runWhoopResourceSync dispatch", () => { findManyMock.mockResolvedValue([{ userId: "a" }, { userId: "b" }]); syncUserWorkout.mockRejectedValueOnce(new Error("boom for a")); - await expect(handleWhoopWorkoutSync([job({})])).resolves.toBeUndefined(); + // The pass is what the outcome judges: the sweep ran, so the job is + // done. The revoked-grant user rides out as `users_failed` rather than + // failing the queue and retrying the whole cohort over one account. + await expect(handleWhoopWorkoutSync([job({})])).resolves.toEqual({ + ok: true, + did: { + total: 2, + users_synced: 1, + users_failed: 1, + measurements_imported: 0, + }, + }); // Both users were attempted despite the first throwing. expect(syncUserWorkout).toHaveBeenCalledTimes(2); }); diff --git a/src/lib/jobs/__tests__/withings-ecg-worker.test.ts b/src/lib/jobs/__tests__/withings-ecg-worker.test.ts index cb11bde9e..e40bf868c 100644 --- a/src/lib/jobs/__tests__/withings-ecg-worker.test.ts +++ b/src/lib/jobs/__tests__/withings-ecg-worker.test.ts @@ -93,7 +93,15 @@ describe("handleWithingsEcgSync", () => { enddate: 1715000060, }); - await expect(handleWithingsEcgSync([queued])).resolves.toBeUndefined(); + await expect(handleWithingsEcgSync([queued])).resolves.toEqual({ + ok: true, + did: { + total: 1, + users_synced: 1, + users_failed: 0, + recordings_imported: 1, + }, + }); expect(ecgFindMany).not.toHaveBeenCalled(); expect(syncUserEcg).toHaveBeenCalledTimes(1); @@ -120,14 +128,24 @@ describe("handleWithingsEcgSync", () => { it("one user's failure never starves the rest of the cohort", async () => { ecgFindMany.mockResolvedValue([{ userId: "a" }, { userId: "b" }]); + syncUserEcg.mockResolvedValue(0); syncUserEcg.mockRejectedValueOnce(new Error("boom for a")); // Per-user catch-and-warn (mirrors the activity / sleep handlers): the - // handler resolves, both users attempted, the leaf records its own ledger - // failure so honesty is preserved without rejecting the whole batch. + // pass ran, so the job is done and the failing user rides out as a + // `users_failed` count. The leaf records its own ledger failure, so + // honesty is preserved without rejecting the whole batch. await expect( handleWithingsEcgSync([job({} as WithingsEcgSyncPayload)]), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ + ok: true, + did: { + total: 2, + users_synced: 1, + users_failed: 1, + recordings_imported: 0, + }, + }); expect(syncUserEcg).toHaveBeenCalledTimes(2); }); }); @@ -143,7 +161,7 @@ describe("withings ECG queue registration", () => { /\[WITHINGS_ECG_SYNC_QUEUE\]:\s*\{[\s\S]*?policy:\s*"short"/, ); expect(registrarSource).toMatch( - /boss\.work\([\s\S]*?WITHINGS_ECG_SYNC_QUEUE[\s\S]*?handleWithingsEcgSync/, + /createAndWork\(\s*boss,\s*WITHINGS_ECG_SYNC_QUEUE[\s\S]{0,120}handleWithingsEcgSync/, ); }); }); diff --git a/src/lib/jobs/__tests__/withings-queues.test.ts b/src/lib/jobs/__tests__/withings-queues.test.ts index 5dc7dac08..9861c41a8 100644 --- a/src/lib/jobs/__tests__/withings-queues.test.ts +++ b/src/lib/jobs/__tests__/withings-queues.test.ts @@ -50,14 +50,14 @@ describe("reminder-worker — Withings activity + sleep v2 queues", () => { it("registers the activity-sync handler against pg-boss", () => { expect(source).toMatch(/handleWithingsActivitySync\b/); expect(source).toMatch( - /boss\.work[\s\S]{0,200}WITHINGS_ACTIVITY_QUEUE[\s\S]{0,200}handleWithingsActivitySync/, + /createAndWork[\s\S]{0,200}WITHINGS_ACTIVITY_QUEUE[\s\S]{0,200}handleWithingsActivitySync/, ); }); it("registers the sleep-sync handler against pg-boss", () => { expect(source).toMatch(/handleWithingsSleepSync\b/); expect(source).toMatch( - /boss\.work[\s\S]{0,200}WITHINGS_SLEEP_QUEUE[\s\S]{0,200}handleWithingsSleepSync/, + /createAndWork[\s\S]{0,200}WITHINGS_SLEEP_QUEUE[\s\S]{0,200}handleWithingsSleepSync/, ); }); @@ -78,7 +78,7 @@ describe("reminder-worker — Withings activity + sleep v2 queues", () => { }); it("schedules the ECG catch-net cron at :41 so watch-only accounts pull strips", () => { - // The queue is already in allQueues + boss.work-bound; W-1 adds the missing + // The queue is already in allQueues + createAndWork-bound; W-1 adds the missing // schedule entry so a payload-less cron tick walks the connection cohort. expect(source).toMatch( /WITHINGS_ECG_SYNC_CRON\s*=\s*["']41 \* \* \* \*["']/, diff --git a/src/lib/jobs/__tests__/withings-subscription-retry.test.ts b/src/lib/jobs/__tests__/withings-subscription-retry.test.ts index e873c9ca5..be6de5b0e 100644 --- a/src/lib/jobs/__tests__/withings-subscription-retry.test.ts +++ b/src/lib/jobs/__tests__/withings-subscription-retry.test.ts @@ -72,7 +72,16 @@ describe("handleWithingsFallbackSync subscription repair", () => { data: { triggeredAt: "2026-07-21T10:00:00.000Z" }, } as Job; - await handleWithingsFallbackSync([queued]); + await expect(handleWithingsFallbackSync([queued])).resolves.toEqual({ + ok: true, + did: { + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + subscription_repair_failed: false, + }, + }); expect(retryDueWithingsWebhookSubscriptions).toHaveBeenCalledTimes(1); expect(syncUserMeasurements).not.toHaveBeenCalled(); @@ -89,7 +98,19 @@ describe("handleWithingsFallbackSync subscription repair", () => { data: { triggeredAt: "2026-07-21T11:00:00.000Z" }, } as Job; - await expect(handleWithingsFallbackSync([queued])).resolves.toBeUndefined(); + // The repair is an auxiliary leg, not the pass this queue is named for: + // the fallback sync still ran for everyone, so the job is done and the + // repair failure rides out as a fact on the outcome. + await expect(handleWithingsFallbackSync([queued])).resolves.toEqual({ + ok: true, + did: { + total: 2, + users_synced: 2, + users_failed: 0, + measurements_imported: 0, + subscription_repair_failed: true, + }, + }); expect(findMany).toHaveBeenCalledWith({ select: { userId: true } }); expect(syncUserMeasurements).toHaveBeenNthCalledWith( diff --git a/src/lib/jobs/__tests__/workout-insight-queue.test.ts b/src/lib/jobs/__tests__/workout-insight-queue.test.ts index 4fdc3ca15..7af32f112 100644 --- a/src/lib/jobs/__tests__/workout-insight-queue.test.ts +++ b/src/lib/jobs/__tests__/workout-insight-queue.test.ts @@ -32,9 +32,9 @@ describe("workout-insight-generate queue wiring", () => { expect(allQueues![1]).toMatch(/\bWORKOUT_INSIGHT_GENERATE_QUEUE\b/); }); - it("binds a boss.work handler that drains it", () => { + it("binds a createAndWork handler that drains it", () => { expect(statusRegistrar).toMatch( - /boss\.work[\s\S]{0,300}WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}handleWorkoutInsightGenerate/, + /createAndWork[\s\S]{0,300}WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}handleWorkoutInsightGenerate/, ); }); @@ -50,8 +50,10 @@ describe("workout-insight-generate queue wiring", () => { it("has no cron schedule — it is dispatched by arrival only", () => { // A cron here would be a standing invitation to regenerate, which is the // saturation surface the design refuses. + // Tolerates the optional third schedule element, so a cron sneaking in + // with send options still trips this. expect(statusRegistrar).not.toMatch( - /\[WORKOUT_INSIGHT_GENERATE_QUEUE,\s*\w+_CRON\]/, + /\[\s*WORKOUT_INSIGHT_GENERATE_QUEUE,\s*\w+_CRON\s*[,\]]/, ); }); diff --git a/src/lib/jobs/reminder/__tests__/poll-cohort.test.ts b/src/lib/jobs/reminder/__tests__/poll-cohort.test.ts index f0dc9123d..111c255ab 100644 --- a/src/lib/jobs/reminder/__tests__/poll-cohort.test.ts +++ b/src/lib/jobs/reminder/__tests__/poll-cohort.test.ts @@ -107,7 +107,17 @@ describe("makePollCohortHandler — marker-aware recordFailure boundary", () => recordFailure, }); - await expect(handler([job()])).resolves.toBeUndefined(); + // The recorder rejection is warned, not propagated: the pass ran, so the + // job is done and the failing user is a `users_failed` count. + await expect(handler([job()])).resolves.toEqual({ + ok: true, + did: { + total: 2, + users_synced: 1, + users_failed: 1, + measurements_imported: 1, + }, + }); // The next user still synced despite the recorder throwing. expect(syncUser).toHaveBeenCalledTimes(2); expect(syncUser).toHaveBeenLastCalledWith("next"); From 1f47db59554daee17a779fc1030bae54a0467d85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Mon, 27 Jul 2026 23:29:20 +0200 Subject: [PATCH 6/6] docs(jobs): name the binding contract the registrars actually use --- src/lib/jobs/pr-detection.ts | 2 +- src/lib/jobs/reminder-worker.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/jobs/pr-detection.ts b/src/lib/jobs/pr-detection.ts index e373e29c4..8062dc74e 100644 --- a/src/lib/jobs/pr-detection.ts +++ b/src/lib/jobs/pr-detection.ts @@ -2,7 +2,7 @@ * v1.4.25 W16c — Enqueue helper + handler glue for the pg-boss * `pr-detection` queue. Lives in `src/lib/jobs/` next to the other * job-side helpers; the worker process binds the handler in - * `reminder-worker.ts` via `boss.work(PR_DETECTION_QUEUE, ...)`. + * the maintenance registrar via `createAndWork(boss, PR_DETECTION_QUEUE, …)`. * * The Next.js side (ingest routes) imports `enqueuePrDetection` and * never touches pg-boss directly — `getGlobalBoss()` returns the diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index 59c7036db..f8a20a384 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -8,8 +8,8 @@ * the domain registrars under `src/lib/jobs/reminder/`. Each registrar owns the * four facts the v1.4.37 dead-queue guards pin — the queue-name constant, its * `allQueues` membership, its `[QUEUE, CRON]` schedule tuple, and its - * `boss.work(QUEUE, …, handler)` binding — so a queue can never be declared - * without being provisioned, scheduled, and drained. + * `createAndWork(boss, QUEUE, …, handler)` binding — so a queue can never be + * declared without being provisioned, scheduled, and drained. * * Usage: Run as a standalone process or call startReminderWorker() from a * custom server setup. In dev, use: npx tsx src/lib/jobs/reminder-worker.ts