diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts new file mode 100644 index 000000000..d90b90e05 --- /dev/null +++ b/src/lib/jobs/job-outcome.ts @@ -0,0 +1,74 @@ +/** + * The value a background-job handler returns to say what it did. + * + * Before this type existed, a pg-boss handler signalled success by falling + * off the end of a function. Every one of the queue bindings in this tree + * resolved to `undefined`, and none of them was read anywhere, so a handler + * that caught an error, logged a warning and returned was indistinguishable + * from one that finished its work. pg-boss marked both `completed`; the + * operator surfaces that read `pgboss.job` saw nothing; the nightly pass + * looked healthy for as long as it kept failing. + * + * A returned `JobOutcome` closes that specific gap: completion is now + * derived from a value the handler had to construct, and `runJob` turns + * `ok: false` into a failed pg-boss job. What the type CANNOT do is make a + * handler honest — `jobDone()` after swallowing an exception compiles and + * passes every check in this repository. It converts omission (forgetting to + * signal a failure) into commission (claiming a success that did not + * happen), which is a smaller and much more reviewable class of defect. + * + * `did` is the small pinned-shape bag of facts about the run — counts, + * identifiers, booleans. It rides into the failure report and, once the + * failure ledger lands, into the operator surface. Keep it to scalars that + * a dashboard can key on; it is not a place for payload data or free text. + */ + +/** A single fact about a run. Scalars only — never payload data. */ +export type JobFact = string | number | boolean; + +/** The pinned-shape bag of facts a handler reports about its run. */ +export type JobFacts = Readonly>; + +/** + * What a handler did. `ok: true` means the work named by the queue actually + * happened; `ok: false` means it did not, and `runJob` will fail the pg-boss + * job so the failure reaches the retry policy and the operator surfaces. + * + * A run that legitimately had nothing to do is `ok: true` with a zero count + * in `did` — absence of work is not failure. A run that could not do its + * work (provider down, query timed out, key missing) is `ok: false` even + * when the handler previously chose to carry on. + */ +export type JobOutcome = + | { + readonly ok: true; + readonly did: JobFacts; + } + | { + readonly ok: false; + /** Short, stable, greppable. `"nightscout fetch failed"`, not a stack. */ + readonly reason: string; + /** The thrown value, when there was one. Preserved for the stack. */ + readonly cause?: unknown; + /** Whatever the run managed before it failed. */ + readonly did?: JobFacts; + }; + +/** The run did its work. Pass the counts that make the run auditable. */ +export function jobDone(did: JobFacts = {}): JobOutcome { + return { ok: true, did }; +} + +/** + * The run did not do its work. `runJob` reports this and rethrows, so the + * pg-boss job fails and the queue's retry policy applies — check that policy + * when converting a handler that used to swallow, because a deterministic + * failure now retries instead of passing quietly. + */ +export function jobFailed( + reason: string, + cause?: unknown, + did?: JobFacts, +): JobOutcome { + return { ok: false, reason, cause, did }; +} diff --git a/src/lib/jobs/reminder/registrar-shared.ts b/src/lib/jobs/reminder/registrar-shared.ts index ceb84ea2b..178c580e6 100644 --- a/src/lib/jobs/reminder/registrar-shared.ts +++ b/src/lib/jobs/reminder/registrar-shared.ts @@ -16,13 +16,18 @@ * assert a single aggregate `allQueues` (defence in depth: the per-registrar * arrays AND the boot-level union both have to agree). */ -import { PgBoss } from "pg-boss"; +import { PgBoss, type WorkOptions } from "pg-boss"; import { prisma } from "@/lib/db"; import { workerLog } from "./shared"; import { withBackgroundEvent } from "@/lib/logging/background"; import { annotate } from "@/lib/logging/context"; +import { + runJob, + type JobHandler, + type JobWithMetadataHandler, +} from "@/lib/jobs/run-job"; /** * Cron schedule tuple: `[queueName, cronExpression, sendOptions?]`. The @@ -198,3 +203,41 @@ export async function createAndSchedule( ); } } + +/** + * Bind a handler to a queue. The one legal way to reach `boss.work` in this + * tree — the `healthlog/job-handler-outcome` lint rule refuses a bare + * `boss.work(` anywhere else. + * + * The point of routing every binding through here is the handler type: + * `JobHandler` resolves to `Promise`, so a handler cannot be + * bound unless it returns a value declaring what it did. `runJob` then fails + * the pg-boss job on `ok: false` instead of completing it. Falling off the + * end of a handler, which used to read as success, is now a type error. + * + * Argument order matches `boss.work(queue, options, handler)` so a binding + * reads the same as it did before, with `boss` moved to the front. + */ +export async function createAndWork( + boss: PgBoss, + queue: string, + options: WorkOptions & { includeMetadata: true }, + handler: JobWithMetadataHandler, +): Promise; +export async function createAndWork( + boss: PgBoss, + queue: string, + options: WorkOptions, + handler: JobHandler, +): Promise; +export async function createAndWork( + boss: PgBoss, + queue: string, + options: WorkOptions, + handler: JobHandler | JobWithMetadataHandler, +): Promise { + // `JobWithMetadata` extends `Job`, so the metadata handler accepts + // everything the plain one does; the overloads above are what keeps the + // call sites honest about which shape they asked pg-boss for. + await boss.work(queue, options, runJob(queue, handler as JobHandler)); +} diff --git a/src/lib/jobs/run-job.ts b/src/lib/jobs/run-job.ts new file mode 100644 index 000000000..27fcd268a --- /dev/null +++ b/src/lib/jobs/run-job.ts @@ -0,0 +1,88 @@ +/** + * The wrapper that turns a returned `JobOutcome` into a pg-boss result. + * + * `boss.work(queue, opts, handler)` completes a job when the handler's + * promise resolves, whatever it resolves to. Wrapping every handler in + * `runJob` moves that decision from "the function returned" to "the function + * said it succeeded": an `ok: false` outcome is reported and rethrown, so + * pg-boss records a failed job, the queue's retry policy applies, and the + * failed-job readers in `job-failures.ts` surface it to the operator. + * + * This changes runtime behaviour, not only visibility. Handlers that used to + * catch, warn and return now fail their job. For a queue whose failure mode + * is deterministic — a DELETE that times out will time out again — retrying + * is wasted load, so the retry policy of every converted queue has to be a + * decision rather than a default. + * + * The wrapper deliberately does not open a wide event of its own. Handlers + * own their `withBackgroundEvent` scope and annotate inside it; a second + * event here would double every worker line for no added fact. + */ +import type { Job, JobWithMetadata } from "pg-boss"; + +import type { JobFact, JobFacts, JobOutcome } from "./job-outcome"; +import { reportWorkerError } from "./report-worker-error"; + +/** A converted handler: it says what it did instead of just finishing. */ +export type JobHandler = (jobs: Job[]) => Promise; + +/** The same, for a queue bound with `includeMetadata: true`. */ +export type JobWithMetadataHandler = ( + jobs: JobWithMetadata[], +) => Promise; + +/** + * The error a failed outcome becomes. Carries the queue and the handler's + * own reason so the pg-boss `output` column, the GlitchTip event and the + * container log all name the same thing. + */ +export class JobFailure extends Error { + readonly queue: string; + readonly reason: string; + + constructor(queue: string, reason: string, cause?: unknown) { + super(`${queue}: ${reason}`, cause === undefined ? undefined : { cause }); + this.name = "JobFailure"; + this.queue = queue; + this.reason = reason; + } +} + +function factMeta(did: JobFacts | undefined): Record { + const meta: Record = {}; + for (const [key, value] of Object.entries(did ?? {})) meta[key] = value; + return meta; +} + +/** + * Wrap a handler for `boss.work`. Resolves on `ok: true`; reports and + * rethrows on `ok: false`. + * + * A handler that throws is left alone — the throw already fails the job, and + * re-wrapping it would only bury the original stack. `runJob` exists for the + * case the throw never happens. + */ +export function runJob( + queue: string, + handler: (jobs: J[]) => Promise, +): (jobs: J[]) => Promise { + return async (jobs: J[]): Promise => { + const outcome = await handler(jobs); + if (outcome.ok) return; + + const failure = new JobFailure(queue, outcome.reason, outcome.cause); + const causeMessage = + outcome.cause instanceof Error + ? outcome.cause.message + : typeof outcome.cause === "string" + ? outcome.cause + : undefined; + + await reportWorkerError(queue, failure, { + ...factMeta(outcome.did), + ...(causeMessage === undefined ? {} : { cause: causeMessage }), + }); + + throw failure; + }; +}