feat(jobs): derive job completion from a returned outcome

A pg-boss handler signalled success by falling off the end of a function.
Every binding in this tree resolved to undefined and nothing read the
value, so a handler that caught an error, logged a warning and returned
was indistinguishable from one that finished its work: pg-boss marked
both completed and the failed-job readers saw nothing.

JobOutcome makes the completion a value the handler has to construct.
runJob turns ok:false into a reported, rethrown failure so the queue's
retry policy applies. createAndWork is the one binding path; its handler
type is (jobs) => Promise<JobOutcome>, so a handler that returns nothing
no longer compiles.

The type cannot make a handler honest — jobDone() after a swallow still
compiles. It converts omission into commission, which is the smaller and
far more reviewable class.

src/lib/jobs/job-outcome.ts

@@ -0,0 +1,74 @@
+/**
+ * The value a background-job handler returns to say what it did.
+ *
+ * Before this type existed, a pg-boss handler signalled success by falling
+ * off the end of a function. Every one of the queue bindings in this tree
+ * resolved to `undefined`, and none of them was read anywhere, so a handler
+ * that caught an error, logged a warning and returned was indistinguishable
+ * from one that finished its work. pg-boss marked both `completed`; the
+ * operator surfaces that read `pgboss.job` saw nothing; the nightly pass
+ * looked healthy for as long as it kept failing.
+ *
+ * A returned `JobOutcome` closes that specific gap: completion is now
+ * derived from a value the handler had to construct, and `runJob` turns
+ * `ok: false` into a failed pg-boss job. What the type CANNOT do is make a
+ * handler honest — `jobDone()` after swallowing an exception compiles and
+ * passes every check in this repository. It converts omission (forgetting to
+ * signal a failure) into commission (claiming a success that did not
+ * happen), which is a smaller and much more reviewable class of defect.
+ *
+ * `did` is the small pinned-shape bag of facts about the run — counts,
+ * identifiers, booleans. It rides into the failure report and, once the
+ * failure ledger lands, into the operator surface. Keep it to scalars that
+ * a dashboard can key on; it is not a place for payload data or free text.
+ */
+
+/** A single fact about a run. Scalars only — never payload data. */
+export type JobFact = string | number | boolean;
+
+/** The pinned-shape bag of facts a handler reports about its run. */
+export type JobFacts = Readonly<Record<string, JobFact>>;
+
+/**
+ * What a handler did. `ok: true` means the work named by the queue actually
+ * happened; `ok: false` means it did not, and `runJob` will fail the pg-boss
+ * job so the failure reaches the retry policy and the operator surfaces.
+ *
+ * A run that legitimately had nothing to do is `ok: true` with a zero count
+ * in `did` — absence of work is not failure. A run that could not do its
+ * work (provider down, query timed out, key missing) is `ok: false` even
+ * when the handler previously chose to carry on.
+ */
+export type JobOutcome =
+  | {
+      readonly ok: true;
+      readonly did: JobFacts;
+    }
+  | {
+      readonly ok: false;
+      /** Short, stable, greppable. `"nightscout fetch failed"`, not a stack. */
+      readonly reason: string;
+      /** The thrown value, when there was one. Preserved for the stack. */
+      readonly cause?: unknown;
+      /** Whatever the run managed before it failed. */
+      readonly did?: JobFacts;
+    };
+
+/** The run did its work. Pass the counts that make the run auditable. */
+export function jobDone(did: JobFacts = {}): JobOutcome {
+  return { ok: true, did };
+}
+
+/**
+ * The run did not do its work. `runJob` reports this and rethrows, so the
+ * pg-boss job fails and the queue's retry policy applies — check that policy
+ * when converting a handler that used to swallow, because a deterministic
+ * failure now retries instead of passing quietly.
+ */
+export function jobFailed(
+  reason: string,
+  cause?: unknown,
+  did?: JobFacts,
+): JobOutcome {
+  return { ok: false, reason, cause, did };
+}

src/lib/jobs/reminder/registrar-shared.ts

@@ -16,13 +16,18 @@
  * assert a single aggregate `allQueues` (defence in depth: the per-registrar
  * arrays AND the boot-level union both have to agree).
  */
-import { PgBoss } from "pg-boss";
+import { PgBoss, type WorkOptions } from "pg-boss";
 
 import { prisma } from "@/lib/db";
 
 import { workerLog } from "./shared";
 import { withBackgroundEvent } from "@/lib/logging/background";
 import { annotate } from "@/lib/logging/context";
+import {
+  runJob,
+  type JobHandler,
+  type JobWithMetadataHandler,
+} from "@/lib/jobs/run-job";
 
 /**
  * Cron schedule tuple: `[queueName, cronExpression, sendOptions?]`. The
@@ -198,3 +203,41 @@ export async function createAndSchedule(
     );
   }
 }
+
+/**
+ * Bind a handler to a queue. The one legal way to reach `boss.work` in this
+ * tree — the `healthlog/job-handler-outcome` lint rule refuses a bare
+ * `boss.work(` anywhere else.
+ *
+ * The point of routing every binding through here is the handler type:
+ * `JobHandler<T>` resolves to `Promise<JobOutcome>`, so a handler cannot be
+ * bound unless it returns a value declaring what it did. `runJob` then fails
+ * the pg-boss job on `ok: false` instead of completing it. Falling off the
+ * end of a handler, which used to read as success, is now a type error.
+ *
+ * Argument order matches `boss.work(queue, options, handler)` so a binding
+ * reads the same as it did before, with `boss` moved to the front.
+ */
+export async function createAndWork<T>(
+  boss: PgBoss,
+  queue: string,
+  options: WorkOptions & { includeMetadata: true },
+  handler: JobWithMetadataHandler<T>,
+): Promise<void>;
+export async function createAndWork<T>(
+  boss: PgBoss,
+  queue: string,
+  options: WorkOptions,
+  handler: JobHandler<T>,
+): Promise<void>;
+export async function createAndWork<T>(
+  boss: PgBoss,
+  queue: string,
+  options: WorkOptions,
+  handler: JobHandler<T> | JobWithMetadataHandler<T>,
+): Promise<void> {
+  // `JobWithMetadata<T>` extends `Job<T>`, so the metadata handler accepts
+  // everything the plain one does; the overloads above are what keeps the
+  // call sites honest about which shape they asked pg-boss for.
+  await boss.work<T>(queue, options, runJob(queue, handler as JobHandler<T>));
+}

src/lib/jobs/run-job.ts

@@ -0,0 +1,88 @@
+/**
+ * The wrapper that turns a returned `JobOutcome` into a pg-boss result.
+ *
+ * `boss.work(queue, opts, handler)` completes a job when the handler's
+ * promise resolves, whatever it resolves to. Wrapping every handler in
+ * `runJob` moves that decision from "the function returned" to "the function
+ * said it succeeded": an `ok: false` outcome is reported and rethrown, so
+ * pg-boss records a failed job, the queue's retry policy applies, and the
+ * failed-job readers in `job-failures.ts` surface it to the operator.
+ *
+ * This changes runtime behaviour, not only visibility. Handlers that used to
+ * catch, warn and return now fail their job. For a queue whose failure mode
+ * is deterministic — a DELETE that times out will time out again — retrying
+ * is wasted load, so the retry policy of every converted queue has to be a
+ * decision rather than a default.
+ *
+ * The wrapper deliberately does not open a wide event of its own. Handlers
+ * own their `withBackgroundEvent` scope and annotate inside it; a second
+ * event here would double every worker line for no added fact.
+ */
+import type { Job, JobWithMetadata } from "pg-boss";
+
+import type { JobFact, JobFacts, JobOutcome } from "./job-outcome";
+import { reportWorkerError } from "./report-worker-error";
+
+/** A converted handler: it says what it did instead of just finishing. */
+export type JobHandler<T> = (jobs: Job<T>[]) => Promise<JobOutcome>;
+
+/** The same, for a queue bound with `includeMetadata: true`. */
+export type JobWithMetadataHandler<T> = (
+  jobs: JobWithMetadata<T>[],
+) => Promise<JobOutcome>;
+
+/**
+ * The error a failed outcome becomes. Carries the queue and the handler's
+ * own reason so the pg-boss `output` column, the GlitchTip event and the
+ * container log all name the same thing.
+ */
+export class JobFailure extends Error {
+  readonly queue: string;
+  readonly reason: string;
+
+  constructor(queue: string, reason: string, cause?: unknown) {
+    super(`${queue}: ${reason}`, cause === undefined ? undefined : { cause });
+    this.name = "JobFailure";
+    this.queue = queue;
+    this.reason = reason;
+  }
+}
+
+function factMeta(did: JobFacts | undefined): Record<string, JobFact> {
+  const meta: Record<string, JobFact> = {};
+  for (const [key, value] of Object.entries(did ?? {})) meta[key] = value;
+  return meta;
+}
+
+/**
+ * Wrap a handler for `boss.work`. Resolves on `ok: true`; reports and
+ * rethrows on `ok: false`.
+ *
+ * A handler that throws is left alone — the throw already fails the job, and
+ * re-wrapping it would only bury the original stack. `runJob` exists for the
+ * case the throw never happens.
+ */
+export function runJob<J>(
+  queue: string,
+  handler: (jobs: J[]) => Promise<JobOutcome>,
+): (jobs: J[]) => Promise<void> {
+  return async (jobs: J[]): Promise<void> => {
+    const outcome = await handler(jobs);
+    if (outcome.ok) return;
+
+    const failure = new JobFailure(queue, outcome.reason, outcome.cause);
+    const causeMessage =
+      outcome.cause instanceof Error
+        ? outcome.cause.message
+        : typeof outcome.cause === "string"
+          ? outcome.cause
+          : undefined;
+
+    await reportWorkerError(queue, failure, {
+      ...factMeta(outcome.did),
+      ...(causeMessage === undefined ? {} : { cause: causeMessage }),
+    });
+
+    throw failure;
+  };
+}