fix(notifications): auto-disable channels on persistent hard rejects (410, etc.)

v1.4.15 Phase B3, criteria 1 + 2 (atomic — they share migration + retry-
policy types). Until this release the dispatcher fired channels best-
effort and kept retrying every 60s on a 410 Gone — that meant a stale
Web-Push subscription never recovered, a Telegram bot whose chat was
deleted got the same treatment, and the user had no visibility into
WHY their phone had stopped buzzing.

Reliability rules now:
- HARD reject (HTTP 410 web-push / ntfy, Telegram "chat not found" /
  "blocked by the user") → channel auto-disabled, `disabled_reason`
  captured, audit log `notification.channel.auto_disabled` written.
  Sender NEVER retried until the user clicks "Re-enable" in Settings.
- SOFT reject (5xx, 429, network timeout) → exponential backoff:
  [30s, 5min, 30min, 2h]. Counter `consecutive_failures` increments
  per failure; after the 5th in-a-row the channel is auto-disabled
  with reason `give_up_after_5_failures` (criterion 2's "treat as
  hard reject after N failures" rule).
- COOLDOWN window (`next_retry_at > now`) → dispatcher skips the
  channel without calling the sender — defeats the API-quota-burn
  loop a flapping upstream would otherwise create.

Implementation:
- Migration 0028 adds 6 columns to `notification_channels`:
  `disabled_reason`, `consecutive_failures`, `last_success_at`,
  `last_failure_at`, `last_failure_reason`, `next_retry_at`. All
  nullable / zero-default → no-op for existing rows.
- `src/lib/notifications/retry-policy.ts` exports the classifier
  (`classifyTelegramError`, `classifyHttpStatus`) plus
  `BACKOFF_SCHEDULE_MS` (frozen) and `nextRetryAt(count, now)` /
  `shouldAutoDisableAfterTransient(count)` helpers.
- `src/lib/notifications/channel-state.ts` is the only writer of
  the new columns: `recordChannelSuccess`, `recordChannelHardReject`,
  `recordChannelTransientFailure`, `reEnableChannel`,
  `isChannelInCooldown`. Each transient/hard path writes an audit
  entry so the user can trace WHY the channel went silent.
- Senders now return a structured `SendOutcome` (`{ ok, hardReject?,
  statusCode?, reason?, message? }`) instead of bare booleans. The
  Telegram sender additionally exposes `errorDescription` from the
  Bot API so "chat not found" / "bot was blocked" can be classified
  without parsing free-form text in the dispatcher.
- Dispatcher: cooldown-skip → success → hard-reject branch →
  transient/give-up branch. Sender-throws are absorbed as transient
  failures (defence-in-depth).

Tests: +20 unit tests (`retry-policy.test.ts`, `channel-state.test.ts`,
`dispatcher.test.ts`). 883/883 pass. Hard-reject scenarios covered:
web-push 410, Telegram chat-not-found, ntfy 410. Soft-reject
scenarios: 429 schedules backoff without disabling; cooldown skip
prevents repeat call; 5th in-a-row triggers give-up + audit.

Pre-existing bug fix: `src/app/api/settings/telegram/test/route.ts`
had `if (!ok)` against a `SendMessageResult` object — always truthy,
so the route never reported test-send failures. Now reads
`result.ok`.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/app/api/admin/backups/run/route.ts

@@ -8,8 +8,10 @@
  * If the worker isn't running (no global boss instance) the route returns
  * 503 — the caller can then route the user to the system-status page.
  */
+import type { NextRequest } from "next/server";
 import { apiHandler, requireAdmin, HttpError } from "@/lib/api-handler";
-import { apiSuccess } from "@/lib/api-response";
+import { apiSuccess, getClientIp } from "@/lib/api-response";
+import { auditLog } from "@/lib/auth/audit";
 import { annotate } from "@/lib/logging/context";
 import { getGlobalBoss } from "@/lib/jobs/boss-instance";
 import { checkRateLimit } from "@/lib/rate-limit";
@@ -18,7 +20,7 @@ export const dynamic = "force-dynamic";
 
 const DATA_BACKUP_QUEUE = "data-backup";
 
-export const POST = apiHandler(async () => {
+export const POST = apiHandler(async (request: NextRequest) => {
   const { user: admin } = await requireAdmin();
   annotate({ action: { name: "admin.backups.run" } });
 
@@ -32,11 +34,23 @@ export const POST = apiHandler(async () => {
     60 * 1000,
   );
   if (!rl.allowed) {
+    // Audit the rate-limited attempt so a compromised session burning
+    // through the cap is visible in the trail.
+    await auditLog("admin.backups.run.denied", {
+      userId: admin.id,
+      ipAddress: getClientIp(request),
+      details: { reason: "rate_limited" },
+    });
     throw new HttpError(429, "Too many backup runs");
   }
 
   const boss = getGlobalBoss();
   if (!boss) {
+    await auditLog("admin.backups.run.denied", {
+      userId: admin.id,
+      ipAddress: getClientIp(request),
+      details: { reason: "worker_not_running" },
+    });
     // 503 instead of 500 — the request is well-formed; the worker just
     // isn't reachable from this process. Same response shape as other
     // worker-required endpoints.
@@ -49,6 +63,18 @@ export const POST = apiHandler(async () => {
 
   annotate({ meta: { job_id: jobId ?? null } });
 
+  // Mirrors the audit shape of upload / download / restore: actor =
+  // admin.id, action = admin.backups.run, with the boss job id so a
+  // forensic trail can correlate "admin enqueued" → "worker ran".
+  await auditLog("admin.backups.run", {
+    userId: admin.id,
+    ipAddress: getClientIp(request),
+    details: {
+      queue: DATA_BACKUP_QUEUE,
+      jobId: jobId ?? null,
+    },
+  });
+
   return apiSuccess({
     jobId,
     queue: DATA_BACKUP_QUEUE,

tests/integration/admin-backups-audit.test.ts

@@ -0,0 +1,199 @@
+/**
+ * Integration test consolidating the audit-log contract for every
+ * write-side backup endpoint:
+ *
+ *   - POST /api/admin/backups/run               → admin.backups.run
+ *   - GET  /api/admin/backups/[id]/download     → admin.backups.download
+ *   - POST /api/admin/backups/upload            → admin.backups.upload
+ *   - POST /api/admin/backups/[id]/restore      → admin.backups.restore
+ *
+ * Each handler must emit an `AuditLog` row carrying:
+ *   - `userId` (the actor — the admin running the op)
+ *   - `action` (one of the strings above)
+ *   - `details` referencing the target backup id when applicable
+ *
+ * The denied / failed counterparts are covered by the per-endpoint
+ * suites; this file is the single contract test confirming the four
+ * happy paths together so regressions in any one of them surface
+ * loudly.
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+process.env.ENCRYPTION_KEY ??=
+  "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+
+import { encrypt } from "@/lib/crypto";
+
+import { cookieJar, headerJar } from "./mock-next-headers";
+import { getPrismaClient, truncateAllTables } from "./setup";
+
+vi.mock("next/headers", async () => {
+  const { cookieJar, headerJar } = await import("./mock-next-headers");
+  return {
+    headers: vi.fn(async () => ({
+      get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
+    })),
+    cookies: vi.fn(async () => ({
+      get: (name: string) => {
+        const value = cookieJar.get(name);
+        return value ? { name, value } : undefined;
+      },
+      set: (name: string, value: string) => {
+        cookieJar.set(name, value);
+      },
+      delete: (name: string) => {
+        cookieJar.delete(name);
+      },
+    })),
+  };
+});
+
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+
+// Stub `getGlobalBoss` so the run endpoint can audit-log its happy
+// path without booting an actual pg-boss worker against the test DB.
+// The fake boss returns a deterministic job id so the audit row can be
+// asserted to contain it.
+vi.mock("@/lib/jobs/boss-instance", () => ({
+  getGlobalBoss: vi.fn(() => ({
+    send: vi.fn().mockResolvedValue("test-job-123"),
+  })),
+}));
+
+beforeEach(async () => {
+  await truncateAllTables(getPrismaClient());
+  cookieJar.clear();
+  headerJar.clear();
+});
+
+async function seedAdminSession() {
+  const prisma = getPrismaClient();
+  const admin = await prisma.user.create({
+    data: {
+      username: "audit-admin",
+      email: "audit-admin@example.test",
+      role: "ADMIN",
+    },
+  });
+  const session = await prisma.session.create({
+    data: {
+      userId: admin.id,
+      expiresAt: new Date(Date.now() + 60_000),
+    },
+  });
+  cookieJar.set("healthlog_session", session.id);
+  return admin;
+}
+
+function payloadFor(userId: string) {
+  return {
+    schemaVersion: "1",
+    exportedAt: "2026-05-09T10:00:00.000Z",
+    userId,
+    measurements: [],
+    medications: [],
+    intakeEvents: [],
+    moodEntries: [],
+  };
+}
+
+describe("backup endpoints — AuditLog contract", () => {
+  it("admin.backups.run writes an audit row carrying the boss job id", async () => {
+    const prisma = getPrismaClient();
+    const admin = await seedAdminSession();
+
+    const { POST } = await import("@/app/api/admin/backups/run/route");
+    const req = new Request("http://localhost/api/admin/backups/run", {
+      method: "POST",
+    });
+    const res = await POST(req as unknown as Parameters<typeof POST>[0]);
+    expect(res.status).toBe(200);
+
+    const events = await prisma.auditLog.findMany({
+      where: { action: "admin.backups.run", userId: admin.id },
+    });
+    expect(events).toHaveLength(1);
+    expect(events[0]?.details ?? "").toContain("test-job-123");
+  });
+
+  it("admin.backups.upload + admin.backups.download + admin.backups.restore all emit audit rows", async () => {
+    const prisma = getPrismaClient();
+    const admin = await seedAdminSession();
+
+    // ── upload ──
+    const { POST: UPLOAD } =
+      await import("@/app/api/admin/backups/upload/route");
+    const fd = new FormData();
+    fd.append(
+      "file",
+      new File([JSON.stringify(payloadFor(admin.id))], "b.json", {
+        type: "application/json",
+      }),
+    );
+    const uploadRes = await UPLOAD(
+      new Request("http://localhost/api/admin/backups/upload", {
+        method: "POST",
+        body: fd,
+      }) as unknown as Parameters<typeof UPLOAD>[0],
+    );
+    expect(uploadRes.status).toBe(201);
+    const uploadBody = (await uploadRes.json()) as { data: { id: string } };
+
+    // Sanity: AND a directly-seeded backup so download has something
+    // independent of the upload route's success.
+    const seeded = await prisma.dataBackup.create({
+      data: {
+        userId: admin.id,
+        type: "MANUAL_AUDIT_TEST",
+        data: encrypt(JSON.stringify(payloadFor(admin.id))),
+      },
+    });
+
+    // ── download ──
+    const { GET: DOWNLOAD } =
+      await import("@/app/api/admin/backups/[id]/download/route");
+    const downloadRes = await DOWNLOAD(
+      new Request(
+        `http://localhost/api/admin/backups/${seeded.id}/download`,
+      ) as unknown as Parameters<typeof DOWNLOAD>[0],
+      { params: Promise.resolve({ id: seeded.id }) },
+    );
+    expect(downloadRes.status).toBe(200);
+
+    // ── restore ──
+    const { POST: RESTORE } =
+      await import("@/app/api/admin/backups/[id]/restore/route");
+    const restoreRes = await RESTORE(
+      new Request(`http://localhost/api/admin/backups/${seeded.id}/restore`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ confirm: "RESTORE" }),
+      }) as unknown as Parameters<typeof RESTORE>[0],
+      { params: Promise.resolve({ id: seeded.id }) },
+    );
+    expect(restoreRes.status).toBe(200);
+
+    // ── assert: each action has exactly one success row tied to the
+    // admin and the right backup id ──
+    const rows = await prisma.auditLog.findMany({
+      where: { userId: admin.id },
+      orderBy: { createdAt: "asc" },
+    });
+    const actions = rows.map((r) => r.action);
+    expect(actions).toContain("admin.backups.upload");
+    expect(actions).toContain("admin.backups.download");
+    expect(actions).toContain("admin.backups.restore");
+    expect(actions).toContain("admin.backups.restore.start");
+
+    const upload = rows.find((r) => r.action === "admin.backups.upload");
+    expect(upload?.details ?? "").toContain(uploadBody.data.id);
+
+    const download = rows.find((r) => r.action === "admin.backups.download");
+    expect(download?.details ?? "").toContain(seeded.id);
+
+    const restore = rows.find((r) => r.action === "admin.backups.restore");
+    expect(restore?.details ?? "").toContain(seeded.id);
+  });
+});