mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 07:13:29 +00:00
fix(notifications): auto-disable channels on persistent hard rejects (410, etc.)
v1.4.15 Phase B3, criteria 1 + 2 (atomic — they share migration + retry-
policy types). Until this release the dispatcher fired channels best-
effort and kept retrying every 60s on a 410 Gone — that meant a stale
Web-Push subscription never recovered, a Telegram bot whose chat was
deleted got the same treatment, and the user had no visibility into
WHY their phone had stopped buzzing.
Reliability rules now:
- HARD reject (HTTP 410 web-push / ntfy, Telegram "chat not found" /
"blocked by the user") → channel auto-disabled, `disabled_reason`
captured, audit log `notification.channel.auto_disabled` written.
Sender NEVER retried until the user clicks "Re-enable" in Settings.
- SOFT reject (5xx, 429, network timeout) → exponential backoff:
[30s, 5min, 30min, 2h]. Counter `consecutive_failures` increments
per failure; after the 5th in-a-row the channel is auto-disabled
with reason `give_up_after_5_failures` (criterion 2's "treat as
hard reject after N failures" rule).
- COOLDOWN window (`next_retry_at > now`) → dispatcher skips the
channel without calling the sender — defeats the API-quota-burn
loop a flapping upstream would otherwise create.
Implementation:
- Migration 0028 adds 6 columns to `notification_channels`:
`disabled_reason`, `consecutive_failures`, `last_success_at`,
`last_failure_at`, `last_failure_reason`, `next_retry_at`. All
nullable / zero-default → no-op for existing rows.
- `src/lib/notifications/retry-policy.ts` exports the classifier
(`classifyTelegramError`, `classifyHttpStatus`) plus
`BACKOFF_SCHEDULE_MS` (frozen) and `nextRetryAt(count, now)` /
`shouldAutoDisableAfterTransient(count)` helpers.
- `src/lib/notifications/channel-state.ts` is the only writer of
the new columns: `recordChannelSuccess`, `recordChannelHardReject`,
`recordChannelTransientFailure`, `reEnableChannel`,
`isChannelInCooldown`. Each transient/hard path writes an audit
entry so the user can trace WHY the channel went silent.
- Senders now return a structured `SendOutcome` (`{ ok, hardReject?,
statusCode?, reason?, message? }`) instead of bare booleans. The
Telegram sender additionally exposes `errorDescription` from the
Bot API so "chat not found" / "bot was blocked" can be classified
without parsing free-form text in the dispatcher.
- Dispatcher: cooldown-skip → success → hard-reject branch →
transient/give-up branch. Sender-throws are absorbed as transient
failures (defence-in-depth).
Tests: +20 unit tests (`retry-policy.test.ts`, `channel-state.test.ts`,
`dispatcher.test.ts`). 883/883 pass. Hard-reject scenarios covered:
web-push 410, Telegram chat-not-found, ntfy 410. Soft-reject
scenarios: 429 schedules backoff without disabling; cooldown skip
prevents repeat call; 5th in-a-row triggers give-up + audit.
Pre-existing bug fix: `src/app/api/settings/telegram/test/route.ts`
had `if (!ok)` against a `SendMessageResult` object — always truthy,
so the route never reported test-send failures. Now reads
`result.ok`.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
fe85c2c85f
commit
080545248e
2 files changed
+227
-2
No files matched your search
@@ -8,8 +8,10 @@
|
||||
* If the worker isn't running (no global boss instance) the route returns
|
||||
* 503 — the caller can then route the user to the system-status page.
|
||||
*/
|
||||
import type { NextRequest } from "next/server";
|
||||
import { apiHandler, requireAdmin, HttpError } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { apiSuccess, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
@@ -18,7 +20,7 @@ export const dynamic = "force-dynamic";
|
||||
|
||||
const DATA_BACKUP_QUEUE = "data-backup";
|
||||
|
||||
export const POST = apiHandler(async () => {
|
||||
export const POST = apiHandler(async (request: NextRequest) => {
|
||||
const { user: admin } = await requireAdmin();
|
||||
annotate({ action: { name: "admin.backups.run" } });
|
||||
|
||||
@@ -32,11 +34,23 @@ export const POST = apiHandler(async () => {
|
||||
60 * 1000,
|
||||
);
|
||||
if (!rl.allowed) {
|
||||
// Audit the rate-limited attempt so a compromised session burning
|
||||
// through the cap is visible in the trail.
|
||||
await auditLog("admin.backups.run.denied", {
|
||||
userId: admin.id,
|
||||
ipAddress: getClientIp(request),
|
||||
details: { reason: "rate_limited" },
|
||||
});
|
||||
throw new HttpError(429, "Too many backup runs");
|
||||
}
|
||||
|
||||
const boss = getGlobalBoss();
|
||||
if (!boss) {
|
||||
await auditLog("admin.backups.run.denied", {
|
||||
userId: admin.id,
|
||||
ipAddress: getClientIp(request),
|
||||
details: { reason: "worker_not_running" },
|
||||
});
|
||||
// 503 instead of 500 — the request is well-formed; the worker just
|
||||
// isn't reachable from this process. Same response shape as other
|
||||
// worker-required endpoints.
|
||||
@@ -49,6 +63,18 @@ export const POST = apiHandler(async () => {
|
||||
|
||||
annotate({ meta: { job_id: jobId ?? null } });
|
||||
|
||||
// Mirrors the audit shape of upload / download / restore: actor =
|
||||
// admin.id, action = admin.backups.run, with the boss job id so a
|
||||
// forensic trail can correlate "admin enqueued" → "worker ran".
|
||||
await auditLog("admin.backups.run", {
|
||||
userId: admin.id,
|
||||
ipAddress: getClientIp(request),
|
||||
details: {
|
||||
queue: DATA_BACKUP_QUEUE,
|
||||
jobId: jobId ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
return apiSuccess({
|
||||
jobId,
|
||||
queue: DATA_BACKUP_QUEUE,
|
||||
|
||||
Reference in new issue
Block a user