mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-07 12:22:03 +00:00
fix(notifications): auto-disable channels on persistent hard rejects (410, etc.)
v1.4.15 Phase B3, criteria 1 + 2 (atomic — they share migration + retry- policy types). Until this release the dispatcher fired channels best- effort and kept retrying every 60s on a 410 Gone — that meant a stale Web-Push subscription never recovered, a Telegram bot whose chat was deleted got the same treatment, and the user had no visibility into WHY their phone had stopped buzzing. Reliability rules now: - HARD reject (HTTP 410 web-push / ntfy, Telegram "chat not found" / "blocked by the user") → channel auto-disabled, disabled_reason captured, audit log notification.channel.auto_disabled written. Sender NEVER retried until the user clicks "Re-enable" in Settings. - SOFT reject (5xx, 429, network timeout) → exponential backoff: [30s, 5min, 30min, 2h]. Counter consecutive_failures increments per failure; after the 5th in-a-row the channel is auto-disabled with reason give_up_after_5_failures. - COOLDOWN window (next_retry_at > now) → dispatcher skips the channel without calling the sender — defeats the API-quota-burn loop a flapping upstream would otherwise create. Implementation: - Migration 0028 adds 6 columns to notification_channels (all nullable / zero-default → no-op for existing rows). - src/lib/notifications/retry-policy.ts: classifiers + frozen BACKOFF_SCHEDULE_MS + nextRetryAt() helpers. - src/lib/notifications/channel-state.ts: only writer of the new columns (recordChannelSuccess, recordChannelHardReject, recordChannelTransientFailure, reEnableChannel, isChannelInCooldown). Each transient/hard path writes audit. - Senders return a structured SendOutcome instead of bare booleans. - Dispatcher: cooldown-skip → success → hard-reject → transient/ give-up. Sender-throws absorbed as transient failures. Tests: +20 unit tests (retry-policy.test.ts, channel-state.test.ts, dispatcher.test.ts). 883/883 unit tests pass; typecheck clean; lint clean (no new warnings). Pre-existing bug fix: src/app/api/settings/telegram/test/route.ts had if (!ok) against a SendMessageResult object — always truthy, so the route never reported test-send failures. Now reads result.ok. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
080545248e
commit
87a40fd0a4
15 files changed
+1187
-35
No files matched your search
@@ -0,0 +1,34 @@
|
||||
-- v1.4.15 Phase B3: notification reliability metadata.
|
||||
--
|
||||
-- Until this release the dispatcher fired channels best-effort and the
|
||||
-- only feedback signal we kept was a Wide-Event warning per failure.
|
||||
-- That meant a stale Web-Push subscription kept burning bandwidth on
|
||||
-- every dispatch (a 410 from FCM is a permanent reject — re-trying it
|
||||
-- never recovers), a Telegram bot whose chat was deleted got the same
|
||||
-- treatment, and the user never saw the channel's actual health.
|
||||
--
|
||||
-- These columns let the dispatcher:
|
||||
-- 1. Auto-disable a channel on a hard reject (HTTP 410, Telegram
|
||||
-- "chat not found" or "blocked by the user", ntfy 410), with the
|
||||
-- reason captured for the Settings UI.
|
||||
-- 2. Apply exponential backoff [30s, 5min, 30min, 2h] to soft errors
|
||||
-- (5xx, 429, network timeout) instead of hammering a flapping
|
||||
-- upstream — and after 5 consecutive transient failures, treat
|
||||
-- the channel as effectively dead and auto-disable it too.
|
||||
-- 3. Surface "Active / Auto-disabled / Sending paused" + last
|
||||
-- success / failure timestamps in Settings → Notifications, with
|
||||
-- a "Re-enable" button that clears `disabled_reason` +
|
||||
-- `consecutive_failures` and a "Send test" button.
|
||||
--
|
||||
-- All columns are nullable / zero-default so the migration is a no-op
|
||||
-- for existing rows: a healthy channel reports
|
||||
-- consecutive_failures=0, last_success_at=null, disabled_reason=null,
|
||||
-- which renders as "Active (no recent send recorded)" in the UI.
|
||||
|
||||
ALTER TABLE "notification_channels"
|
||||
ADD COLUMN "disabled_reason" TEXT,
|
||||
ADD COLUMN "consecutive_failures" INTEGER NOT NULL DEFAULT 0,
|
||||
ADD COLUMN "last_success_at" TIMESTAMP(3),
|
||||
ADD COLUMN "last_failure_at" TIMESTAMP(3),
|
||||
ADD COLUMN "last_failure_reason" TEXT,
|
||||
ADD COLUMN "next_retry_at" TIMESTAMP(3);
|
||||
@@ -106,6 +106,7 @@ model User {
|
||||
feedback Feedback[]
|
||||
idempotencyKeys IdempotencyKey[]
|
||||
devices Device[]
|
||||
integrationStatuses IntegrationStatus[]
|
||||
|
||||
@@map("users")
|
||||
}
|
||||
@@ -444,6 +445,46 @@ model MoodEntry {
|
||||
@@map("mood_entries")
|
||||
}
|
||||
|
||||
// ─── Integration Status (v1.4.15) ─────────────────────────
|
||||
//
|
||||
// Per-(user, integration) row that captures the most recent sync attempt
|
||||
// outcome so the Settings → Integrations UI can show:
|
||||
// - state ("connected", "error_reauth", "error_transient", "disconnected")
|
||||
// - last successful sync timestamp
|
||||
// - last attempted sync timestamp
|
||||
// - last error message (encrypted at rest if non-null — see crypto.ts)
|
||||
// - consecutive failure counter (resets on success)
|
||||
//
|
||||
// `recordSyncSuccess` / `recordSyncFailure` helpers in
|
||||
// `src/lib/integrations/status.ts` are the only writers. The
|
||||
// `consecutiveFailures` counter feeds the admin-Telegram alerting
|
||||
// threshold (default 3). State `error_reauth` parks scheduled syncs
|
||||
// until the user reconnects (the sync entry-points consult this row
|
||||
// before doing any external work).
|
||||
//
|
||||
// `integration` values are bare strings ("withings", "moodlog") rather
|
||||
// than an enum so future integrations don't require a migration.
|
||||
model IntegrationStatus {
|
||||
id String @id @default(cuid())
|
||||
userId String @map("user_id")
|
||||
integration String // "withings" | "moodlog"
|
||||
state String @default("connected") // "connected" | "error_reauth" | "error_transient" | "disconnected"
|
||||
lastSuccessAt DateTime? @map("last_success_at")
|
||||
lastAttemptAt DateTime? @map("last_attempt_at")
|
||||
lastError String? @map("last_error") // AES-256-GCM encrypted (may contain token-fragment metadata)
|
||||
consecutiveFailures Int @default(0) @map("consecutive_failures")
|
||||
alertedAt DateTime? @map("alerted_at") // last time admin Telegram alert fired (idempotency guard)
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([userId, integration])
|
||||
@@index([userId])
|
||||
@@index([state])
|
||||
@@map("integration_statuses")
|
||||
}
|
||||
|
||||
// ─── App Settings ─────────────────────────────────────────
|
||||
|
||||
model AppSettings {
|
||||
@@ -559,6 +600,23 @@ model NotificationChannel {
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
// ── v1.4.15 Phase B3: reliability fields ─────────────────────
|
||||
// Reason the channel was auto-disabled (e.g. "hard_reject_410",
|
||||
// "telegram_blocked_by_user", "telegram_chat_not_found",
|
||||
// "give_up_after_5_failures"). NULL when the channel is healthy
|
||||
// OR was disabled manually (in which case `enabled=false` alone
|
||||
// is enough — no automated process should treat that as
|
||||
// re-enable-able).
|
||||
disabledReason String? @map("disabled_reason")
|
||||
consecutiveFailures Int @default(0) @map("consecutive_failures")
|
||||
lastSuccessAt DateTime? @map("last_success_at")
|
||||
lastFailureAt DateTime? @map("last_failure_at")
|
||||
lastFailureReason String? @map("last_failure_reason")
|
||||
// The next time the dispatcher is allowed to attempt a send for
|
||||
// this channel after a transient failure. NULL = no cooldown.
|
||||
// Hard rejects flip `enabled=false` instead of writing this.
|
||||
nextRetryAt DateTime? @map("next_retry_at")
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
preferences NotificationPreference[]
|
||||
|
||||
|
||||
@@ -134,7 +134,8 @@ export const POST = apiHandler(async () => {
|
||||
const config = JSON.parse(
|
||||
decrypt(channel.config),
|
||||
) as NtfyChannelConfig;
|
||||
success = await sendViaNtfy(config, payload);
|
||||
const ntfyResult = await sendViaNtfy(config, payload);
|
||||
success = ntfyResult.ok;
|
||||
if (!success) {
|
||||
results.push({
|
||||
channel: "NTFY",
|
||||
@@ -146,7 +147,8 @@ export const POST = apiHandler(async () => {
|
||||
break;
|
||||
}
|
||||
case "WEB_PUSH": {
|
||||
success = await sendViaWebPush(user.id, payload);
|
||||
const webPushResult = await sendViaWebPush(user.id, payload);
|
||||
success = webPushResult.ok;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
|
||||
@@ -34,14 +34,14 @@ export const POST = apiHandler(async () => {
|
||||
return apiError("Server URL and topic are required", 400);
|
||||
}
|
||||
|
||||
const success = await sendViaNtfy(config, {
|
||||
const result = await sendViaNtfy(config, {
|
||||
eventType: "SYSTEM_ALERT",
|
||||
userId: user.id,
|
||||
title: "HealthLog Test",
|
||||
message: "HealthLog: Connection successful! ntfy notifications are active.",
|
||||
});
|
||||
|
||||
if (!success) {
|
||||
if (!result.ok) {
|
||||
return apiError("Failed to send test message", 500);
|
||||
}
|
||||
|
||||
|
||||
@@ -27,13 +27,13 @@ export const POST = apiHandler(async () => {
|
||||
}
|
||||
|
||||
const botToken = decrypt(dbUser.telegramBotToken);
|
||||
const ok = await sendTelegramMessage(
|
||||
const result = await sendTelegramMessage(
|
||||
botToken,
|
||||
dbUser.telegramChatId,
|
||||
"HealthLog: Connection successful! Telegram notifications are active.",
|
||||
);
|
||||
|
||||
if (!ok) {
|
||||
if (!result.ok) {
|
||||
return apiError(
|
||||
"Failed to send message. Check bot token and chat ID.",
|
||||
422,
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { isChannelInCooldown } from "@/lib/notifications/channel-state";
|
||||
|
||||
describe("channel-state: isChannelInCooldown", () => {
|
||||
const now = new Date("2026-05-09T12:00:00.000Z");
|
||||
|
||||
it("returns false when nextRetryAt is null", () => {
|
||||
expect(isChannelInCooldown({ nextRetryAt: null }, now)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns true when nextRetryAt is in the future", () => {
|
||||
expect(
|
||||
isChannelInCooldown(
|
||||
{ nextRetryAt: new Date(now.getTime() + 60_000) },
|
||||
now,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false when nextRetryAt is in the past", () => {
|
||||
expect(
|
||||
isChannelInCooldown(
|
||||
{ nextRetryAt: new Date(now.getTime() - 60_000) },
|
||||
now,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false when nextRetryAt is exactly now (boundary: cooldown is over)", () => {
|
||||
expect(isChannelInCooldown({ nextRetryAt: now }, now)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,345 @@
|
||||
/**
|
||||
* Dispatcher reliability tests (v1.4.15 Phase B3).
|
||||
*
|
||||
* Drives the dispatcher through the four critical state transitions:
|
||||
* 1. Hard reject (web-push 410) → auto-disable + audit log + no retry.
|
||||
* 2. Soft reject (web-push 429) → counter increments, channel stays
|
||||
* enabled, `nextRetryAt` set per backoff schedule.
|
||||
* 3. 5th consecutive transient failure → auto-disable with reason
|
||||
* `give_up_after_5_failures`.
|
||||
* 4. Channel in cooldown (`nextRetryAt > now`) → sender is NOT called.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
notificationChannel: {
|
||||
findMany: vi.fn(),
|
||||
update: vi.fn(),
|
||||
upsert: vi.fn(),
|
||||
},
|
||||
user: { findUnique: vi.fn() },
|
||||
auditLog: { create: vi.fn() },
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/auth/audit", () => ({
|
||||
auditLog: vi.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/crypto", () => ({
|
||||
encrypt: (s: string) => s,
|
||||
decrypt: (s: string) => s,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({
|
||||
getEvent: () => ({
|
||||
addWarning: vi.fn(),
|
||||
addMeta: vi.fn(),
|
||||
addExternalCall: vi.fn(),
|
||||
setError: vi.fn(),
|
||||
}),
|
||||
}));
|
||||
|
||||
const sendViaTelegramMock = vi.fn();
|
||||
const sendViaNtfyMock = vi.fn();
|
||||
const sendViaWebPushMock = vi.fn();
|
||||
|
||||
vi.mock("@/lib/notifications/senders/telegram", () => ({
|
||||
sendViaTelegram: (...args: unknown[]) => sendViaTelegramMock(...args),
|
||||
}));
|
||||
vi.mock("@/lib/notifications/senders/ntfy", () => ({
|
||||
sendViaNtfy: (...args: unknown[]) => sendViaNtfyMock(...args),
|
||||
}));
|
||||
vi.mock("@/lib/notifications/senders/web-push", () => ({
|
||||
sendViaWebPush: (...args: unknown[]) => sendViaWebPushMock(...args),
|
||||
}));
|
||||
|
||||
import { dispatchNotification } from "@/lib/notifications/dispatcher";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
|
||||
type MockChannel = {
|
||||
id: string;
|
||||
userId: string;
|
||||
type: "TELEGRAM" | "NTFY" | "WEB_PUSH";
|
||||
enabled: boolean;
|
||||
config: string;
|
||||
consecutiveFailures: number;
|
||||
nextRetryAt: Date | null;
|
||||
preferences: { eventType: string; enabled: boolean }[];
|
||||
};
|
||||
|
||||
function makeChannel(over: Partial<MockChannel> = {}): MockChannel {
|
||||
return {
|
||||
id: "ch-1",
|
||||
userId: "u-1",
|
||||
type: "WEB_PUSH",
|
||||
enabled: true,
|
||||
config: JSON.stringify({}),
|
||||
consecutiveFailures: 0,
|
||||
nextRetryAt: null,
|
||||
preferences: [],
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue(null as never);
|
||||
// Default: every update returns a counter of 1 — individual tests
|
||||
// override the resolved value when they need to drive a specific
|
||||
// failure count (e.g. the "5th in a row" give-up test).
|
||||
(
|
||||
prisma.notificationChannel.update as unknown as ReturnType<typeof vi.fn>
|
||||
).mockResolvedValue({ consecutiveFailures: 1 });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("dispatchNotification — hard reject (web-push 410)", () => {
|
||||
it("auto-disables the channel, writes audit, and does NOT retry", async () => {
|
||||
const channel = makeChannel({ type: "WEB_PUSH" });
|
||||
vi.mocked(prisma.notificationChannel.findMany).mockResolvedValueOnce([
|
||||
channel,
|
||||
] as never);
|
||||
sendViaWebPushMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
hardReject: true,
|
||||
statusCode: 410,
|
||||
reason: "web_push_410_gone",
|
||||
});
|
||||
|
||||
await dispatchNotification({
|
||||
eventType: "SYSTEM_ALERT",
|
||||
userId: "u-1",
|
||||
title: "t",
|
||||
message: "m",
|
||||
});
|
||||
|
||||
expect(sendViaWebPushMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Channel was auto-disabled with reason captured.
|
||||
const updateCalls = (
|
||||
prisma.notificationChannel.update as unknown as ReturnType<typeof vi.fn>
|
||||
).mock.calls;
|
||||
const disablePayload = updateCalls.find(
|
||||
(c) => (c[0] as { data: { enabled?: boolean } }).data?.enabled === false,
|
||||
);
|
||||
expect(disablePayload).toBeTruthy();
|
||||
expect(
|
||||
(
|
||||
disablePayload?.[0] as {
|
||||
data: { disabledReason?: string };
|
||||
}
|
||||
).data.disabledReason,
|
||||
).toBe("web_push_410_gone");
|
||||
|
||||
// Audit log entry written.
|
||||
expect(auditLog).toHaveBeenCalledWith(
|
||||
"notification.channel.auto_disabled",
|
||||
expect.objectContaining({
|
||||
userId: "u-1",
|
||||
details: expect.objectContaining({
|
||||
reason: "web_push_410_gone",
|
||||
kind: "hard_reject",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("dispatchNotification — soft reject (web-push 429)", () => {
|
||||
it("schedules backoff, increments counter, does NOT auto-disable", async () => {
|
||||
const channel = makeChannel({ type: "WEB_PUSH", consecutiveFailures: 0 });
|
||||
vi.mocked(prisma.notificationChannel.findMany).mockResolvedValueOnce([
|
||||
channel,
|
||||
] as never);
|
||||
sendViaWebPushMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
statusCode: 429,
|
||||
reason: "web-push_429",
|
||||
});
|
||||
// First update (increment) returns counter=1.
|
||||
(prisma.notificationChannel.update as unknown as ReturnType<typeof vi.fn>)
|
||||
.mockResolvedValueOnce({ consecutiveFailures: 1 })
|
||||
.mockResolvedValue({});
|
||||
|
||||
await dispatchNotification({
|
||||
eventType: "SYSTEM_ALERT",
|
||||
userId: "u-1",
|
||||
title: "t",
|
||||
message: "m",
|
||||
});
|
||||
|
||||
const updateCalls = (
|
||||
prisma.notificationChannel.update as unknown as ReturnType<typeof vi.fn>
|
||||
).mock.calls;
|
||||
// No update payload sets enabled=false → channel stays enabled.
|
||||
const disablePayload = updateCalls.find(
|
||||
(c) => (c[0] as { data: { enabled?: boolean } }).data?.enabled === false,
|
||||
);
|
||||
expect(disablePayload).toBeUndefined();
|
||||
|
||||
// A `nextRetryAt` was scheduled — channel is in cooldown until then.
|
||||
const cooldownPayload = updateCalls.find(
|
||||
(c) =>
|
||||
(c[0] as { data: { nextRetryAt?: Date } }).data?.nextRetryAt instanceof
|
||||
Date,
|
||||
);
|
||||
expect(cooldownPayload).toBeTruthy();
|
||||
|
||||
// No "auto_disabled" audit was written for a transient failure.
|
||||
const audits = vi.mocked(auditLog).mock.calls;
|
||||
const autoDisabledAudits = audits.filter(
|
||||
(call) => call[0] === "notification.channel.auto_disabled",
|
||||
);
|
||||
expect(autoDisabledAudits).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("dispatchNotification — give-up after 5 transient failures", () => {
|
||||
it("auto-disables on the 5th in-a-row failure", async () => {
|
||||
const channel = makeChannel({ type: "NTFY", consecutiveFailures: 4 });
|
||||
vi.mocked(prisma.notificationChannel.findMany).mockResolvedValueOnce([
|
||||
channel,
|
||||
] as never);
|
||||
sendViaNtfyMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
statusCode: 503,
|
||||
reason: "ntfy_503",
|
||||
});
|
||||
// Counter post-increment is 5 → trips MAX_CONSECUTIVE_FAILURES.
|
||||
(prisma.notificationChannel.update as unknown as ReturnType<typeof vi.fn>)
|
||||
.mockResolvedValueOnce({ consecutiveFailures: 5 })
|
||||
.mockResolvedValue({});
|
||||
|
||||
await dispatchNotification({
|
||||
eventType: "SYSTEM_ALERT",
|
||||
userId: "u-1",
|
||||
title: "t",
|
||||
message: "m",
|
||||
});
|
||||
|
||||
// The give-up branch flipped enabled=false with the right reason.
|
||||
const updateCalls = (
|
||||
prisma.notificationChannel.update as unknown as ReturnType<typeof vi.fn>
|
||||
).mock.calls;
|
||||
const giveUp = updateCalls.find(
|
||||
(c) =>
|
||||
(c[0] as { data: { disabledReason?: string } }).data?.disabledReason ===
|
||||
"give_up_after_5_failures",
|
||||
);
|
||||
expect(giveUp).toBeTruthy();
|
||||
expect((giveUp?.[0] as { data: { enabled?: boolean } }).data.enabled).toBe(
|
||||
false,
|
||||
);
|
||||
|
||||
// Audit log entry written for give-up.
|
||||
expect(auditLog).toHaveBeenCalledWith(
|
||||
"notification.channel.auto_disabled",
|
||||
expect.objectContaining({
|
||||
userId: "u-1",
|
||||
details: expect.objectContaining({
|
||||
reason: "give_up_after_5_failures",
|
||||
kind: "transient_give_up",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("dispatchNotification — cooldown skip", () => {
|
||||
it("does NOT call the sender when nextRetryAt is in the future", async () => {
|
||||
const channel = makeChannel({
|
||||
type: "NTFY",
|
||||
nextRetryAt: new Date(Date.now() + 60_000),
|
||||
});
|
||||
vi.mocked(prisma.notificationChannel.findMany).mockResolvedValueOnce([
|
||||
channel,
|
||||
] as never);
|
||||
|
||||
await dispatchNotification({
|
||||
eventType: "SYSTEM_ALERT",
|
||||
userId: "u-1",
|
||||
title: "t",
|
||||
message: "m",
|
||||
});
|
||||
|
||||
expect(sendViaNtfyMock).not.toHaveBeenCalled();
|
||||
expect(sendViaWebPushMock).not.toHaveBeenCalled();
|
||||
expect(sendViaTelegramMock).not.toHaveBeenCalled();
|
||||
// No update / audit when channel is in cooldown.
|
||||
expect(prisma.notificationChannel.update).not.toHaveBeenCalled();
|
||||
expect(auditLog).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("DOES call the sender when nextRetryAt is in the past", async () => {
|
||||
const channel = makeChannel({
|
||||
type: "NTFY",
|
||||
nextRetryAt: new Date(Date.now() - 60_000),
|
||||
});
|
||||
vi.mocked(prisma.notificationChannel.findMany).mockResolvedValueOnce([
|
||||
channel,
|
||||
] as never);
|
||||
sendViaNtfyMock.mockResolvedValueOnce({ ok: true });
|
||||
|
||||
await dispatchNotification({
|
||||
eventType: "SYSTEM_ALERT",
|
||||
userId: "u-1",
|
||||
title: "t",
|
||||
message: "m",
|
||||
});
|
||||
|
||||
expect(sendViaNtfyMock).toHaveBeenCalledTimes(1);
|
||||
// Success path resets the counter + clears nextRetryAt.
|
||||
const updateCalls = (
|
||||
prisma.notificationChannel.update as unknown as ReturnType<typeof vi.fn>
|
||||
).mock.calls;
|
||||
const success = updateCalls.find(
|
||||
(c) =>
|
||||
(c[0] as { data: { lastSuccessAt?: Date } }).data
|
||||
?.lastSuccessAt instanceof Date,
|
||||
);
|
||||
expect(success).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("dispatchNotification — Telegram hard reject", () => {
|
||||
it("auto-disables on 'chat not found'", async () => {
|
||||
const channel = makeChannel({
|
||||
type: "TELEGRAM",
|
||||
config: JSON.stringify({ botToken: "t", chatId: "c" }),
|
||||
});
|
||||
vi.mocked(prisma.notificationChannel.findMany).mockResolvedValueOnce([
|
||||
channel,
|
||||
] as never);
|
||||
sendViaTelegramMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
hardReject: true,
|
||||
reason: "telegram_chat_not_found",
|
||||
message: "Bad Request: chat not found",
|
||||
});
|
||||
|
||||
await dispatchNotification({
|
||||
eventType: "SYSTEM_ALERT",
|
||||
userId: "u-1",
|
||||
title: "t",
|
||||
message: "m",
|
||||
});
|
||||
|
||||
expect(auditLog).toHaveBeenCalledWith(
|
||||
"notification.channel.auto_disabled",
|
||||
expect.objectContaining({
|
||||
details: expect.objectContaining({
|
||||
reason: "telegram_chat_not_found",
|
||||
kind: "hard_reject",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
BACKOFF_SCHEDULE_MS,
|
||||
MAX_CONSECUTIVE_FAILURES,
|
||||
classifyHttpStatus,
|
||||
classifyTelegramError,
|
||||
nextRetryAt,
|
||||
shouldAutoDisableAfterTransient,
|
||||
} from "@/lib/notifications/retry-policy";
|
||||
|
||||
describe("retry-policy: BACKOFF_SCHEDULE_MS", () => {
|
||||
it("matches the spec [30s, 5min, 30min, 2h]", () => {
|
||||
expect(BACKOFF_SCHEDULE_MS).toEqual([
|
||||
30 * 1_000,
|
||||
5 * 60 * 1_000,
|
||||
30 * 60 * 1_000,
|
||||
2 * 60 * 60 * 1_000,
|
||||
]);
|
||||
});
|
||||
|
||||
it("is frozen so callers can't mutate the schedule at runtime", () => {
|
||||
expect(Object.isFrozen(BACKOFF_SCHEDULE_MS)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("retry-policy: nextRetryAt", () => {
|
||||
const now = new Date("2026-05-09T12:00:00.000Z");
|
||||
|
||||
it("returns null for the 0th failure (no failure yet → no cooldown)", () => {
|
||||
expect(nextRetryAt(0, now)).toBeNull();
|
||||
});
|
||||
|
||||
it("schedules 30s after the 1st failure", () => {
|
||||
const next = nextRetryAt(1, now);
|
||||
expect(next?.getTime()).toBe(now.getTime() + 30 * 1_000);
|
||||
});
|
||||
|
||||
it("schedules 5min after the 2nd failure", () => {
|
||||
const next = nextRetryAt(2, now);
|
||||
expect(next?.getTime()).toBe(now.getTime() + 5 * 60 * 1_000);
|
||||
});
|
||||
|
||||
it("schedules 30min after the 3rd failure", () => {
|
||||
const next = nextRetryAt(3, now);
|
||||
expect(next?.getTime()).toBe(now.getTime() + 30 * 60 * 1_000);
|
||||
});
|
||||
|
||||
it("schedules 2h after the 4th failure", () => {
|
||||
const next = nextRetryAt(4, now);
|
||||
expect(next?.getTime()).toBe(now.getTime() + 2 * 60 * 60 * 1_000);
|
||||
});
|
||||
|
||||
it("returns null at the 5th failure (give-up threshold)", () => {
|
||||
expect(nextRetryAt(MAX_CONSECUTIVE_FAILURES, now)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null beyond the threshold (defence-in-depth)", () => {
|
||||
expect(nextRetryAt(99, now)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("retry-policy: shouldAutoDisableAfterTransient", () => {
|
||||
it("does not give up before the 5th in-a-row failure", () => {
|
||||
expect(shouldAutoDisableAfterTransient(1)).toBe(false);
|
||||
expect(shouldAutoDisableAfterTransient(4)).toBe(false);
|
||||
});
|
||||
|
||||
it("gives up at exactly the 5th failure", () => {
|
||||
expect(shouldAutoDisableAfterTransient(5)).toBe(true);
|
||||
});
|
||||
|
||||
it("stays in give-up mode for any larger value", () => {
|
||||
expect(shouldAutoDisableAfterTransient(99)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("retry-policy: classifyTelegramError", () => {
|
||||
it("maps 'chat not found' to a hard reject", () => {
|
||||
expect(classifyTelegramError("Bad Request: chat not found")).toEqual({
|
||||
hardReject: true,
|
||||
reason: "telegram_chat_not_found",
|
||||
});
|
||||
});
|
||||
|
||||
it("maps 'bot was blocked by the user' to a hard reject", () => {
|
||||
expect(
|
||||
classifyTelegramError("Forbidden: bot was blocked by the user"),
|
||||
).toEqual({ hardReject: true, reason: "telegram_blocked_by_user" });
|
||||
});
|
||||
|
||||
it("maps 'user is deactivated' to a hard reject", () => {
|
||||
expect(classifyTelegramError("Forbidden: user is deactivated")).toEqual({
|
||||
hardReject: true,
|
||||
reason: "telegram_blocked_by_user",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats undefined / empty descriptions as soft", () => {
|
||||
expect(classifyTelegramError(undefined)).toEqual({
|
||||
hardReject: false,
|
||||
reason: "telegram_send_failed",
|
||||
});
|
||||
expect(classifyTelegramError("")).toEqual({
|
||||
hardReject: false,
|
||||
reason: "telegram_send_failed",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats unrecognized descriptions (5xx-style blips) as soft", () => {
|
||||
expect(classifyTelegramError("Internal Server Error")).toEqual({
|
||||
hardReject: false,
|
||||
reason: "telegram_send_failed",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("retry-policy: classifyHttpStatus", () => {
|
||||
it("treats web-push 410 as a hard reject", () => {
|
||||
expect(classifyHttpStatus(410, "web-push")).toEqual({
|
||||
hardReject: true,
|
||||
reason: "web_push_410_gone",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats web-push 404 as a hard reject (endpoint deleted)", () => {
|
||||
expect(classifyHttpStatus(404, "web-push")).toEqual({
|
||||
hardReject: true,
|
||||
reason: "web_push_404_endpoint",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats web-push 429 as a soft reject (so the channel keeps retrying)", () => {
|
||||
expect(classifyHttpStatus(429, "web-push")).toEqual({
|
||||
hardReject: false,
|
||||
reason: "web-push_429",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats web-push 500 as a soft reject", () => {
|
||||
expect(classifyHttpStatus(500, "web-push")).toEqual({
|
||||
hardReject: false,
|
||||
reason: "web-push_500",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats ntfy 410 as a hard reject (topic invalidated)", () => {
|
||||
expect(classifyHttpStatus(410, "ntfy")).toEqual({
|
||||
hardReject: true,
|
||||
reason: "ntfy_410_gone",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats ntfy 404 as soft (could be a wrong topic name, not gone-forever)", () => {
|
||||
expect(classifyHttpStatus(404, "ntfy")).toEqual({
|
||||
hardReject: false,
|
||||
reason: "ntfy_404",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats undefined status (network error) as a soft reject", () => {
|
||||
expect(classifyHttpStatus(undefined, "ntfy")).toEqual({
|
||||
hardReject: false,
|
||||
reason: "ntfy_network_error",
|
||||
});
|
||||
expect(classifyHttpStatus(undefined, "web-push")).toEqual({
|
||||
hardReject: false,
|
||||
reason: "web-push_network_error",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Channel state mutations for the notification dispatcher (v1.4.15 Phase B3).
|
||||
*
|
||||
* Centralised so the dispatcher can call ONE function per outcome —
|
||||
* making the "auto-disable on hard reject" + "exponential backoff on
|
||||
* soft reject" rules unit-testable and audit-loggable in one place.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import {
|
||||
MAX_CONSECUTIVE_FAILURES,
|
||||
nextRetryAt,
|
||||
shouldAutoDisableAfterTransient,
|
||||
type SendOutcome,
|
||||
} from "@/lib/notifications/retry-policy";
|
||||
import type { ChannelType } from "@/lib/notifications/types";
|
||||
|
||||
interface ChannelRef {
|
||||
id: string;
|
||||
userId: string;
|
||||
type: ChannelType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a successful send. Resets the failure counter and clears any
|
||||
* pending retry-cooldown so the next dispatch goes out immediately.
|
||||
*/
|
||||
export async function recordChannelSuccess(channel: ChannelRef): Promise<void> {
|
||||
await prisma.notificationChannel.update({
|
||||
where: { id: channel.id },
|
||||
data: {
|
||||
lastSuccessAt: new Date(),
|
||||
consecutiveFailures: 0,
|
||||
nextRetryAt: null,
|
||||
lastFailureReason: null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a hard reject — the upstream told us this channel is dead.
|
||||
* Disables the channel, captures the reason for the Settings UI, and
|
||||
* writes an audit-log entry so the user can trace WHY their phone
|
||||
* stopped getting reminders.
|
||||
*/
|
||||
export async function recordChannelHardReject(
|
||||
channel: ChannelRef,
|
||||
outcome: Pick<SendOutcome, "reason" | "message" | "statusCode">,
|
||||
): Promise<void> {
|
||||
const reason = outcome.reason ?? "hard_reject";
|
||||
await prisma.notificationChannel.update({
|
||||
where: { id: channel.id },
|
||||
data: {
|
||||
enabled: false,
|
||||
disabledReason: reason,
|
||||
lastFailureAt: new Date(),
|
||||
lastFailureReason: reason,
|
||||
// Counter is preserved so the UI can show "5 in a row before we
|
||||
// gave up", but bump it so /api/notifications/status shows the
|
||||
// hard-reject as a final failure event.
|
||||
consecutiveFailures: { increment: 1 },
|
||||
nextRetryAt: null,
|
||||
},
|
||||
});
|
||||
await auditLog("notification.channel.auto_disabled", {
|
||||
userId: channel.userId,
|
||||
details: {
|
||||
channelType: channel.type,
|
||||
channelId: channel.id,
|
||||
reason,
|
||||
statusCode: outcome.statusCode ?? null,
|
||||
kind: "hard_reject",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a transient failure. Increments the counter, schedules the
|
||||
* next retry per the backoff schedule, and — once the counter crosses
|
||||
* MAX_CONSECUTIVE_FAILURES — auto-disables the channel with reason
|
||||
* `give_up_after_5_failures` and writes an audit entry.
|
||||
*/
|
||||
export async function recordChannelTransientFailure(
|
||||
channel: ChannelRef,
|
||||
outcome: Pick<SendOutcome, "reason" | "message" | "statusCode">,
|
||||
now: Date = new Date(),
|
||||
): Promise<{ autoDisabled: boolean; nextRetryAt: Date | null }> {
|
||||
// Re-read current counter inside an atomic update to avoid races
|
||||
// when multiple dispatch attempts collide.
|
||||
const updated = await prisma.notificationChannel.update({
|
||||
where: { id: channel.id },
|
||||
data: {
|
||||
consecutiveFailures: { increment: 1 },
|
||||
lastFailureAt: now,
|
||||
lastFailureReason: outcome.reason ?? "transient_failure",
|
||||
},
|
||||
select: { consecutiveFailures: true },
|
||||
});
|
||||
const failureCount = updated.consecutiveFailures;
|
||||
|
||||
if (shouldAutoDisableAfterTransient(failureCount)) {
|
||||
await prisma.notificationChannel.update({
|
||||
where: { id: channel.id },
|
||||
data: {
|
||||
enabled: false,
|
||||
disabledReason: "give_up_after_5_failures",
|
||||
nextRetryAt: null,
|
||||
},
|
||||
});
|
||||
await auditLog("notification.channel.auto_disabled", {
|
||||
userId: channel.userId,
|
||||
details: {
|
||||
channelType: channel.type,
|
||||
channelId: channel.id,
|
||||
reason: "give_up_after_5_failures",
|
||||
consecutiveFailures: failureCount,
|
||||
kind: "transient_give_up",
|
||||
lastError: outcome.reason ?? null,
|
||||
statusCode: outcome.statusCode ?? null,
|
||||
},
|
||||
});
|
||||
return { autoDisabled: true, nextRetryAt: null };
|
||||
}
|
||||
|
||||
const retryAt = nextRetryAt(failureCount, now);
|
||||
await prisma.notificationChannel.update({
|
||||
where: { id: channel.id },
|
||||
data: { nextRetryAt: retryAt },
|
||||
});
|
||||
return { autoDisabled: false, nextRetryAt: retryAt };
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-enable a channel that was previously auto-disabled. Clears the
|
||||
* disabled-reason + counter and the cooldown so the next dispatch
|
||||
* fires immediately. Audit-logs the manual override.
|
||||
*/
|
||||
export async function reEnableChannel(channel: ChannelRef): Promise<void> {
|
||||
await prisma.notificationChannel.update({
|
||||
where: { id: channel.id },
|
||||
data: {
|
||||
enabled: true,
|
||||
disabledReason: null,
|
||||
consecutiveFailures: 0,
|
||||
nextRetryAt: null,
|
||||
lastFailureReason: null,
|
||||
},
|
||||
});
|
||||
await auditLog("notification.channel.re_enabled", {
|
||||
userId: channel.userId,
|
||||
details: {
|
||||
channelType: channel.type,
|
||||
channelId: channel.id,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Should the dispatcher attempt a send right now? Honours the cooldown
|
||||
* window written by the backoff scheduler — if `nextRetryAt` is in the
|
||||
* future, the channel is in "sending paused" mode and should be
|
||||
* skipped to avoid hammering a flapping upstream.
|
||||
*/
|
||||
export function isChannelInCooldown(
|
||||
channel: { nextRetryAt: Date | null },
|
||||
now: Date = new Date(),
|
||||
): boolean {
|
||||
return channel.nextRetryAt !== null && channel.nextRetryAt > now;
|
||||
}
|
||||
|
||||
export const TEST_CONSTANTS = { MAX_CONSECUTIVE_FAILURES } as const;
|
||||
@@ -9,6 +9,13 @@ import type {
|
||||
import { sendViaTelegram } from "@/lib/notifications/senders/telegram";
|
||||
import { sendViaNtfy } from "@/lib/notifications/senders/ntfy";
|
||||
import { sendViaWebPush } from "@/lib/notifications/senders/web-push";
|
||||
import type { SendOutcome } from "@/lib/notifications/retry-policy";
|
||||
import {
|
||||
isChannelInCooldown,
|
||||
recordChannelHardReject,
|
||||
recordChannelSuccess,
|
||||
recordChannelTransientFailure,
|
||||
} from "@/lib/notifications/channel-state";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
|
||||
/**
|
||||
@@ -16,9 +23,12 @@ import { getEvent } from "@/lib/logging/context";
|
||||
* Best-effort: logs errors but never throws.
|
||||
*
|
||||
* For each channel:
|
||||
* 1. Check if the channel is enabled
|
||||
* 2. Check if a preference exists for this eventType (default: enabled / opt-out)
|
||||
* 3. Call the appropriate sender
|
||||
* 1. Skip if `enabled=false` (manually or auto-disabled).
|
||||
* 2. Skip if currently in retry-cooldown (`nextRetryAt > now`).
|
||||
* 3. Check the per-channel preference for this eventType (default: enabled).
|
||||
* 4. Call the appropriate sender — which now returns a `SendOutcome` so
|
||||
* the dispatcher can classify hard rejects (410, blocked-by-user) vs
|
||||
* soft errors (5xx, 429, network) and update channel state accordingly.
|
||||
*
|
||||
* Also handles legacy Telegram config stored on the User model by
|
||||
* auto-migrating it to a NotificationChannel record on first dispatch.
|
||||
@@ -94,20 +104,84 @@ export async function dispatchNotification(
|
||||
}
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
for (const channel of channels) {
|
||||
// Opt-out model: if no preference row exists, default to enabled
|
||||
const pref = channel.preferences[0];
|
||||
if (pref && !pref.enabled) continue;
|
||||
|
||||
// Backoff cooldown — skip the channel until `nextRetryAt`. Without
|
||||
// this guard, a flapping upstream would burn API quota every
|
||||
// reminder-tick (every 60s by default), defeating the whole point
|
||||
// of the exponential backoff.
|
||||
if (isChannelInCooldown({ nextRetryAt: channel.nextRetryAt }, now)) {
|
||||
getEvent()?.addMeta(
|
||||
`notification_skip_${channel.type.toLowerCase()}_cooldown`,
|
||||
channel.nextRetryAt?.toISOString() ?? "unknown",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
await sendToChannel(
|
||||
const outcome = await sendToChannel(
|
||||
channel.type as ChannelType,
|
||||
channel.config,
|
||||
payload,
|
||||
);
|
||||
} catch (err) {
|
||||
|
||||
if (outcome.ok) {
|
||||
await recordChannelSuccess({
|
||||
id: channel.id,
|
||||
userId: channel.userId,
|
||||
type: channel.type as ChannelType,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (outcome.hardReject) {
|
||||
await recordChannelHardReject(
|
||||
{
|
||||
id: channel.id,
|
||||
userId: channel.userId,
|
||||
type: channel.type as ChannelType,
|
||||
},
|
||||
outcome,
|
||||
);
|
||||
getEvent()?.addWarning(
|
||||
`Notification channel ${channel.type} auto-disabled: ${outcome.reason}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const result = await recordChannelTransientFailure(
|
||||
{
|
||||
id: channel.id,
|
||||
userId: channel.userId,
|
||||
type: channel.type as ChannelType,
|
||||
},
|
||||
outcome,
|
||||
now,
|
||||
);
|
||||
getEvent()?.addWarning(
|
||||
`Notification dispatch failed for channel ${channel.type}: ${err}`,
|
||||
`Notification dispatch failed for channel ${channel.type}: ${outcome.reason}` +
|
||||
(result.autoDisabled ? " (auto-disabled after 5 failures)" : ""),
|
||||
);
|
||||
} catch (err) {
|
||||
// Sender threw unexpectedly — treat as a soft failure so the
|
||||
// backoff schedule absorbs the blip. (Senders are supposed to
|
||||
// be no-throw, this is defence-in-depth.)
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
await recordChannelTransientFailure(
|
||||
{
|
||||
id: channel.id,
|
||||
userId: channel.userId,
|
||||
type: channel.type as ChannelType,
|
||||
},
|
||||
{ reason: "sender_threw", message },
|
||||
now,
|
||||
);
|
||||
getEvent()?.addWarning(
|
||||
`Notification sender threw for ${channel.type}: ${message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -120,13 +194,17 @@ async function sendToChannel(
|
||||
type: ChannelType,
|
||||
encryptedConfig: string,
|
||||
payload: NotificationPayload,
|
||||
): Promise<boolean> {
|
||||
): Promise<SendOutcome> {
|
||||
let decrypted: string;
|
||||
try {
|
||||
decrypted = decrypt(encryptedConfig);
|
||||
} catch {
|
||||
getEvent()?.addWarning(`Failed to decrypt ${type} channel config`);
|
||||
return false;
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
reason: `${type.toLowerCase()}_config_decrypt_failed`,
|
||||
};
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
@@ -136,10 +214,13 @@ async function sendToChannel(
|
||||
config = JSON.parse(decrypted) as TelegramChannelConfig;
|
||||
} catch {
|
||||
getEvent()?.addWarning("Failed to parse Telegram channel config");
|
||||
return false;
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
reason: "telegram_config_parse_failed",
|
||||
};
|
||||
}
|
||||
const result = await sendViaTelegram(config, payload);
|
||||
return result.ok;
|
||||
return sendViaTelegram(config, payload);
|
||||
}
|
||||
case "NTFY": {
|
||||
let config: NtfyChannelConfig;
|
||||
@@ -147,7 +228,11 @@ async function sendToChannel(
|
||||
config = JSON.parse(decrypted) as NtfyChannelConfig;
|
||||
} catch {
|
||||
getEvent()?.addWarning("Failed to parse ntfy channel config");
|
||||
return false;
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
reason: "ntfy_config_parse_failed",
|
||||
};
|
||||
}
|
||||
return sendViaNtfy(config, payload);
|
||||
}
|
||||
@@ -156,6 +241,10 @@ async function sendToChannel(
|
||||
}
|
||||
default:
|
||||
getEvent()?.addWarning(`Unknown notification channel type: ${type}`);
|
||||
return false;
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
reason: "unknown_channel_type",
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* Notification dispatcher retry policy (v1.4.15 Phase B3).
|
||||
*
|
||||
* Two failure classes:
|
||||
* - HARD reject (`hardReject: true`) → channel is dead, do NOT retry,
|
||||
* auto-disable immediately and capture a `disabledReason` so the UI can
|
||||
* explain *why* the user needs to re-pair.
|
||||
* - SOFT reject (`hardReject: false`) → upstream temporarily unhappy
|
||||
* (network timeout, 5xx, 429, generic fetch failure). Retry per the
|
||||
* backoff schedule and only auto-disable after `MAX_CONSECUTIVE_FAILURES`
|
||||
* in a row (5).
|
||||
*
|
||||
* Hard-reject signal mapping (per spec):
|
||||
* - Web-Push : statusCode 410 (FCM/Mozilla "subscription gone")
|
||||
* - Web-Push : statusCode 404 (subscription endpoint deleted) — already
|
||||
* deletes the row in the sender; treated as hard at the
|
||||
* channel level only when ALL subs were 410/404 (i.e. the
|
||||
* channel can no longer succeed).
|
||||
* - Telegram : Bot API description "chat not found" or
|
||||
* "Forbidden: bot was blocked by the user"
|
||||
* - ntfy : statusCode 410 (topic invalidated)
|
||||
*
|
||||
* Soft-reject signal mapping:
|
||||
* - Any HTTP 5xx, HTTP 429, fetch network error / timeout, generic non-OK
|
||||
* response that doesn't match the hard-reject rules.
|
||||
*/
|
||||
|
||||
export type RejectKind = "hard" | "soft" | "ok";
|
||||
|
||||
export interface SendOutcome {
|
||||
ok: boolean;
|
||||
/** True iff the upstream signaled a permanent failure (410, blocked-by-user, …). */
|
||||
hardReject?: boolean;
|
||||
/** HTTP status, when the sender went over HTTP. */
|
||||
statusCode?: number;
|
||||
/** Stable reason code suitable for `disabled_reason` / `last_failure_reason`. */
|
||||
reason?: string;
|
||||
/** Human-readable error fragment for logs (NOT for the disabled_reason column). */
|
||||
message?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Backoff schedule used to compute `next_retry_at` for soft failures.
|
||||
* Indexed by `consecutiveFailures` (0-based AFTER the failed attempt).
|
||||
*
|
||||
* Spec: [30s, 5min, 30min, 2h, gave-up]. The 5th transient failure flips
|
||||
* the channel to auto-disabled with reason `give_up_after_5_failures`.
|
||||
*/
|
||||
export const BACKOFF_SCHEDULE_MS: readonly number[] = Object.freeze([
|
||||
30 * 1_000, // 1st failure → wait 30s
|
||||
5 * 60 * 1_000, // 2nd failure → wait 5min
|
||||
30 * 60 * 1_000, // 3rd failure → wait 30min
|
||||
2 * 60 * 60 * 1_000, // 4th failure → wait 2h
|
||||
]);
|
||||
|
||||
export const MAX_CONSECUTIVE_FAILURES = 5;
|
||||
|
||||
/**
|
||||
* Given the new `consecutiveFailures` value (1, 2, …) AFTER incrementing,
|
||||
* return the wall-clock time when a retry is allowed. Returns null once the
|
||||
* counter crosses `MAX_CONSECUTIVE_FAILURES` (caller must auto-disable).
|
||||
*
|
||||
* The schedule is offset by 1 because `consecutiveFailures=1` means the
|
||||
* first failure has just happened and we want a 30s cooldown.
|
||||
*/
|
||||
export function nextRetryAt(
|
||||
consecutiveFailures: number,
|
||||
now: Date = new Date(),
|
||||
): Date | null {
|
||||
if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) return null;
|
||||
if (consecutiveFailures < 1) return null;
|
||||
const idx = Math.min(consecutiveFailures - 1, BACKOFF_SCHEDULE_MS.length - 1);
|
||||
return new Date(now.getTime() + BACKOFF_SCHEDULE_MS[idx]);
|
||||
}
|
||||
|
||||
/**
|
||||
* After applying a transient failure, decide whether the channel should be
|
||||
* auto-disabled (i.e. give-up-after-N-failures fired).
|
||||
*/
|
||||
export function shouldAutoDisableAfterTransient(
|
||||
consecutiveFailures: number,
|
||||
): boolean {
|
||||
return consecutiveFailures >= MAX_CONSECUTIVE_FAILURES;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a Telegram Bot API error string into hard/soft.
|
||||
* The Bot API uses the `description` field (HTTP 200 with `ok=false`), so the
|
||||
* sender doesn't have a clean status code — this is the canonical source.
|
||||
*/
|
||||
export function classifyTelegramError(description: string | undefined): {
|
||||
hardReject: boolean;
|
||||
reason: string;
|
||||
} {
|
||||
const d = (description ?? "").toLowerCase();
|
||||
// Permanent: chat does not exist (user deleted the chat or blocked us).
|
||||
if (d.includes("chat not found")) {
|
||||
return { hardReject: true, reason: "telegram_chat_not_found" };
|
||||
}
|
||||
if (
|
||||
d.includes("blocked by the user") ||
|
||||
d.includes("bot was blocked") ||
|
||||
d.includes("user is deactivated")
|
||||
) {
|
||||
return { hardReject: true, reason: "telegram_blocked_by_user" };
|
||||
}
|
||||
return { hardReject: false, reason: "telegram_send_failed" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a numeric HTTP status into our hard/soft classification for
|
||||
* "regular" HTTP-shaped senders (web-push, ntfy).
|
||||
*
|
||||
* Anything that isn't a hard 410/404 (where applicable) and isn't a
|
||||
* 2xx is treated as soft: a transient blip the dispatcher should
|
||||
* retry rather than burn the user's channel for.
|
||||
*/
|
||||
export function classifyHttpStatus(
|
||||
status: number | undefined,
|
||||
service: "web-push" | "ntfy",
|
||||
): { hardReject: boolean; reason: string } {
|
||||
if (status === undefined) {
|
||||
return { hardReject: false, reason: `${service}_network_error` };
|
||||
}
|
||||
if (service === "web-push") {
|
||||
// 410 Gone = subscription is permanently invalid (push provider
|
||||
// told us so). 404 = endpoint never existed / was unsubscribed.
|
||||
if (status === 410)
|
||||
return { hardReject: true, reason: "web_push_410_gone" };
|
||||
if (status === 404)
|
||||
return { hardReject: true, reason: "web_push_404_endpoint" };
|
||||
}
|
||||
if (service === "ntfy") {
|
||||
// ntfy.sh treats 410 as "topic deleted, do not retry".
|
||||
if (status === 410) return { hardReject: true, reason: "ntfy_410_gone" };
|
||||
}
|
||||
return { hardReject: false, reason: `${service}_${status}` };
|
||||
}
|
||||
@@ -2,16 +2,23 @@ import type {
|
||||
NtfyChannelConfig,
|
||||
NotificationPayload,
|
||||
} from "@/lib/notifications/types";
|
||||
import type { SendOutcome } from "@/lib/notifications/retry-policy";
|
||||
import { classifyHttpStatus } from "@/lib/notifications/retry-policy";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
|
||||
/**
|
||||
* Send notification via ntfy (simple HTTP POST).
|
||||
* See https://docs.ntfy.sh/publish/
|
||||
*
|
||||
* Returns a structured `SendOutcome` so the dispatcher (v1.4.15 Phase B3)
|
||||
* can distinguish hard rejects (HTTP 410, topic deleted) from soft errors
|
||||
* (5xx, 429, network timeout) and apply the right retry / auto-disable
|
||||
* policy.
|
||||
*/
|
||||
export async function sendViaNtfy(
|
||||
config: NtfyChannelConfig,
|
||||
payload: NotificationPayload,
|
||||
): Promise<boolean> {
|
||||
): Promise<SendOutcome> {
|
||||
const start = performance.now();
|
||||
try {
|
||||
const url = `${config.serverUrl.replace(/\/$/, "")}/${encodeURIComponent(config.topic)}`;
|
||||
@@ -44,14 +51,29 @@ export async function sendViaNtfy(
|
||||
status: res.status,
|
||||
});
|
||||
|
||||
return res.ok;
|
||||
if (res.ok) {
|
||||
return { ok: true, statusCode: res.status };
|
||||
}
|
||||
const classified = classifyHttpStatus(res.status, "ntfy");
|
||||
return {
|
||||
ok: false,
|
||||
statusCode: res.status,
|
||||
hardReject: classified.hardReject,
|
||||
reason: classified.reason,
|
||||
};
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "request_failed";
|
||||
getEvent()?.addExternalCall({
|
||||
service: "ntfy",
|
||||
method: "sendNotification",
|
||||
duration_ms: Math.round(performance.now() - start),
|
||||
error: err instanceof Error ? err.message : "request_failed",
|
||||
error: message,
|
||||
});
|
||||
return false;
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
reason: "ntfy_network_error",
|
||||
message,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,23 @@
|
||||
import { sendTelegramMessage, deleteMessage } from "@/lib/telegram";
|
||||
import type { SendMessageResult } from "@/lib/telegram";
|
||||
import type {
|
||||
TelegramChannelConfig,
|
||||
NotificationPayload,
|
||||
} from "@/lib/notifications/types";
|
||||
import type { SendOutcome } from "@/lib/notifications/retry-policy";
|
||||
import { classifyTelegramError } from "@/lib/notifications/retry-policy";
|
||||
import { prisma } from "@/lib/db";
|
||||
import type { ReminderPhase } from "@/generated/prisma/client";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
|
||||
/**
|
||||
* Telegram sender result. Inherits from `SendOutcome` so the dispatcher
|
||||
* can classify hard vs soft rejects, and tacks on `messageId` for the
|
||||
* reminder-tracking flow that wants to delete the message later.
|
||||
*/
|
||||
export interface TelegramSendResult extends SendOutcome {
|
||||
messageId?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete ALL existing Telegram reminder messages for a medication (any date).
|
||||
* Ensures max one active message per medication in the chat.
|
||||
@@ -52,7 +62,7 @@ async function deleteExistingReminders(
|
||||
export async function sendViaTelegram(
|
||||
config: TelegramChannelConfig,
|
||||
payload: NotificationPayload,
|
||||
): Promise<SendMessageResult> {
|
||||
): Promise<TelegramSendResult> {
|
||||
const medicationId = payload.metadata?.medicationId as string | undefined;
|
||||
const scheduleId = payload.metadata?.scheduleId as string | undefined;
|
||||
const phase = payload.metadata?.phase as string | undefined;
|
||||
@@ -143,5 +153,14 @@ export async function sendViaTelegram(
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
if (result.ok) {
|
||||
return { ok: true, messageId: result.messageId };
|
||||
}
|
||||
const classified = classifyTelegramError(result.errorDescription);
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: classified.hardReject,
|
||||
reason: classified.reason,
|
||||
message: result.errorDescription,
|
||||
};
|
||||
}
|
||||
@@ -1,18 +1,30 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import type { NotificationPayload } from "@/lib/notifications/types";
|
||||
import type { SendOutcome } from "@/lib/notifications/retry-policy";
|
||||
import { classifyHttpStatus } from "@/lib/notifications/retry-policy";
|
||||
import { getVapidConfig } from "@/lib/notifications/vapid-config";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
|
||||
/**
|
||||
* Send Web Push notification to all subscribed devices of a user.
|
||||
* VAPID config is loaded from app settings first, then env as fallback.
|
||||
* Returns true if at least one delivery succeeded.
|
||||
*
|
||||
* Returns a structured `SendOutcome` (v1.4.15 Phase B3) so the dispatcher
|
||||
* can classify the channel state:
|
||||
* - `ok: true` if at least one device received the push.
|
||||
* - `hardReject: true` only when ALL surviving subscriptions returned a
|
||||
* permanent reject (410/404) and the channel can no longer succeed.
|
||||
* The 410 subscriptions themselves are deleted from the DB regardless,
|
||||
* so a hard-channel-reject means the user has zero devices left.
|
||||
* - `hardReject: false` for any 5xx, 429, fetch failure, or
|
||||
* "no subscriptions at all" (treated as soft so the user can re-pair
|
||||
* a device without the channel staying auto-disabled forever).
|
||||
*/
|
||||
export async function sendViaWebPush(
|
||||
userId: string,
|
||||
payload: NotificationPayload,
|
||||
): Promise<boolean> {
|
||||
): Promise<SendOutcome> {
|
||||
try {
|
||||
// Lazy import to avoid issues when web-push is not installed
|
||||
const webpush = await import("web-push");
|
||||
@@ -20,7 +32,11 @@ export async function sendViaWebPush(
|
||||
const config = await getVapidConfig();
|
||||
if (!config) {
|
||||
getEvent()?.addWarning("Web Push: VAPID keys not configured");
|
||||
return false;
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
reason: "web_push_vapid_not_configured",
|
||||
};
|
||||
}
|
||||
|
||||
webpush.setVapidDetails(
|
||||
@@ -33,7 +49,17 @@ export async function sendViaWebPush(
|
||||
where: { userId },
|
||||
});
|
||||
|
||||
if (!subscriptions.length) return false;
|
||||
if (!subscriptions.length) {
|
||||
// No devices registered yet — not a hard channel reject. The user
|
||||
// may simply not have hit "Subscribe" on this browser. Counting
|
||||
// this as a give-up signal would lock the user out of the channel
|
||||
// they just configured. Treat it as a soft "no recipient" instead.
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
reason: "web_push_no_subscriptions",
|
||||
};
|
||||
}
|
||||
|
||||
const pushPayload = JSON.stringify({
|
||||
title: payload.title,
|
||||
@@ -44,6 +70,8 @@ export async function sendViaWebPush(
|
||||
|
||||
let anySuccess = false;
|
||||
const expiredIds: string[] = [];
|
||||
let lastTransientStatus: number | undefined;
|
||||
let allPermanentReject = true;
|
||||
const pushStart = performance.now();
|
||||
|
||||
for (const sub of subscriptions) {
|
||||
@@ -59,11 +87,19 @@ export async function sendViaWebPush(
|
||||
pushPayload,
|
||||
);
|
||||
anySuccess = true;
|
||||
allPermanentReject = false;
|
||||
} catch (err: unknown) {
|
||||
const status = (err as { statusCode?: number }).statusCode;
|
||||
if (status === 410 || status === 404) {
|
||||
// Subscription expired or invalid — mark for cleanup
|
||||
// Subscription expired or invalid — mark for cleanup, but
|
||||
// don't flip the whole channel hard until we know NO sub
|
||||
// succeeded.
|
||||
expiredIds.push(sub.id);
|
||||
} else {
|
||||
// Soft failure on at least one sub — channel is alive enough
|
||||
// that we shouldn't auto-disable on this dispatch.
|
||||
allPermanentReject = false;
|
||||
lastTransientStatus = status;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -82,13 +118,40 @@ export async function sendViaWebPush(
|
||||
});
|
||||
}
|
||||
|
||||
return anySuccess;
|
||||
if (anySuccess) {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
// Every sub returned a permanent 410/404 → channel is dead until
|
||||
// the user re-pairs a device. This is a hard reject.
|
||||
if (allPermanentReject && expiredIds.length === subscriptions.length) {
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: true,
|
||||
statusCode: 410,
|
||||
reason: "web_push_410_gone",
|
||||
};
|
||||
}
|
||||
|
||||
// Soft failure (5xx / 429 / fetch error) on at least one sub.
|
||||
const classified = classifyHttpStatus(lastTransientStatus, "web-push");
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
statusCode: lastTransientStatus,
|
||||
reason: classified.reason,
|
||||
};
|
||||
} catch (err) {
|
||||
getEvent()?.setError(
|
||||
err instanceof Error
|
||||
? err
|
||||
: new Error("[web-push] sendViaWebPush failed"),
|
||||
);
|
||||
return false;
|
||||
return {
|
||||
ok: false,
|
||||
hardReject: false,
|
||||
reason: "web_push_unexpected_error",
|
||||
message: err instanceof Error ? err.message : "unknown",
|
||||
};
|
||||
}
|
||||
}
|
||||
+9
-2
@@ -63,11 +63,18 @@ async function telegramApiRequest(
|
||||
export interface SendMessageResult {
|
||||
ok: boolean;
|
||||
messageId?: number;
|
||||
/**
|
||||
* Bot-API `description` field on failure. Surfaced so the dispatcher
|
||||
* (v1.4.15 Phase B3) can classify "chat not found" / "blocked by the
|
||||
* user" as hard rejects vs a generic 5xx as a soft reject.
|
||||
*/
|
||||
errorDescription?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a text message via the Telegram Bot API.
|
||||
* Returns { ok, messageId } on success, { ok: false } on failure (never throws).
|
||||
* Returns { ok, messageId } on success, { ok: false, errorDescription } on
|
||||
* failure (never throws).
|
||||
*/
|
||||
export async function sendTelegramMessage(
|
||||
botToken: string,
|
||||
@@ -85,7 +92,7 @@ export async function sendTelegramMessage(
|
||||
getEvent()?.addWarning(
|
||||
`[telegram] sendMessage failed: ${json.description}`,
|
||||
);
|
||||
return { ok: false };
|
||||
return { ok: false, errorDescription: json.description };
|
||||
}
|
||||
const messageId = (json.result as { message_id?: number })?.message_id;
|
||||
return { ok: true, messageId };
|
||||
|
||||
Reference in new issue
Block a user