Files
HealthLog/src/lib/integrations/status.ts
T
Marc-André Bombeck 689796db33 fix(integrations): log a failed sync as a signal, not an audit row
The audit log is the trail of what a person or an admin did to an account.
Every failed provider sync wrote a row there, burying that trail, and kept
the provider's error message in plain text while the status row seals it.
A failed sync now emits an integrations.sync.failed warning with the same
counts and no message.
2026-10-20 10:03:30 +02:00

1446 lines
55 KiB
TypeScript

/**
* Per-(user, integration) sync-status bookkeeping.
*
* v1.4.15 Phase B2 introduces three responsibilities for every sync attempt:
*
* 1. Record success / failure timestamps so Settings → Integrations
* can show "last sync 23 minutes ago" instead of guessing from
* `lastSyncedAt` (which only updates on success and tells the user
* nothing about a 2-day failure streak).
* 2. Emit one structured `AuditLog` row per failure — successes are
* not audited (would be noisy and `lastSuccessAt` already tracks
* them).
* 3. After N consecutive failures (default 3), notify the admin via
* Telegram so a token-revoke or upstream outage doesn't silently
* strand a user's data. The dispatcher is reused as-is — B3 owns
* reliability/retry of the dispatcher; we are only a caller.
*
* The state machine is deliberately small:
*
* connected → happy path. cleared by recordSyncSuccess().
* error_transient → at least one failure since last success. The
* sync entry-point still attempts on the next
* run (network blip, 5xx, etc.).
* error_reauth → refresh-token grant has revoked. `getValidToken`
* short-circuits until the user reconnects. Cleared
* by markReconnected() on the OAuth callback.
* disconnected → user clicked "Disconnect". Set explicitly by
* the disconnect routes; clears any prior error
* state.
*
* `state` is stored as a free-form string column (not a Postgres enum)
* so adding new sentinels in v1.5 doesn't require a migration.
*/
import { prisma } from "@/lib/db";
import { encrypt, decrypt } from "@/lib/crypto";
import { auditLog } from "@/lib/auth/audit";
import { annotate, getEvent } from "@/lib/logging/context";
import { emitSignal } from "@/lib/logging/signal";
import {
dispatchNotification,
type DispatchOutcome,
} from "@/lib/notifications/dispatcher";
import type { IntegrationClassification } from "@/lib/integrations/http-status-classifier";
import {
RECENT_CALLS_LIMIT,
safeEndpointName,
safeErrorCode,
type RecentCallStatus,
} from "@/lib/integrations/connection-health";
export type IntegrationKey =
| "withings"
| "whoop"
| "fitbit"
// v1.17.0 — Nightscout glucose (F1) + Polar / Oura OAuth (F4).
| "nightscout"
| "polar"
| "oura"
// v1.27.0 — Google Health (Fitbit + Pixel Watch + Fitbit Air) via the
// successor Google Health API. Separate connection from the classic
// `fitbit` transport, which sunsets Sept 2026.
| "google-health"
// v1.28.x — Strava OAuth workout source.
| "strava"
// v1.43 — Dawarich movement context (daily values, nightly).
| "dawarich";
/**
* Failure kinds carried into `recordSyncFailure`.
*
* - `transient` : retry on the next sync; user not blocked.
* - `reauth_required` : permanent revoke / invalid_grant; park at
* `error_reauth` until the user reconnects.
* - `persistent` : contract mismatch (invalid params, missing
* field, unknown action). Surfaces in the
* integration-status card AND audit log so an
* operator can investigate, but does NOT skip
* future sync attempts — those may succeed
* once the upstream side resolves. v1.4.43
* W14: after 24h of unbroken persistent
* failures the row is `parked` and the
* integration stops attempting until the user
* / operator reconnects.
*
* v1.4.42 W6 extended this union from `transient | reauth_required` to
* the three-state taxonomy above; the state-mapping function turns
* `persistent` into `error_transient` for now (a Withings 293 still
* lets the next sync run), but the audit detail carries the explicit
* kind so operations can filter.
*/
export type FailureKind = "transient" | "reauth_required" | "persistent";
/**
* Map a shared `IntegrationClassification` onto the ledger `FailureKind`.
*
* `success` never reaches a failure path, so it collapses into `transient`
* alongside the genuinely-retryable verdicts; `reauth_required` and
* `persistent` pass through unchanged. Lifted out of the per-vendor
* `classifyXFailure` adapters (Polar / Oura / Nightscout), which each carried a
* byte-identical copy of this mapping.
*/
export function toFailureKind(
classification: IntegrationClassification,
): FailureKind {
if (classification === "reauth_required") return "reauth_required";
if (classification === "persistent") return "persistent";
return "transient";
}
/**
* Non-enumerable marker stamped on an error a provider sync ALREADY recorded on
* the integration ledger before rethrowing. The poll-cohort boundary
* (`makePollCohortHandler`) reads it so a record-then-rethrow failure is not
* recorded a SECOND time when it surfaces in the cohort catch — a double-record
* would inflate the per-kind failure buckets (firing the 3-strike admin alert a
* tick early) and churn `lastError`. A provider-blind recorder at the boundary
* would also persist an UNREDACTED message (e.g. Nightscout's token-bearing
* URL), so the marker keeps classification + redaction provider-owned: only an
* UNMARKED escape reaches the boundary recorder.
*
* No-op on non-objects; tolerant of a frozen error (the boundary then records
* it once, which is the correct fallback — never a double-record).
*/
const SYNC_FAILURE_RECORDED: unique symbol = Symbol(
"healthlog.syncFailureRecorded",
);
/** Stamp `err` as already-recorded and return it (for `throw markSync…(err)`). */
export function markSyncFailureRecorded<T>(err: T): T {
if (err !== null && typeof err === "object") {
try {
Object.defineProperty(err, SYNC_FAILURE_RECORDED, {
value: true,
enumerable: false,
configurable: true,
writable: true,
});
} catch {
// A frozen / sealed error can't carry the marker; the boundary records
// it once — the safe fallback, never a double-record.
}
}
return err;
}
/** True when `err` was already recorded on the ledger by its source site. */
export function isSyncFailureRecorded(err: unknown): boolean {
return (
err !== null &&
typeof err === "object" &&
(err as Record<symbol, unknown>)[SYNC_FAILURE_RECORDED] === true
);
}
/**
* Recognised IntegrationStatus states.
*
* `parked` (v1.4.43 W14) is set when an integration's persistent
* failure streak has exceeded `PARK_PERSISTENT_FAILURE_AFTER_MS` (24h
* by default). A parked integration STOPS RETRYING — the next
* scheduled sync skips, and the Settings UI surfaces a "Paused —
* reconnect manually" pill with a "Wieder verbinden" CTA that POSTs
* to `/api/integrations/withings/resume` to clear the park. This is
* intentionally heavier than `error_transient`: a contract-mismatch
* that's been failing for a full day is no longer "the upstream might
* recover on its own" — it's an operator-shaped problem, and
* retrying every 15 minutes for another week just buries the audit
* trail.
*/
export type IntegrationState =
| "connected"
| "error_transient"
| "error_reauth"
| "disconnected"
| "parked"
/**
* v1.38.19 — no ledger row exists for this (user, provider) pair, so the
* ledger has nothing to say. It is NOT a claim about the connection either
* way: `connected` used to be the synthetic default here, which made the
* field's default read as the opposite of the truth for every account that
* had never connected anything — the setup flow's connect step believed it
* and told a brand-new account its wearable was connected. Consumers treat
* `unknown` the way they treat `disconnected`; `syncHealth.verdict` stays
* the only liveness truth.
*/
| "unknown";
/**
* The ladder at which a streak of failures escalates from "user-visible
* banner" to "admin-paged on Telegram". 3 is small enough to catch a
* truly broken integration before the user notices missing data, large
* enough that one network blip doesn't page anyone.
*
* Override via env `INTEGRATION_FAILURE_ALERT_THRESHOLD` for ops who
* want a louder or quieter signal — we read it lazily so tests can
* mutate it per case.
*/
export function getPersistentFailureThreshold(): number {
const raw = process.env.INTEGRATION_FAILURE_ALERT_THRESHOLD;
const parsed = raw ? Number.parseInt(raw, 10) : NaN;
if (Number.isFinite(parsed) && parsed >= 1) return parsed;
return 3;
}
/**
* One page per failure streak. `alertedAt` is cleared by every path that ends
* a streak (a success entitled to clear the row, a reconnect, a resume, a
* disconnect), so a stamped `alertedAt` means "this streak was already
* announced". It used to re-page every 24 h for as long as the streak ran,
* which turned a week-long outage into seven identical messages; the
* integration watchdog (`./watchdog`) now owns the rest of the incident,
* including the one recovery note when it ends.
*/
/**
* v1.4.43 — once a `persistent` failure streak has been running
* for this long without a single intervening success, flip the
* integration to `parked`. 24h matches the same window the alert
* ladder uses for re-paging and gives the operator a full business
* day to notice a 293/294 surge before the integration disables
* itself.
*/
const PARK_PERSISTENT_FAILURE_AFTER_MS = 24 * 60 * 60 * 1000;
/**
* How long an unbroken failure streak may keep claiming to be `transient`
* before the claim is treated as refuted and the failure is recorded as
* `persistent` instead.
*
* `transient` is a hypothesis, not an observation: the classifier defaults a
* timeout, a 429, a 5xx and any unrecognised response to it precisely because
* the wire cannot tell "the upstream is rebooting" from "this will never work
* again". That default is right for one attempt and wrong for a thousand — a
* streak that has been retried hourly for a full day without a single success
* has falsified the hypothesis by experiment, and continuing to call it
* transient is the ledger asserting a certainty the wire never gave it.
*
* 24 h is the same window the persistent park already uses. An upstream
* maintenance night, a rate-limit cool-off, a redeploy and a DNS wobble all
* fit comfortably inside it, so a genuine blip recovers and resets the streak
* long before this trips — nothing that heals on its own takes a day. The
* escalation is additionally gated on `getPersistentFailureThreshold()`
* consecutive failures, so a provider whose cron ticks rarely cannot escalate
* on its second-ever attempt just because wall-clock time passed.
*
* Escalating feeds the EXISTING ladder rather than a second one: the
* `persistent` bucket starts counting, `persistentFailureStartedAt` anchors,
* and `PARK_PERSISTENT_FAILURE_AFTER_MS` later the row parks and stops
* retrying — 48 h of unbroken failure end to end before anything is disabled.
*/
const TRANSIENT_ESCALATION_AFTER_MS = 24 * 60 * 60 * 1000;
/** Row states that mean "an error is currently held on this row". */
const ERROR_STATES: ReadonlySet<string> = new Set<IntegrationState>([
"error_transient",
"error_reauth",
"parked",
]);
/**
* Read the `failingLegs` JSON column into an ordered list of leg names.
*
* Anything that is not an array of strings — a legacy `null`, a hand-edited
* row, a shape from a future schema — reads as the empty set, which is the
* pre-existing "unattributed failure" semantics under which any success clears
* the row. Non-string members are dropped rather than coerced: a leg name is an
* identifier the caller passes, and inventing one from a number would let a
* success match a leg nobody ever recorded.
*/
function readFailingLegs(value: unknown): string[] {
if (!Array.isArray(value)) return [];
return value.filter((entry): entry is string => typeof entry === "string");
}
/**
* Append `leg` as the most recent failure, moving it to the end if it is
* already in the set. Ordering is load-bearing: the last element is the leg
* that wrote the message `lastError` currently holds, which is what lets a
* partial recovery tell "this message still describes a failing leg" from
* "this message belongs to a leg that has since recovered".
*/
function withLegRecorded(current: readonly string[], leg: string): string[] {
return [...current.filter((entry) => entry !== leg), leg];
}
export interface IntegrationStatusSnapshot {
integration: IntegrationKey;
state: IntegrationState;
lastSuccessAt: string | null;
lastAttemptAt: string | null;
lastError: string | null;
/** v1.4.43 — per-kind bucketed counters; `null` for rows
* that have never recorded a write yet. v1.4.47 W1 — sole source
* of truth now that the legacy `consecutiveFailures` column is
* gone. */
consecutiveFailuresByKind: ConsecutiveFailuresByKind | null;
/**
* Start of the current unbroken failure streak, whatever the kind. The
* buckets count attempts; this dates them, which is what tells a blip apart
* from a pipe that has been dead for weeks. `null` whenever the row is not
* currently failing.
*/
failingSince: string | null;
}
/**
* Bucketed consecutive-failure counters keyed by `FailureKind`.
* v1.4.43 — exposed so the response shape stays explicit and
* tests can pin the bucket increments.
*/
export type ConsecutiveFailuresByKind = Record<FailureKind, number>;
/**
* Type guard for the JSON payload Prisma returns for the
* `consecutiveFailuresByKind` column. Anything that's not a plain
* object with three numeric keys is treated as "no value yet" so the
* writer starts from a zero envelope.
*/
function isFailureBucketObject(
value: unknown,
): value is ConsecutiveFailuresByKind {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return false;
}
const obj = value as Record<string, unknown>;
return (
typeof obj.transient === "number" &&
typeof obj.reauth_required === "number" &&
typeof obj.persistent === "number"
);
}
/**
* Read the JSON `consecutiveFailuresByKind` column off the Prisma row
* with a typed view. Returns `null` for rows that have never written a
* bucket payload — callers seed a fresh zero envelope.
*/
function readBucketColumn(value: unknown): ConsecutiveFailuresByKind | null {
return isFailureBucketObject(value) ? value : null;
}
/**
* Zero-bucket envelope. Inlined where we need the literal so eslint
* doesn't flag a re-assignment of the shared constant.
*/
function zeroBuckets(): ConsecutiveFailuresByKind {
return { transient: 0, reauth_required: 0, persistent: 0 };
}
/**
* Read the current snapshot. Returns a synthetic "unknown, never attempted"
* record when no row exists yet — the ledger holds no history, and says so
* rather than guessing a state. `resolveSyncVerdict` reads the same absence
* off the null timestamps and answers `pending_first_sync` for a live
* connection, `disconnected` for one that is not there.
*/
export async function getIntegrationStatus(
userId: string,
integration: IntegrationKey,
): Promise<IntegrationStatusSnapshot> {
const row = await prisma.integrationStatus.findUnique({
where: { userId_integration: { userId, integration } },
});
if (!row) {
return {
integration,
state: "unknown",
lastSuccessAt: null,
lastAttemptAt: null,
lastError: null,
consecutiveFailuresByKind: null,
failingSince: null,
};
}
return {
integration,
state: row.state as IntegrationState,
lastSuccessAt: row.lastSuccessAt?.toISOString() ?? null,
lastAttemptAt: row.lastAttemptAt?.toISOString() ?? null,
lastError: row.lastError ? safeDecryptError(row.lastError) : null,
consecutiveFailuresByKind: readBucketColumn(row.consecutiveFailuresByKind),
failingSince: row.failingSinceAt?.toISOString() ?? null,
};
}
/**
* Read the current `state` cheaply — used by the sync entry-points
* to short-circuit when reauth is required.
*
* v1.4.43 — `parked` is treated as "reauth required" for the
* purpose of sync short-circuit: the user has to call `/resume` (or
* complete the OAuth flow again) before any further sync work
* happens. Returning `true` for both states means existing call sites
* keep their current "skip the cron tick" behaviour without churn,
* and the Settings UI distinguishes the two via the pill copy.
*/
export async function isReauthRequired(
userId: string,
integration: IntegrationKey,
): Promise<boolean> {
const row = await prisma.integrationStatus.findUnique({
where: { userId_integration: { userId, integration } },
select: { state: true },
});
return row?.state === "error_reauth" || row?.state === "parked";
}
export interface RecordSyncSuccessOptions {
/**
* The sync leg that succeeded, for a provider whose legs run on separate
* schedules (Withings measures / activity / sleep / ECG, the four WHOOP
* resources, the two Polar legs). Omit it for a single-leg provider or for a
* full pass that covered every leg — an omitted leg clears the row whole,
* which is the pre-existing behaviour and the correct one for those callers.
*/
leg?: string;
}
/**
* Record a successful sync. Resets the failure counter, clears any
* prior error message, and (importantly) flips state back to
* `connected` even from `error_reauth` — the Withings flow re-enters
* after the OAuth callback writes a new refresh token.
*
* A success only clears what it is entitled to clear. `IntegrationStatus` is
* keyed `(userId, integration)`, but Withings runs four legs on four crons and
* WHOOP runs four resources on four more — so an unconditional clear meant the
* ECG leg succeeding at :41 wiped the error, the buckets, the streak anchor and
* the state that the sleep leg had recorded at :15. The failing leg's strike
* ladder was reset several times an hour, so it could never reach the alert
* threshold or the park window, and the card painted a green "connected · 12
* minutes ago" over a data class that had stopped arriving.
*
* This success removes its OWN leg from the failing set and clears the row only
* once that set is empty. The set matters because the previous single slot
* could remember one leg at a time: with sleep and activity both failing, the
* second failure overwrote the first, and the second leg recovering then wiped
* an error the first was still causing. The ledger forgot a live failure and
* went green over it — the exact case this set exists to hold.
*
* The partial path deliberately does NOT advance `lastSuccessAt`. That field is
* what the freshness arms and the card's "connected · X ago" read, and moving
* it forward while another leg is dead is exactly the sentence this change
* exists to stop the ledger from saying. `lastAttemptAt` does advance, so the
* row still reads as "being tried" rather than ageing into `stalled`.
*
* `lastError` is cleared on the partial path when the recovering leg is the one
* that wrote it. The message then describes a leg that is now healthy, and a
* row that keeps showing it is describing the wrong failure; the next attempt
* by a still-failing leg writes its own. No message is honest here — a guessed
* one would not be.
*/
export async function recordSyncSuccess(
userId: string,
integration: IntegrationKey,
options?: RecordSyncSuccessOptions,
): Promise<void> {
const now = new Date();
const leg = options?.leg ?? null;
if (leg !== null) {
const existing = await prisma.integrationStatus.findUnique({
where: { userId_integration: { userId, integration } },
select: { state: true, failingLegs: true },
});
const failingLegs = readFailingLegs(existing?.failingLegs);
const remaining = failingLegs.filter((entry) => entry !== leg);
const heldByAnotherLeg =
existing != null &&
remaining.length > 0 &&
ERROR_STATES.has(existing.state);
if (heldByAnotherLeg) {
// The recovering leg owned `lastError` iff it was the most recent
// failure, i.e. the last element. Dropping it leaves the row without a
// message for any leg that is still failing, which is the honest state
// until one of them fails again and writes its own.
const ownedLastError = failingLegs[failingLegs.length - 1] === leg;
await prisma.integrationStatus.update({
where: { userId_integration: { userId, integration } },
data: {
lastAttemptAt: now,
failingLegs: remaining,
...(ownedLastError ? { lastError: null } : {}),
},
});
await appendRecentCall(userId, integration, {
endpoint: leg,
status: "ok",
at: now,
errorCode: null,
});
// Meta only — the enclosing wide event owns its own action name (the
// cron task), and overwriting it here would rename the job.
annotate({
meta: {
integration_partial_success: {
integration,
succeeded_leg: leg,
failing_legs: remaining,
},
},
});
return;
}
}
// v1.4.43 — a success resets ALL per-kind buckets back to zero
// and clears the persistent-streak start timestamp.
// v1.4.47 — the legacy `consecutiveFailures` column was dropped
// (migration 0077), so the bucket reset is the only counter write.
await prisma.integrationStatus.upsert({
where: { userId_integration: { userId, integration } },
create: {
userId,
integration,
state: "connected",
lastSuccessAt: now,
lastAttemptAt: now,
consecutiveFailuresByKind: zeroBuckets(),
failingLegs: [],
},
update: {
state: "connected",
lastSuccessAt: now,
lastAttemptAt: now,
lastError: null,
consecutiveFailuresByKind: zeroBuckets(),
persistentFailureStartedAt: null,
failingSinceAt: null,
failingLegs: [],
alertedAt: null,
},
});
await appendRecentCall(userId, integration, {
endpoint: leg,
status: "ok",
at: now,
errorCode: null,
});
}
export interface RecordSyncFailureInput {
userId: string;
integration: IntegrationKey;
kind: FailureKind;
message: string;
/** Optional structured error code (e.g. "invalid_grant", "401"). */
errorCode?: string;
/**
* The sync leg that failed, for a provider whose legs run on separate
* schedules. Recorded on the row so a sibling leg's success cannot clear an
* error it did not cause. Omit it for a single-leg provider.
*/
leg?: string;
}
/**
* Record a sync failure. Always:
* - increments the per-kind bucket in `consecutiveFailuresByKind`
* - persists the encrypted error message
* - emits one `integrations.sync.failed` signal (not an audit row)
*
* If `kind === "reauth_required"` the row is parked at `error_reauth`
* so the next scheduled sync skips. If `kind === "persistent"` and
* the persistent streak has been running for >24h, the row is parked
* at `parked` so the next sync also skips — the operator / user has
* to call `resumeIntegrationFromPark` to clear it. Otherwise the state
* is `error_transient` and the next sync will try again.
*
* If the post-update bucket max crosses the alerting threshold AND the
* alerting window has lapsed, dispatch a Telegram notification to all
* admins. Failures here are best-effort: a failed dispatch DOES NOT
* swallow the audit log.
*
* v1.4.43 — per-kind counter migration. Each failure increments
* ONLY its own bucket; a transient hiccup followed by a persistent
* failure no longer masks the persistent streak's true age.
* v1.4.47 — the legacy single-column `consecutiveFailures` integer
* was dropped (migration 0077); the bucket is now the sole counter
* and the back-fill branch is gone.
*
* A `transient` failure whose streak has run unbroken for longer than
* `TRANSIENT_ESCALATION_AFTER_MS` is recorded as `persistent` instead. Before
* that, a `transient` streak had no exit at all: the park test read
* `kind === "persistent"`, `persistentFailureStartedAt` was only stamped on a
* persistent failure, and so a streak the classifier had bucketed as transient
* could run for a thousand attempts with the persistent counter at zero, the
* state frozen at `error_transient`, and no ladder it could ever climb. See
* the constant for why time, rather than count, is the honest signal.
*/
export async function recordSyncFailure(
input: RecordSyncFailureInput,
): Promise<void> {
const { userId, integration, kind, message, errorCode } = input;
const leg = input.leg ?? null;
const now = new Date();
const encryptedError = safeEncryptError(message);
// Read the current row first so we can:
// (a) compute the new per-kind bucket value
// (b) decide whether the streak has outlived the `transient` claim
// (c) decide whether the persistent streak has exceeded the
// park threshold
const existing = await prisma.integrationStatus.findUnique({
where: { userId_integration: { userId, integration } },
select: {
consecutiveFailuresByKind: true,
persistentFailureStartedAt: true,
failingSinceAt: true,
failingLegs: true,
alertedAt: true,
},
});
// The failing set, with this leg appended as the most recent failure. An
// unattributed failure (no leg — a single-leg provider, or a recorder that
// does not know which leg it caught) adds nothing and removes nothing: it
// carries no attribution to record, and erasing what other legs already
// reported would let one of their successes clear an error it did not fix.
const existingLegs = readFailingLegs(existing?.failingLegs);
const failingLegs =
leg !== null ? withLegRecorded(existingLegs, leg) : existingLegs;
// Resolve the starting bucket envelope:
// - existing JSON value if present
// - zero envelope for a row that has never written a bucket payload
// or for the first-ever write of this (user, integration) pair
const startingBuckets = readBucketColumn(existing?.consecutiveFailuresByKind);
// Streak anchor: the first failure of the current unbroken run. Every path
// that resolves a failure (success entitled to clear, reconnect, disconnect,
// resume) nulls it, so a non-null value always dates a live streak.
const failingSinceAt = existing?.failingSinceAt ?? now;
const streakAgeMs = now.getTime() - failingSinceAt.getTime();
// The `transient` hypothesis, tested against the clock. Both conditions are
// required: the wall-clock window rules out a blip, and the consecutive-count
// gate keeps a rarely-polled provider from escalating on its second attempt
// just because a day of real time went by. `+ 1` because this failure has not
// been written into the bucket yet.
const transientStreakAfterThis = (startingBuckets?.transient ?? 0) + 1;
const escalatedFromTransient =
kind === "transient" &&
streakAgeMs >= TRANSIENT_ESCALATION_AFTER_MS &&
transientStreakAfterThis >= getPersistentFailureThreshold();
// Everything downstream — bucket, state mapping, park test, audit detail,
// alert copy — reads the effective kind, so the escalation feeds the ladder
// that already exists instead of growing a parallel one.
const effectiveKind: FailureKind = escalatedFromTransient
? "persistent"
: kind;
// Snapshot the persistent-bucket count BEFORE we increment so the
// streak-anchor decision below can see the pre-increment value.
// Otherwise we'd inspect the freshly-incremented bucket and never
// recognise a "first persistent failure of a fresh streak".
const persistentStreakBefore = startingBuckets?.persistent ?? 0;
// Now build the new bucket envelope. `buckets` is always a fresh
// object so the upsert payload doesn't share a reference with the
// existing-row snapshot (which would couple the in-memory mutation
// to the audit-log read path below).
const buckets: ConsecutiveFailuresByKind = startingBuckets
? { ...startingBuckets }
: zeroBuckets();
// Increment only the bucket matching this failure's kind.
// The other two buckets stay at their current value so a persistent
// streak isn't reset by an intervening transient hiccup.
buckets[effectiveKind] = (buckets[effectiveKind] ?? 0) + 1;
// Track the persistent-streak start so the >24h park check has a
// wall-clock anchor. Only stamped on the FIRST persistent failure of
// a streak; cleared on success or when the persistent bucket goes
// back to zero (which today only happens via success, but the logic
// is symmetric for future "transient drained the streak" rules).
//
// An escalated transient anchors here too, at the moment of escalation
// rather than at the streak start. That is deliberate: it dates when the
// failure started being treated as unrecoverable, which is what the park
// window measures, and it puts a full second day between "this is no longer
// transient" and "stop retrying" — 48 h of unbroken failure before an
// integration disables itself over a verdict the wire never actually gave.
const isPersistent = effectiveKind === "persistent";
let persistentFailureStartedAt: Date | null =
existing?.persistentFailureStartedAt ?? null;
if (isPersistent && persistentStreakBefore === 0) {
persistentFailureStartedAt = now;
}
// Park decision: a persistent failure whose streak has exceeded the
// 24h window flips the state to `parked`. This is sticky — once
// parked, the row stays parked until either a success arrives
// (unlikely, since the sync entry-point short-circuits) or the
// user calls `resumeIntegrationFromPark` via the API.
const persistentStreakAgeMs =
isPersistent && persistentFailureStartedAt
? now.getTime() - persistentFailureStartedAt.getTime()
: 0;
const shouldPark =
isPersistent && persistentStreakAgeMs > PARK_PERSISTENT_FAILURE_AFTER_MS;
// State mapping:
// reauth_required → error_reauth (sync entry-point short-circuits)
// transient → error_transient (next sync still runs)
// persistent → error_transient unless we just crossed the
// 24h park threshold, in which case → parked
// (sync entry-point short-circuits, audit detail
// carries the explicit kind so operations can
// grep for contract-bug bursts)
const newState: IntegrationState = shouldPark
? "parked"
: effectiveKind === "reauth_required"
? "error_reauth"
: "error_transient";
const row = await prisma.integrationStatus.upsert({
where: { userId_integration: { userId, integration } },
create: {
userId,
integration,
state: newState,
lastAttemptAt: now,
lastError: encryptedError,
consecutiveFailuresByKind: buckets,
persistentFailureStartedAt: isPersistent ? now : null,
failingSinceAt,
failingLegs,
},
update: {
state: newState,
lastAttemptAt: now,
lastError: encryptedError,
consecutiveFailuresByKind: buckets,
persistentFailureStartedAt,
failingSinceAt,
failingLegs,
},
});
await appendRecentCall(userId, integration, {
endpoint: leg,
status: effectiveKind,
at: now,
errorCode: errorCode ?? null,
});
// One signal per failed sync, not an audit row. The audit log is the
// trail of what a person or an admin did to an account; a provider that
// answered 503 is neither, and one row per failed sync drowned that trail
// (and kept the provider's message there in plain text, while the status
// row seals it). The signal carries the same counts to the log store, and
// the status row keeps the current streak and the sealed message.
//
// `attempt_number` is the sum of the buckets, the running total the
// dropped `consecutiveFailures` column carried; the alert below uses the
// largest single bucket (docs/ops/attempt-number-semantics.md).
const bucketTotal =
buckets.transient + buckets.reauth_required + buckets.persistent;
emitSignal({
action: "integrations.sync.failed",
level: "warn",
meta: {
integration,
kind: effectiveKind,
// Non-null only when the clock overrode the classifier: "the upstream
// said this was permanent" against "a day of retrying concluded it".
escalated_from: escalatedFromTransient ? kind : null,
failing_leg: leg,
// Every leg the row holds an error for, to reconstruct an overlapping
// outage.
failing_legs: failingLegs,
failing_since_at: failingSinceAt.toISOString(),
streak_age_ms: streakAgeMs,
error_code: errorCode ?? null,
attempt_number: bucketTotal,
bucket_count: buckets[effectiveKind],
state: newState,
},
});
// Persistent-failure alerting. Only trip when:
// 1. We're at or above the threshold AFTER this failure.
// 2. We haven't paged on this streak in the last 24h.
//
// Both conditions matter: (1) prevents premature paging, (2)
// prevents loops where a flapping integration that fails once an
// hour pages every hour.
//
// v1.4.43 — the threshold check reads `Math.max(...buckets)` so
// a row with a 3-deep persistent streak still pages even when the
// transient bucket sat at 0.
// v1.4.47 — the legacy `consecutiveFailures` column was dropped
// (migration 0077); the bucket max is now the sole alert signal.
const threshold = getPersistentFailureThreshold();
const alertSignal = Math.max(...Object.values(buckets));
if (alertSignal >= threshold) {
const previouslyAlerted = row.alertedAt != null;
// The watchdog may have announced this connection already (it pages a
// day-long provider error streak, a reauth or a silence on its own). Its
// open incident covers this streak too, and its recovery note ends it;
// a second page about the same trouble would only repeat it.
const watchdogAnnounced =
!previouslyAlerted &&
!!(await prisma.integrationWatch.findFirst({
where: {
userId,
integration,
incidentReason: { not: null },
alertedAt: { not: null },
},
select: { id: true },
}));
if (watchdogAnnounced) {
await prisma.integrationStatus.update({
where: { userId_integration: { userId, integration } },
data: { alertedAt: now },
});
annotate({
action: {
name: "integrations.alert.covered_by_watchdog",
details: { integration },
},
});
} else if (!previouslyAlerted) {
const outcome = await maybeAlertAdmins({
userId,
integration,
kind: effectiveKind,
message,
errorCode,
consecutiveFailures: alertSignal,
}).catch((err) => {
getEvent()?.addWarning(
`Admin alert dispatch failed: ${err instanceof Error ? err.message : String(err)}`,
);
return null;
});
if (outcome?.channelsAttempted) {
// An attempted channel owns its own retry/cooldown policy. Stamp the
// alert window even when every attempt failed so a backend outage
// cannot turn this failure streak into an hourly notification flood.
await prisma.integrationStatus.update({
where: { userId_integration: { userId, integration } },
data: { alertedAt: now },
});
} else if (outcome) {
// No recipient channel accepted an attempt. Leaving alertedAt empty
// keeps the missing delivery visible instead of recording a page that
// never left the process.
getEvent()?.addWarning(
`No admin notification channel available for ${integration} failure alert`,
);
}
}
}
// Park-event audit row — written once per transition into the
// `parked` state so the operations trail shows "this integration
// disabled itself after 24h of persistent failures" without the
// operator having to correlate the sync.failed rows by timestamp.
if (shouldPark && existing?.persistentFailureStartedAt) {
await auditLog("integrations.parked", {
userId,
details: {
integration,
reason: "persistent_24h",
persistentFailureStartedAt:
existing.persistentFailureStartedAt.toISOString(),
persistentStreakAgeMs,
// The whole streak, which for an escalated transient is a full day
// longer than the persistent window above.
failingSinceAt: failingSinceAt.toISOString(),
streakAgeMs,
errorCode: errorCode ?? null,
message,
},
});
}
}
/**
* v1.4.43 — clear a `parked` integration so the next scheduled
* sync runs again. Used by `/api/integrations/withings/resume` and
* the OAuth-callback path. The state moves to `connected`; all
* per-kind buckets reset to zero; the persistent-streak anchor
* clears; the alert window resets so the next genuine 3-strike burst
* still pages admins.
*
* Idempotent: calling against a connected row is a no-op (same
* post-state, no audit row). Calling against any non-parked error
* state also clears the row — the resume CTA is the universal
* "unstick this integration" button.
*/
export async function resumeIntegrationFromPark(
userId: string,
integration: IntegrationKey,
): Promise<{ wasParked: boolean }> {
const existing = await prisma.integrationStatus.findUnique({
where: { userId_integration: { userId, integration } },
select: { state: true },
});
const wasParked = existing?.state === "parked";
await prisma.integrationStatus.upsert({
where: { userId_integration: { userId, integration } },
create: {
userId,
integration,
state: "connected",
consecutiveFailuresByKind: zeroBuckets(),
},
update: {
state: "connected",
lastError: null,
consecutiveFailuresByKind: zeroBuckets(),
persistentFailureStartedAt: null,
failingSinceAt: null,
failingLegs: [],
alertedAt: null,
},
});
if (wasParked) {
await auditLog("integrations.resumed", {
userId,
details: { integration, source: "user_resume" },
});
}
return { wasParked };
}
/**
* Park a connection at `error_reauth` from a deliberate scope-skip
* short-circuit. Unlike `recordSyncFailure`, this helper:
*
* 1. does NOT increment the per-kind failure buckets.
* 2. does NOT emit the `integrations.sync.failed` signal of
* `recordSyncFailure`. A standalone `integrations.reauth_required`
* row is written instead so the operations trail still shows the
* park event.
* 3. does NOT enter the 3-strike alert ladder — no admin Telegram
* page fires.
*
* Idempotent: a second call for the same scope-skip leaves the row at
* `error_reauth` with the same encrypted message and the same bucket
* values (no increment). Use it from sync routines that have detected a
* deliberate, structural scope gap (e.g. legacy Withings connection
* missing `user.activity`). The defence-in-depth catch-block path
* stays on `recordSyncFailure` because a 403 reaching the catch is
* genuinely unexpected once the scope-skip lands.
*
* The audit-row-once semantics mean a row is only written if the row
* is not already parked at `error_reauth` with the same `lastError`.
* Re-parking the same scope-skip is a no-op for the audit log.
*/
export async function parkIntegrationAtReauth(opts: {
userId: string;
integration: IntegrationKey;
message: string;
errorCode: string;
/** The leg whose scope gap forced the park, for multi-leg providers. */
leg?: string;
}): Promise<void> {
const { userId, integration, message, errorCode } = opts;
const leg = opts.leg ?? null;
const now = new Date();
const encryptedError = safeEncryptError(message);
// Idempotency probe: re-parking the same scope-skip should NOT emit
// another audit row. We read the current row first; only write the
// audit log when the state or error changes.
const existing = await prisma.integrationStatus.findUnique({
where: { userId_integration: { userId, integration } },
select: {
state: true,
lastError: true,
failingSinceAt: true,
failingLegs: true,
},
});
const isFreshPark =
existing?.state !== "error_reauth" || existing.lastError !== encryptedError;
// Same set semantics as `recordSyncFailure`: a scope-skip park adds its leg
// to whatever the row already holds rather than replacing it, so a sibling
// leg's independent failure survives the park.
const existingLegs = readFailingLegs(existing?.failingLegs);
const failingLegs =
leg !== null ? withLegRecorded(existingLegs, leg) : existingLegs;
await prisma.integrationStatus.upsert({
where: { userId_integration: { userId, integration } },
create: {
userId,
integration,
state: "error_reauth",
lastError: encryptedError,
lastAttemptAt: now,
// First-ever row for this (user, integration) — buckets stay at
// zero so a later genuine transient burst still has the full
// 3-strike runway before paging.
consecutiveFailuresByKind: zeroBuckets(),
failingSinceAt: now,
failingLegs,
},
update: {
state: "error_reauth",
lastError: encryptedError,
lastAttemptAt: now,
// Deliberately omit `consecutiveFailuresByKind` — the existing
// bucket values are preserved exactly. This is the whole point
// of the helper.
failingSinceAt: existing?.failingSinceAt ?? now,
failingLegs,
},
});
await appendRecentCall(userId, integration, {
endpoint: leg,
status: "reauth_required",
at: now,
errorCode,
});
if (isFreshPark) {
await auditLog("integrations.reauth_required", {
userId,
details: { integration, message, errorCode, source: "scope_skip" },
});
}
}
/**
* Mark a connection as needing re-auth without recording a fresh
* "attempt". Used by the OAuth/refresh-token flows when they detect
* an `invalid_grant`-style permanent revocation OUTSIDE of a sync —
* e.g. the status endpoint that proactively refreshes tokens.
*/
export async function markReauthRequired(
userId: string,
integration: IntegrationKey,
message: string,
): Promise<void> {
const now = new Date();
const existing = await prisma.integrationStatus.findUnique({
where: { userId_integration: { userId, integration } },
select: { failingSinceAt: true },
});
// v1.4.47 — the legacy `consecutiveFailures` column was dropped
// (migration 0077). Out-of-band reauth detection (proactive token
// refresh) seeds the `reauth_required` bucket at 1 on a first-ever
// row so subsequent reauth detections accumulate against the same
// bucket the alert ladder reads.
await prisma.integrationStatus.upsert({
where: { userId_integration: { userId, integration } },
create: {
userId,
integration,
state: "error_reauth",
lastError: safeEncryptError(message),
consecutiveFailuresByKind: {
transient: 0,
reauth_required: 1,
persistent: 0,
},
lastAttemptAt: now,
failingSinceAt: now,
},
update: {
state: "error_reauth",
lastError: safeEncryptError(message),
// Never re-stamp a streak already running — the anchor dates the
// failure, not the detection.
failingSinceAt: existing?.failingSinceAt ?? now,
},
});
await appendRecentCall(userId, integration, {
endpoint: "token_refresh",
status: "reauth_required",
at: now,
errorCode: null,
});
await auditLog("integrations.reauth_required", {
userId,
details: { integration, message },
});
}
/**
* Reset the row when the user disconnects. We keep the row (so the UI
* can still show a tombstone "disconnected at <time>") but clear the
* error state.
*/
export async function markDisconnected(
userId: string,
integration: IntegrationKey,
): Promise<void> {
await prisma.integrationStatus.upsert({
where: { userId_integration: { userId, integration } },
create: {
userId,
integration,
state: "disconnected",
consecutiveFailuresByKind: zeroBuckets(),
},
update: {
state: "disconnected",
lastError: null,
consecutiveFailuresByKind: zeroBuckets(),
persistentFailureStartedAt: null,
failingSinceAt: null,
failingLegs: [],
alertedAt: null,
},
});
}
/**
* Inverse of markReauthRequired — used when the user successfully
* re-completes the OAuth flow. We don't record a fresh sync (no work
* was done) — just reset the streak so the next sync can run.
*/
export async function markReconnected(
userId: string,
integration: IntegrationKey,
): Promise<void> {
await prisma.integrationStatus.upsert({
where: { userId_integration: { userId, integration } },
create: {
userId,
integration,
state: "connected",
consecutiveFailuresByKind: zeroBuckets(),
},
update: {
state: "connected",
lastError: null,
consecutiveFailuresByKind: zeroBuckets(),
persistentFailureStartedAt: null,
failingSinceAt: null,
failingLegs: [],
alertedAt: null,
},
});
}
// ── helpers ────────────────────────────────────────────────────────
interface RecentCallWrite {
endpoint: string | null;
status: RecentCallStatus;
at: Date;
errorCode: string | null;
}
/**
* Prepend one attempt to the row's `recentCalls` ring and keep the newest
* `RECENT_CALLS_LIMIT`. One statement, no read: the ring is rebuilt inside
* Postgres, so a hot sync path pays one UPDATE rather than a read-modify-write.
*
* The entry is built from closed shapes only. The leg name and error code go
* through the same filters the reader applies (`safeEndpointName`,
* `safeErrorCode`), and the failure message is never part of it, so nothing a
* provider sent can reach the diagnostics a user copies into a bug report.
*
* Fail-soft: the ring is diagnostics. A failed append must never fail or undo
* the ledger write it describes, so the error becomes a wide-event warning.
*/
async function appendRecentCall(
userId: string,
integration: IntegrationKey,
call: RecentCallWrite,
): Promise<void> {
const entry = JSON.stringify({
endpoint: safeEndpointName(call.endpoint),
status: call.status,
at: call.at.toISOString(),
errorCode: safeErrorCode(call.errorCode),
});
try {
// Parameter-bound throughout; the ring is rebuilt newest-first and cut at
// the limit. A row whose column is NULL or not an array starts empty.
await prisma.$executeRaw`
UPDATE "integration_statuses"
SET "recent_calls" = (
SELECT COALESCE(jsonb_agg(e ORDER BY ord), '[]'::jsonb)
FROM (
SELECT e, ord
FROM jsonb_array_elements(
jsonb_build_array(${entry}::jsonb) ||
CASE WHEN jsonb_typeof("recent_calls") = 'array'
THEN "recent_calls" ELSE '[]'::jsonb END
) WITH ORDINALITY AS t(e, ord)
ORDER BY ord
LIMIT ${RECENT_CALLS_LIMIT}
) newest
)
WHERE "user_id" = ${userId} AND "integration" = ${integration}
`;
} catch (err) {
getEvent()?.addWarning(
`IntegrationStatus recent-call append failed: ${err instanceof Error ? err.message : String(err)}`,
);
}
}
/**
* Encrypt with a fallback that swallows crypto-config errors so a
* misconfigured ENCRYPTION_KEY can never break the audit/error path.
* Worst case we store ciphertext with the literal string `"<encrypt
* failed>"` — the row is still useful (state, attempt, counter) and
* the underlying crash gets a Wide-Event warning.
*/
function safeEncryptError(message: string): string {
try {
return encrypt(message.slice(0, 1024));
} catch (err) {
getEvent()?.addWarning(
`IntegrationStatus error-encrypt failed: ${err instanceof Error ? err.message : String(err)}`,
);
return "<encrypt failed>";
}
}
function safeDecryptError(ciphertext: string): string {
if (ciphertext === "<encrypt failed>") return "(error message unavailable)";
try {
return decrypt(ciphertext);
} catch {
return "(error message unavailable)";
}
}
export interface AlertInput {
userId: string;
integration: IntegrationKey;
kind: FailureKind;
message: string;
errorCode: string | undefined;
consecutiveFailures: number;
/** Caller-resolved subject label (usually the user's email). */
subjectLabel?: string;
}
/**
* Pure formatter for the admin-Telegram payload — extracted so we
* can unit-test the message shape without standing up Prisma. Kept
* deterministic on purpose: same input, byte-identical output.
*
* The 280-char trim on the upstream error message protects admins
* from a 4 KB stack trace landing in chat. Telegram's own cap is
* 4096 characters but our envelope (title + summary + action line)
* eats ~150 chars so we keep a comfortable margin.
*/
/**
* Reason + action copy keyed off `FailureKind`. Adding a new failure
* kind is a one-row table edit instead of two more arms in two
* different ternary stacks (the style-guide forbids nested ternaries).
*/
/**
* Display label per `IntegrationKey`. A one-row table edit replaces the
* nested-ternary chain the admin-alert formatter used to carry — adding
* a new integration is a single line here instead of another arm in a
* ternary stack (the style-guide forbids nested ternaries).
*/
const INTEGRATION_LABELS: Record<IntegrationKey, string> = {
withings: "Withings",
whoop: "WHOOP",
fitbit: "Fitbit",
nightscout: "Nightscout",
polar: "Polar",
oura: "Oura",
"google-health": "Google Health",
strava: "Strava",
dawarich: "Dawarich",
};
const FAILURE_KIND_COPY: Record<
FailureKind,
{ reason: string; action: string }
> = {
reauth_required: {
reason: "re-auth required",
action: "ask the user to reconnect the integration.",
},
persistent: {
reason: "persistent error",
action:
"investigate the upstream contract — params/scope/action likely mismatched.",
},
transient: {
reason: "transient error",
action: "investigate the upstream service.",
},
};
/**
* SECURITY INVARIANT (v1.4.43 W13 M-2 — MUST NOT be relaxed):
*
* The body produced here is dispatched to Telegram via
* `dispatchNotification`. `input.message` is upstream-influenced — the
* Withings classifier (`src/lib/withings/client.ts`) builds it as
* `Withings <verb> error: <status> - <json.error>` where `json.error`
* is whatever the upstream API put in the response body. Today that
* lands in Telegram on plain text (no `parseMode`), so the upstream
* string is rendered literally and a malicious / buggy response body
* is inert.
*
* Do NOT flip the Telegram callers downstream to `parseMode: "HTML"`
* or `"MarkdownV2"`. The medication-reminder paths use HTML mode
* because their bodies are server-built from sanitised data only; the
* admin-alert body is NOT sanitised. If HTML / Markdown parsing is
* ever enabled for this payload, escape every interpolated field
* (`input.message`, `subjectLabel`, `errorCode`) at the same time —
* otherwise an upstream-controlled string becomes an HTML / Markdown
* injection vector reaching every admin chat.
*/
export function formatAdminAlertPayload(input: AlertInput): {
title: string;
message: string;
metadata: Record<string, unknown>;
} {
const integrationLabel = INTEGRATION_LABELS[input.integration];
const subjectLabel = input.subjectLabel ?? input.userId;
const { reason: reasonLabel, action: actionLabel } =
FAILURE_KIND_COPY[input.kind];
const codeLabel = input.errorCode ? ` (${input.errorCode})` : "";
const trimmed =
input.message.length > 280
? `${input.message.slice(0, 277)}...`
: input.message;
const title = `${integrationLabel} sync failing for ${subjectLabel}`;
const message =
`${integrationLabel} sync has failed ${input.consecutiveFailures} times in a row for ${subjectLabel}.\n` +
`Last error: ${reasonLabel}${codeLabel} — ${trimmed}\n` +
`Action: ${actionLabel}`;
return {
title,
message,
metadata: {
integration: input.integration,
affectedUserId: input.userId,
consecutiveFailures: input.consecutiveFailures,
errorCode: input.errorCode ?? null,
},
};
}
/**
* Page admins through the existing dispatcher. We do NOT add a new sender,
* channel type, or retry path. The dispatcher is opt-in per event and channel,
* and its outcome is the authority on whether a delivery was attempted.
*
* The notification is sent to each admin user. The returned outcome aggregates
* the dispatcher's real per-admin outcomes so one attempted delivery cannot be
* hidden by another admin having no enabled channel.
*/
async function maybeAlertAdmins(input: AlertInput): Promise<DispatchOutcome> {
const subject = await prisma.user.findUnique({
where: { id: input.userId },
select: { email: true },
});
const payload = formatAdminAlertPayload({
...input,
subjectLabel: subject?.email ?? input.userId,
});
return dispatchAdminSystemAlert({
subjectUserId: input.userId,
title: payload.title,
message: payload.message,
metadata: payload.metadata,
});
}
/**
* Send one `SYSTEM_ALERT` to every admin through the existing dispatcher, the
* operator path the sync ledger and the integration watchdog share. Each admin
* receives it on whatever channels they enabled (Telegram, ntfy, Web Push,
* APNs); nothing here adds a channel.
*
* The returned outcome aggregates the dispatcher's real per-admin outcomes, so
* one attempted delivery cannot be hidden by another admin having no enabled
* channel. A caller stamps its own "already announced" marker only when
* `channelsAttempted > 0`.
*/
export async function dispatchAdminSystemAlert(input: {
/** The account the alert is about; picks the tap target per admin. */
subjectUserId: string;
title: string;
message: string;
metadata: Record<string, unknown>;
}): Promise<DispatchOutcome> {
const admins = await prisma.user.findMany({
where: { role: "ADMIN" },
select: { id: true },
});
if (admins.length === 0) {
return {
dispatched: false,
channelsAttempted: 0,
channelsSucceeded: 0,
};
}
const payload = input;
const outcome: DispatchOutcome = {
dispatched: false,
channelsAttempted: 0,
channelsSucceeded: 0,
};
for (const admin of admins) {
const adminOutcome = await dispatchNotification({
eventType: "SYSTEM_ALERT",
userId: admin.id,
title: payload.title,
message: payload.message,
metadata: {
...payload.metadata,
// Tap target: an admin whose own sync fails fixes it in their own
// integration settings; anyone else's shows on the admin page.
url:
admin.id === input.subjectUserId
? "/settings/integrations"
: "/admin/integrations",
},
});
outcome.dispatched ||= adminOutcome.dispatched;
outcome.channelsAttempted += adminOutcome.channelsAttempted;
outcome.channelsSucceeded += adminOutcome.channelsSucceeded;
}
return outcome;
}