fix(integrations): page only for trouble an operator can act on

The watchdog opened an incident for any degraded state that lasted a
day, so an Apple Health account whose phone app stopped running read as
no_new_data for good and paged. Incidents now open for action_required,
for provider errors and for the watchdog's own unexpected silence; stale
or stalled data alone never pages. The first pass after a deploy, with
no watch row yet, records what stands as already announced instead of
paging every long-broken connection at once.
This commit is contained in:
Marc-André Bombeck committed 2026-10-15 16:38:12 +02:00
1 parent 1c05813b82
commit 3e95755c3b
6 files changed
+198 -9

No files matched your search

@@ -244,6 +244,43 @@ describe("currentIncident", () => {
).toEqual({ reason: "unexpected_silence", since: ago(30) });
});
it("never opens an incident for stale or stalled data on its own, however long it lasts", () => {
// An Apple Health account whose phone app stopped running reads as
// `no_new_data` for good; nothing an operator can do about it.
for (const reason of ["no_new_data", "sync_stalled"] as const) {
expect(
currentIncident({
health: health("degraded", reason, ago(24 * 90)),
silentSince: null,
ledgerAlerted: false,
now: NOW,
}),
).toBeNull();
}
});
it("opens an incident for provider errors that have lasted more than a day", () => {
expect(
currentIncident({
health: health("degraded", "provider_errors", ago(25)),
silentSince: null,
ledgerAlerted: false,
now: NOW,
}),
).toEqual({ reason: "provider_errors", since: ago(25) });
});
it("still opens a silence incident beside stale data", () => {
expect(
currentIncident({
health: health("degraded", "no_new_data", ago(50)),
silentSince: ago(30),
ledgerAlerted: false,
now: NOW,
}),
).toEqual({ reason: "unexpected_silence", since: ago(30) });
});
it("has no incident for a limited connection", () => {
expect(
currentIncident({
+15 -5
View File
@@ -13,10 +13,14 @@
* is watched for errors only.
*
* 2. Is there an incident the operator should hear about?
* Unexpected silence; or a health state of `action_required` /
* `degraded` that has lasted longer than `INCIDENT_AFTER_MS`; or a
* failure streak the sync ledger already paged about (adopted, so the
* operator still gets the one recovery note when it ends).
* Unexpected silence; or an `action_required` state, or a `degraded`
* one because the provider keeps erroring (`provider_errors`), that has
* lasted longer than `INCIDENT_AFTER_MS`; or a failure streak the sync
* ledger already paged about (adopted, so the operator still gets the
* one recovery note when it ends). `no_new_data` and `sync_stalled` on
* their own never page: they mostly mean the owner stopped using the
* source (an Apple Health account whose phone app has not run for a week
* reads as stale for good), which is nothing an operator can fix.
*
* One incident per connection at a time. It is announced once and closed once;
* a change of reason while it is open (silence turning into a reauth) is the
@@ -162,7 +166,13 @@ export function currentIncident(input: {
if (health === null) return null;
const troubled =
health.state === "action_required" || health.state === "degraded";
if (troubled) {
// What pages on its own: the owner has to act, or the provider errors. A
// streak the ledger already paged about is followed whatever it reads as
// now, so its end is still reported.
const pages =
health.state === "action_required" ||
(health.state === "degraded" && health.reason === "provider_errors");
if (troubled && (pages || input.ledgerAlerted)) {
// A provider-side reauth flag (Google Health) can stand without a ledger
// streak to date it; the last success is then when it last worked.
const sinceIso = health.since ?? health.lastSuccessAt;
+26 -3
View File
@@ -50,6 +50,13 @@ export interface WatchdogSummary {
incidentsOpened: number;
incidentsClosed: number;
notified: number;
/**
* The pass found no watch row at all: the first one after the watchdog was
* deployed. Incidents standing at that moment are recorded as already
* announced instead of paged, so a deploy does not page once for every
* connection that has been broken or quiet for months.
*/
firstRun: boolean;
}
const DAY_MS = 24 * 60 * 60 * 1000;
@@ -219,6 +226,7 @@ export async function runIntegrationWatchdog(opts: {
incidentsOpened: 0,
incidentsClosed: 0,
notified: 0,
firstRun: (await prisma.integrationWatch.count()) === 0,
};
const userIds = await listWatchedUserIds();
for (const userId of userIds) {
@@ -357,7 +365,13 @@ async function watchConnection(input: {
summary.incidentsOpened += 1;
incidentReason = transition.reason;
incidentOpenedAt = now;
await announce(transition.reason, transition.since);
if (summary.firstRun) {
// Standing since before the watchdog first looked: recorded as
// announced, not paged (see `WatchdogSummary.firstRun`).
alertedAt = now;
} else {
await announce(transition.reason, transition.since);
}
break;
case "adopt":
// The ledger already paged about this streak; record it as announced so
@@ -405,8 +419,17 @@ async function watchConnection(input: {
break;
}
// Nothing to remember and nothing remembered: no row.
if (row === null && silentSince === null && incidentReason === null) return;
// Nothing to remember and nothing remembered: no row. Except on the first
// pass, which leaves a row for every connection it looked at, so the pass
// after it is not taken for a first one too.
if (
row === null &&
silentSince === null &&
incidentReason === null &&
!(summary.firstRun && fact.connected)
) {
return;
}
await prisma.integrationWatch.upsert({
where: { userId_integration: { userId, integration: key } },
+1
View File
@@ -57,6 +57,7 @@ function handlerFor(mode: WatchdogMode) {
incidents_opened: s.incidentsOpened,
incidents_closed: s.incidentsClosed,
notified: s.notified,
first_run: s.firstRun ? 1 : 0,
});
} catch (err) {
evt.addWarning(`integration watchdog failed: ${err}`);
+2
View File
@@ -113,6 +113,8 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"fetches",
"finalised",
"finalized",
// v1.42.4 — integration watchdog: the pass found no watch row (first run).
"first_run",
"forced",
"generated",
"geo_backfill_carrier_resolved",