fix(integrations): send no recovery for an incident the first run recorded silently

The watchdog's first pass after a deploy records standing incidents as
announced without paging. Their end then sent a recovery note for
something the operator was never told about. Such incidents now carry
the epoch as their announcement time, which no real announcement can
have, and close without a note.
This commit is contained in:
Marc-André Bombeck committed 2026-10-15 16:42:41 +02:00
1 parent 6dd71f0103
commit 346825149d
4 files changed
+61 -3

No files matched your search

@@ -4,6 +4,7 @@ import {
NORMAL_DELIVERY_MIN_DAYS,
SOURCE_PROFILES,
classifySilence,
FIRST_RUN_ALERTED_AT,
currentIncident,
decideWatchTransition,
type WatchRow,
@@ -293,6 +294,27 @@ describe("currentIncident", () => {
});
});
describe("decideWatchTransition: an incident recorded on the first run", () => {
it("closes without a recovery note, since none was ever sent", () => {
expect(
decideWatchTransition({
row: {
silentSince: ago(30),
incidentReason: "unexpected_silence",
incidentOpenedAt: ago(2),
alertedAt: FIRST_RUN_ALERTED_AT,
},
incident: null,
ledgerAlerted: false,
}),
).toEqual({
kind: "close",
notify: false,
previousReason: "unexpected_silence",
});
});
});
describe("decideWatchTransition", () => {
const incident = { reason: "unexpected_silence" as const, since: ago(30) };
+17 -1
View File
@@ -189,6 +189,22 @@ export function currentIncident(input: {
return null;
}
/**
* The `alertedAt` an incident gets when the watchdog's first pass after a
* deploy found it already standing: counted as announced, so it is never
* paged, but never actually sent either, so its end sends no recovery note.
* The epoch, because no real announcement can carry it, and it keeps the
* marker inside the existing row.
*/
export const FIRST_RUN_ALERTED_AT = new Date(0);
/** Whether an `alertedAt` records a message that was really sent. */
export function wasAnnounced(alertedAt: Date | null | undefined): boolean {
return (
alertedAt != null && alertedAt.getTime() !== FIRST_RUN_ALERTED_AT.getTime()
);
}
export type WatchTransition =
/** No incident before, none now. */
| { kind: "none" }
@@ -220,7 +236,7 @@ export function decideWatchTransition(input: {
if (input.incident === null) {
return {
kind: "close",
notify: input.row?.alertedAt != null,
notify: wasAnnounced(input.row?.alertedAt),
previousReason: open,
};
}
+4 -2
View File
@@ -31,6 +31,7 @@ import {
} from "./connection-health-load";
import { INTEGRATION_MEASUREMENT_SOURCE } from "./metric-freshness";
import {
FIRST_RUN_ALERTED_AT,
HISTORY_DAYS,
SOURCE_PROFILES,
classifySilence,
@@ -529,8 +530,9 @@ async function watchConnection(input: {
incidentOpenedAt = now;
if (summary.firstRun) {
// Standing since before the watchdog first looked: recorded as
// announced, not paged (see `WatchdogSummary.firstRun`).
alertedAt = now;
// announced, not paged (see `WatchdogSummary.firstRun`), and marked
// so its end sends no recovery note either.
alertedAt = FIRST_RUN_ALERTED_AT;
} else {
announce(transition.reason, transition.since);
}
@@ -333,6 +333,24 @@ describe("what pages and what does not", () => {
expect(second.firstRun).toBe(false);
expect(dispatchNotification).not.toHaveBeenCalled();
// It ends without a recovery note: nothing was ever sent about it.
await prisma().measurement.create({
data: {
userId: USER,
type: "RESTING_HEART_RATE",
value: 61,
unit: "bpm",
source: "OURA",
measuredAt: new Date(later.getTime() - 10 * 60 * 1000),
},
});
const recovered = await runIntegrationWatchdog({
mode: "hourly",
now: new Date(later.getTime() + HOUR),
});
expect(recovered.incidentsClosed).toBe(1);
expect(dispatchNotification).not.toHaveBeenCalled();
// A new incident after the first pass pages as usual.
for (let i = 0; i < 3; i++) {
await recordSyncFailure({