mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-07 04:12:04 +00:00
fix(integrations): send no recovery for an incident the first run recorded silently
The watchdog's first pass after a deploy records standing incidents as announced without paging. Their end then sent a recovery note for something the operator was never told about. Such incidents now carry the epoch as their announcement time, which no real announcement can have, and close without a note.
This commit is contained in:
1 parent
6dd71f0103
commit
346825149d
4 files changed
+61
-3
No files matched your search
@@ -4,6 +4,7 @@ import {
|
||||
NORMAL_DELIVERY_MIN_DAYS,
|
||||
SOURCE_PROFILES,
|
||||
classifySilence,
|
||||
FIRST_RUN_ALERTED_AT,
|
||||
currentIncident,
|
||||
decideWatchTransition,
|
||||
type WatchRow,
|
||||
@@ -293,6 +294,27 @@ describe("currentIncident", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("decideWatchTransition: an incident recorded on the first run", () => {
|
||||
it("closes without a recovery note, since none was ever sent", () => {
|
||||
expect(
|
||||
decideWatchTransition({
|
||||
row: {
|
||||
silentSince: ago(30),
|
||||
incidentReason: "unexpected_silence",
|
||||
incidentOpenedAt: ago(2),
|
||||
alertedAt: FIRST_RUN_ALERTED_AT,
|
||||
},
|
||||
incident: null,
|
||||
ledgerAlerted: false,
|
||||
}),
|
||||
).toEqual({
|
||||
kind: "close",
|
||||
notify: false,
|
||||
previousReason: "unexpected_silence",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("decideWatchTransition", () => {
|
||||
const incident = { reason: "unexpected_silence" as const, since: ago(30) };
|
||||
|
||||
|
||||
@@ -189,6 +189,22 @@ export function currentIncident(input: {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The `alertedAt` an incident gets when the watchdog's first pass after a
|
||||
* deploy found it already standing: counted as announced, so it is never
|
||||
* paged, but never actually sent either, so its end sends no recovery note.
|
||||
* The epoch, because no real announcement can carry it, and it keeps the
|
||||
* marker inside the existing row.
|
||||
*/
|
||||
export const FIRST_RUN_ALERTED_AT = new Date(0);
|
||||
|
||||
/** Whether an `alertedAt` records a message that was really sent. */
|
||||
export function wasAnnounced(alertedAt: Date | null | undefined): boolean {
|
||||
return (
|
||||
alertedAt != null && alertedAt.getTime() !== FIRST_RUN_ALERTED_AT.getTime()
|
||||
);
|
||||
}
|
||||
|
||||
export type WatchTransition =
|
||||
/** No incident before, none now. */
|
||||
| { kind: "none" }
|
||||
@@ -220,7 +236,7 @@ export function decideWatchTransition(input: {
|
||||
if (input.incident === null) {
|
||||
return {
|
||||
kind: "close",
|
||||
notify: input.row?.alertedAt != null,
|
||||
notify: wasAnnounced(input.row?.alertedAt),
|
||||
previousReason: open,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
} from "./connection-health-load";
|
||||
import { INTEGRATION_MEASUREMENT_SOURCE } from "./metric-freshness";
|
||||
import {
|
||||
FIRST_RUN_ALERTED_AT,
|
||||
HISTORY_DAYS,
|
||||
SOURCE_PROFILES,
|
||||
classifySilence,
|
||||
@@ -529,8 +530,9 @@ async function watchConnection(input: {
|
||||
incidentOpenedAt = now;
|
||||
if (summary.firstRun) {
|
||||
// Standing since before the watchdog first looked: recorded as
|
||||
// announced, not paged (see `WatchdogSummary.firstRun`).
|
||||
alertedAt = now;
|
||||
// announced, not paged (see `WatchdogSummary.firstRun`), and marked
|
||||
// so its end sends no recovery note either.
|
||||
alertedAt = FIRST_RUN_ALERTED_AT;
|
||||
} else {
|
||||
announce(transition.reason, transition.since);
|
||||
}
|
||||
|
||||
@@ -333,6 +333,24 @@ describe("what pages and what does not", () => {
|
||||
expect(second.firstRun).toBe(false);
|
||||
expect(dispatchNotification).not.toHaveBeenCalled();
|
||||
|
||||
// It ends without a recovery note: nothing was ever sent about it.
|
||||
await prisma().measurement.create({
|
||||
data: {
|
||||
userId: USER,
|
||||
type: "RESTING_HEART_RATE",
|
||||
value: 61,
|
||||
unit: "bpm",
|
||||
source: "OURA",
|
||||
measuredAt: new Date(later.getTime() - 10 * 60 * 1000),
|
||||
},
|
||||
});
|
||||
const recovered = await runIntegrationWatchdog({
|
||||
mode: "hourly",
|
||||
now: new Date(later.getTime() + HOUR),
|
||||
});
|
||||
expect(recovered.incidentsClosed).toBe(1);
|
||||
expect(dispatchNotification).not.toHaveBeenCalled();
|
||||
|
||||
// A new incident after the first pass pages as usual.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await recordSyncFailure({
|
||||
|
||||
Reference in new issue
Block a user