mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 13:22:14 +00:00
fix(integrations): send no recovery for an incident the first run recorded silently
The watchdog's first pass after a deploy records standing incidents as announced without paging. Their end then sent a recovery note for something the operator was never told about. Such incidents now carry the epoch as their announcement time, which no real announcement can have, and close without a note.
This commit is contained in:
1 parent
6dd71f0103
commit
346825149d
4 files changed
+61
-3
No files matched your search
@@ -4,6 +4,7 @@ import {
|
||||
NORMAL_DELIVERY_MIN_DAYS,
|
||||
SOURCE_PROFILES,
|
||||
classifySilence,
|
||||
FIRST_RUN_ALERTED_AT,
|
||||
currentIncident,
|
||||
decideWatchTransition,
|
||||
type WatchRow,
|
||||
@@ -293,6 +294,27 @@ describe("currentIncident", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("decideWatchTransition: an incident recorded on the first run", () => {
|
||||
it("closes without a recovery note, since none was ever sent", () => {
|
||||
expect(
|
||||
decideWatchTransition({
|
||||
row: {
|
||||
silentSince: ago(30),
|
||||
incidentReason: "unexpected_silence",
|
||||
incidentOpenedAt: ago(2),
|
||||
alertedAt: FIRST_RUN_ALERTED_AT,
|
||||
},
|
||||
incident: null,
|
||||
ledgerAlerted: false,
|
||||
}),
|
||||
).toEqual({
|
||||
kind: "close",
|
||||
notify: false,
|
||||
previousReason: "unexpected_silence",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("decideWatchTransition", () => {
|
||||
const incident = { reason: "unexpected_silence" as const, since: ago(30) };
|
||||
|
||||
|
||||
@@ -189,6 +189,22 @@ export function currentIncident(input: {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The `alertedAt` an incident gets when the watchdog's first pass after a
|
||||
* deploy found it already standing: counted as announced, so it is never
|
||||
* paged, but never actually sent either, so its end sends no recovery note.
|
||||
* The epoch, because no real announcement can carry it, and it keeps the
|
||||
* marker inside the existing row.
|
||||
*/
|
||||
export const FIRST_RUN_ALERTED_AT = new Date(0);
|
||||
|
||||
/** Whether an `alertedAt` records a message that was really sent. */
|
||||
export function wasAnnounced(alertedAt: Date | null | undefined): boolean {
|
||||
return (
|
||||
alertedAt != null && alertedAt.getTime() !== FIRST_RUN_ALERTED_AT.getTime()
|
||||
);
|
||||
}
|
||||
|
||||
export type WatchTransition =
|
||||
/** No incident before, none now. */
|
||||
| { kind: "none" }
|
||||
@@ -220,7 +236,7 @@ export function decideWatchTransition(input: {
|
||||
if (input.incident === null) {
|
||||
return {
|
||||
kind: "close",
|
||||
notify: input.row?.alertedAt != null,
|
||||
notify: wasAnnounced(input.row?.alertedAt),
|
||||
previousReason: open,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
} from "./connection-health-load";
|
||||
import { INTEGRATION_MEASUREMENT_SOURCE } from "./metric-freshness";
|
||||
import {
|
||||
FIRST_RUN_ALERTED_AT,
|
||||
HISTORY_DAYS,
|
||||
SOURCE_PROFILES,
|
||||
classifySilence,
|
||||
@@ -529,8 +530,9 @@ async function watchConnection(input: {
|
||||
incidentOpenedAt = now;
|
||||
if (summary.firstRun) {
|
||||
// Standing since before the watchdog first looked: recorded as
|
||||
// announced, not paged (see `WatchdogSummary.firstRun`).
|
||||
alertedAt = now;
|
||||
// announced, not paged (see `WatchdogSummary.firstRun`), and marked
|
||||
// so its end sends no recovery note either.
|
||||
alertedAt = FIRST_RUN_ALERTED_AT;
|
||||
} else {
|
||||
announce(transition.reason, transition.since);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user