fix(backups): give the off-host verdicts slack for a Berlin night that is 25 hours long

The threshold was the schedule period with no budget on top of it. Two things
spend that budget on a host where nothing is wrong. The cron runs 02:30
Europe/Berlin, so on the DST fall-back Sunday the gap between two runs is 25
hours and every account on every host would cross one period on the same
morning. And the ledger instant is when that account's object landed, by
design, so on a cohort walked one account at a time an account reached an hour
later tonight than last night drifts past 24 with two clean runs behind it.

The schedule period and the verdict threshold are now separate numbers. The
period is still what the card quotes; the threshold is the period plus six
hours, applied to both steps, so due still means one run produced nothing and
stale still means two. The fall-back night is pinned in the unit file by its
real dates rather than by an arithmetic 25, and a genuinely missed night at 31
hours still reads due. The runbook says why the slack is there.
This commit is contained in:
Marc-André Bombeck committed 2026-09-10 10:09:59 +02:00
1 parent e64435f6a5
commit cfc19a211c
5 files changed
+97 -17

No files matched your search

@@ -1,12 +1,14 @@
import { describe, expect, it } from "vitest";
import {
classifyOffhostBackup,
OFFHOST_BACKUP_GRACE_HOURS,
OFFHOST_BACKUP_PERIOD_HOURS,
} from "@/lib/jobs/offhost-backup-freshness";
const NOW = new Date("2026-09-10T09:00:00.000Z");
const HOUR = 3_600_000;
const PERIOD = OFFHOST_BACKUP_PERIOD_HOURS * HOUR;
const GRACE = OFFHOST_BACKUP_GRACE_HOURS * HOUR;
/** `age` hours before NOW. */
function ago(ms: number): Date {
@@ -21,27 +23,64 @@ describe("classifyOffhostBackup", () => {
});
});
it("is fresh right up to the one-period boundary and due one millisecond past it", () => {
it("is fresh right up to one period plus the grace and due one millisecond past it", () => {
expect(
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD), now: NOW }).freshness,
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD + GRACE), now: NOW })
.freshness,
).toBe("fresh");
expect(
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD + 1), now: NOW })
.freshness,
classifyOffhostBackup({
lastSuccessAt: ago(PERIOD + GRACE + 1),
now: NOW,
}).freshness,
).toBe("due");
});
it("is due right up to the two-period boundary and stale one millisecond past it", () => {
it("is due right up to two periods plus the grace and stale one millisecond past it", () => {
expect(
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD * 2), now: NOW })
.freshness,
classifyOffhostBackup({
lastSuccessAt: ago(PERIOD * 2 + GRACE),
now: NOW,
}).freshness,
).toBe("due");
expect(
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD * 2 + 1), now: NOW })
.freshness,
classifyOffhostBackup({
lastSuccessAt: ago(PERIOD * 2 + GRACE + 1),
now: NOW,
}).freshness,
).toBe("stale");
});
it("keeps the whole cohort fresh across the Berlin DST fall-back night", () => {
// 2026-10-25 is the fall-back Sunday in Europe/Berlin: the clocks go back
// at 03:00 CEST, so 02:30 local happens 25 hours after the previous
// 02:30 local. Both instants are pinned in UTC because that is what the
// ledger stores, and the test must not depend on the runner's own zone.
const beforeTheChange = new Date("2026-10-24T00:30:00.000Z"); // 02:30 CEST
const afterTheChange = new Date("2026-10-25T01:30:00.000Z"); // 02:30 CET
const gapHours =
(afterTheChange.getTime() - beforeTheChange.getTime()) / HOUR;
expect(gapHours).toBe(25);
// Without slack the whole cohort would read `due` on that one morning,
// on every host, with nothing wrong anywhere.
expect(
classifyOffhostBackup({
lastSuccessAt: beforeTheChange,
now: afterTheChange,
}).freshness,
).toBe("fresh");
});
it("still calls a genuinely missed night due", () => {
// 31 hours: past one period even with six hours of slack, so a run that
// produced nothing for this account still shows.
expect(
classifyOffhostBackup({ lastSuccessAt: ago(31 * HOUR), now: NOW })
.freshness,
).toBe("due");
});
it("reports whole hours of age", () => {
expect(
classifyOffhostBackup({ lastSuccessAt: ago(90 * 60_000), now: NOW }),
@@ -50,7 +89,8 @@ describe("classifyOffhostBackup", () => {
it("honours a caller's own period rather than the nightly default", () => {
// A six-hour schedule: the same 12-hour-old object that is fresh under the
// nightly cron is two periods behind under this one.
// nightly cron is two periods behind under this one. The grace is passed
// as zero so the thresholds under test are the caller's period alone.
const twelveHours = ago(12 * HOUR);
expect(
classifyOffhostBackup({ lastSuccessAt: twelveHours, now: NOW }).freshness,
@@ -60,6 +100,7 @@ describe("classifyOffhostBackup", () => {
lastSuccessAt: twelveHours,
now: NOW,
periodHours: 6,
graceHours: 0,
}).freshness,
).toBe("due");
expect(
@@ -67,6 +108,7 @@ describe("classifyOffhostBackup", () => {
lastSuccessAt: twelveHours,
now: NOW,
periodHours: 5,
graceHours: 0,
}).freshness,
).toBe("stale");
});
+32 -5
View File
@@ -18,8 +18,9 @@
* event as four:
*
* never — the worker has never put an object there for this account.
* fresh — inside one schedule period. This is what every account looks like
* on a host where the cron is doing its job.
* fresh — inside one schedule period plus the grace below. This is what
* every account looks like on a host where the cron is doing its
* job.
* due — past one period, inside two. One night produced nothing: a retry, a
* long run, a restart during the window. Worth showing, not worth
* alarming about.
@@ -29,10 +30,30 @@
/**
* Hours between two scheduled off-host runs. Tracks `OFFHOST_BACKUP_CRON`
* (`30 2 * * *`) in `src/lib/jobs/reminder/register-maintenance.ts` — nightly.
* (`30 2 * * *`) in `src/lib/jobs/reminder/register-maintenance.ts`, which
* every schedule runs in `Europe/Berlin` (`tz` in `registrar-shared.ts`).
* This is the number the card quotes — the schedule's own period, not the
* threshold a verdict is measured against.
*/
export const OFFHOST_BACKUP_PERIOD_HOURS = 24;
/**
* Slack on top of one period, before a verdict moves.
*
* Two things make a healthy host miss a bare 24 hours. The schedule is
* Berlin-local, so on the DST fall-back night the gap between two 02:30 runs
* is 25 hours and every account on every host would cross one period on the
* same morning. And the ledger instant is when THAT account's object landed,
* deliberately: on a cohort walked one account at a time, an account reached
* at 05:30 one night and 06:30 the next is 25 hours old with two entirely
* successful runs behind it.
*
* Six hours is the budget for both. A card whose argument is that a count
* trains an operator to stop reading it cannot afford an annual cohort-wide
* false `due`.
*/
export const OFFHOST_BACKUP_GRACE_HOURS = 6;
export type OffhostBackupFreshness = "never" | "fresh" | "due" | "stale";
export interface OffhostBackupVerdict {
@@ -46,12 +67,15 @@ export function classifyOffhostBackup(args: {
now: Date;
/** Defaults to the nightly schedule. Tests pass their own. */
periodHours?: number;
/** Defaults to `OFFHOST_BACKUP_GRACE_HOURS`. Tests pass their own. */
graceHours?: number;
}): OffhostBackupVerdict {
const { lastSuccessAt, now } = args;
if (lastSuccessAt === null) return { freshness: "never", ageHours: null };
const periodMs =
(args.periodHours ?? OFFHOST_BACKUP_PERIOD_HOURS) * 3_600_000;
const graceMs = (args.graceHours ?? OFFHOST_BACKUP_GRACE_HOURS) * 3_600_000;
const ageMs = now.getTime() - lastSuccessAt.getTime();
// A clock that ran backwards between the upload and this read reports a
// negative age. That is a host-clock problem, not a backup problem, and
@@ -59,7 +83,10 @@ export function classifyOffhostBackup(args: {
// thing.
const ageHours = Math.max(0, Math.floor(ageMs / 3_600_000));
if (ageMs <= periodMs) return { freshness: "fresh", ageHours };
if (ageMs <= periodMs * 2) return { freshness: "due", ageHours };
// The grace rides on each threshold rather than on the period, so `due`
// still means "one scheduled run produced nothing" and `stale` still means
// two — the slack only keeps a long night from being read as a missed one.
if (ageMs <= periodMs + graceMs) return { freshness: "fresh", ageHours };
if (ageMs <= periodMs * 2 + graceMs) return { freshness: "due", ageHours };
return { freshness: "stale", ageHours };
}