fix(restore-drill): a drill that read nothing back is no pass

A drill that stopped on its time budget before the first object recorded
ok with zero accounts checked, and the readiness tile said passed. Such a
run now records its time and counts and no verdict, so the last pass or
failure stands, and readiness calls it attention (nothing_checked) in every
locale.

The upsert that keeps a pass and a failure apart was stubbed in the drill's
tests and tested nowhere; it now has an integration test read back through
the readiness rule. The drill test that claimed to keep a failed verdict
when keeping it fails is named for what it checks.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 17:26:40 +01:00
1 parent a857fd0264
commit 6051488b65
15 files changed
+175 -18

No files matched your search

+2 -1
View File
@@ -8727,7 +8727,8 @@
"not_run_yet": "Die monatliche Wiederherstellungsprobe ist noch nicht gelaufen.",
"failed": "Die letzte Wiederherstellungsprobe ist fehlgeschlagen: {failed} von {checked} Kopien ließen sich nicht zurücklesen.",
"failed_run": "Die letzte Wiederherstellungsprobe konnte nicht zu Ende laufen.",
"passed": "Die letzte Wiederherstellungsprobe hat {checked} Kopien zurückgelesen."
"passed": "Die letzte Wiederherstellungsprobe hat {checked} Kopien zurückgelesen.",
"nothing_checked": "Die letzte Wiederherstellungsprobe hat ihr Zeitlimit erreicht, bevor sie eine einzige Kopie zurückgelesen hat, und hat damit nichts belegt."
},
"offhost_setup": {
"configured": "Kopien verlassen diesen Server jede Nacht.",
+2 -1
View File
@@ -8727,7 +8727,8 @@
"not_run_yet": "The monthly restore drill has not run yet.",
"failed": "The last restore drill failed: {failed} of {checked} copies could not be read back.",
"failed_run": "The last restore drill could not finish.",
"passed": "The last restore drill read back {checked} copies."
"passed": "The last restore drill read back {checked} copies.",
"nothing_checked": "The last restore drill stopped on its time limit before it read back a single copy, so it proved nothing."
},
"offhost_setup": {
"configured": "Copies leave this server every night.",
+2 -1
View File
@@ -8727,7 +8727,8 @@
"not_run_yet": "El simulacro de restauración mensual aún no se ha ejecutado.",
"failed": "El último simulacro de restauración falló: no se pudieron volver a leer {failed} de {checked} copias.",
"failed_run": "El último simulacro de restauración no pudo terminar.",
"passed": "El último simulacro de restauración volvió a leer {checked} copias."
"passed": "El último simulacro de restauración volvió a leer {checked} copias.",
"nothing_checked": "El último simulacro de restauración se detuvo por su límite de tiempo antes de leer una sola copia, así que no demostró nada."
},
"offhost_setup": {
"configured": "Las copias salen de este servidor cada noche.",
+2 -1
View File
@@ -8727,7 +8727,8 @@
"not_run_yet": "Le test de restauration mensuel n'a pas encore été lancé.",
"failed": "Le dernier test de restauration a échoué : {failed} copies sur {checked} n'ont pas pu être relues.",
"failed_run": "Le dernier test de restauration n'a pas pu aller jusqu'au bout.",
"passed": "Le dernier test de restauration a relu {checked} copies."
"passed": "Le dernier test de restauration a relu {checked} copies.",
"nothing_checked": "Le dernier test de restauration s'est arrêté sur sa limite de temps avant de relire une seule copie : il n'a rien prouvé."
},
"offhost_setup": {
"configured": "Les copies quittent ce serveur chaque nuit.",
+2 -1
View File
@@ -8727,7 +8727,8 @@
"not_run_yet": "La prova di ripristino mensile non è ancora stata eseguita.",
"failed": "L'ultima prova di ripristino non è riuscita: {failed} copie su {checked} non si sono potute rileggere.",
"failed_run": "L'ultima prova di ripristino non è riuscita ad arrivare in fondo.",
"passed": "L'ultima prova di ripristino ha riletto {checked} copie."
"passed": "L'ultima prova di ripristino ha riletto {checked} copie.",
"nothing_checked": "L'ultima prova di ripristino si è fermata al suo limite di tempo prima di rileggere una sola copia, quindi non ha dimostrato nulla."
},
"offhost_setup": {
"configured": "Le copie lasciano questo server ogni notte.",
+2 -1
View File
@@ -8727,7 +8727,8 @@
"not_run_yet": "월간 복원 훈련이 아직 실행되지 않았습니다.",
"failed": "마지막 복원 훈련이 실패했습니다: 사본 {checked}개 중 {failed}개를 다시 읽지 못했습니다.",
"failed_run": "마지막 복원 훈련을 끝까지 마치지 못했습니다.",
"passed": "마지막 복원 훈련에서 사본 {checked}개를 다시 읽었습니다."
"passed": "마지막 복원 훈련에서 사본 {checked}개를 다시 읽었습니다.",
"nothing_checked": "마지막 복원 훈련이 사본을 하나도 다시 읽기 전에 시간 제한으로 멈췄기 때문에 아무것도 확인하지 못했습니다."
},
"offhost_setup": {
"configured": "사본이 매일 밤 이 서버 밖으로 나갑니다.",
+2 -1
View File
@@ -8727,7 +8727,8 @@
"not_run_yet": "De maandelijkse hersteltest is nog niet uitgevoerd.",
"failed": "De laatste hersteltest is mislukt: {failed} van {checked} kopieën konden niet worden teruggelezen.",
"failed_run": "De laatste hersteltest kon niet worden afgerond.",
"passed": "De laatste hersteltest heeft {checked} kopieën teruggelezen."
"passed": "De laatste hersteltest heeft {checked} kopieën teruggelezen.",
"nothing_checked": "De laatste hersteltest stopte op zijn tijdslimiet voordat hij ook maar één kopie had teruggelezen, en heeft dus niets aangetoond."
},
"offhost_setup": {
"configured": "Kopieën verlaten deze server elke nacht.",
+2 -1
View File
@@ -8727,7 +8727,8 @@
"not_run_yet": "Comiesięczna próba przywracania jeszcze się nie odbyła.",
"failed": "Ostatnia próba przywracania nie powiodła się: nie udało się odczytać {failed} z {checked} kopii.",
"failed_run": "Ostatnia próba przywracania nie dobiegła końca.",
"passed": "Ostatnia próba przywracania odczytała kopie: {checked}."
"passed": "Ostatnia próba przywracania odczytała kopie: {checked}.",
"nothing_checked": "Ostatnia próba przywracania zatrzymała się na limicie czasu, zanim odczytała choć jedną kopię, więc niczego nie potwierdziła."
},
"offhost_setup": {
"configured": "Kopie opuszczają ten serwer każdej nocy.",
+2 -1
View File
@@ -8727,7 +8727,8 @@
"not_run_yet": "O teste mensal de restauração ainda não foi executado.",
"failed": "O último teste de restauração falhou: {failed} de {checked} cópias não puderam ser lidas de volta.",
"failed_run": "O último teste de restauração não pôde ser concluído.",
"passed": "O último teste de restauração leu de volta {checked} cópias."
"passed": "O último teste de restauração leu de volta {checked} cópias.",
"nothing_checked": "O último teste de restauração parou no limite de tempo antes de ler de volta uma única cópia, então não comprovou nada."
},
"offhost_setup": {
"configured": "As cópias saem deste servidor todas as noites.",
+18 -1
View File
@@ -284,7 +284,24 @@ describe("handleRestoreDrill", () => {
);
});
it("keeps a failed drill's verdict even when keeping it fails", async () => {
it("records no pass for a run that stopped before its first object", async () => {
stubConfigEnv();
const s3 = makeS3Mock({
"2026-06-01/user-aaa.json.enc": backupObject({ userId: "user-aaa" }),
});
const offhost = await import("../offhost-backup");
vi.spyOn(offhost, "getS3Client").mockResolvedValue(s3 as never);
// The job's budget is already spent: pg-boss aborted it.
await handleRestoreDrill([
{ signal: { aborted: true }, expireInSeconds: 900 } as never,
]);
expect(recordRestoreDrillOutcome).toHaveBeenCalledTimes(1);
expect(vi.mocked(recordRestoreDrillOutcome).mock.calls[0][0]).toMatchObject(
{ ok: null, accountsChecked: 0 },
);
});
it("still reports the drill's own failure when its verdict cannot be kept", async () => {
stubConfigEnv();
vi.stubEnv("BACKUP_ENCRYPTION_KEY", "not-a-key");
vi.mocked(recordRestoreDrillOutcome).mockRejectedValueOnce(
+16 -7
View File
@@ -19,7 +19,12 @@ import { redactSecrets } from "@/lib/logging/redact";
const MAX_REASON_CHARS = 300;
export interface RestoreDrillResult {
ok: boolean;
/**
* The verdict; null for a run that checked nothing (it stopped on its time
* budget before the first object), which is no verdict at all: the run and
* its counts are kept, the last pass or failure stands.
*/
ok: boolean | null;
accountsChecked: number;
accountsFailed: number;
/** Why it failed; ignored when `ok`. */
@@ -30,17 +35,21 @@ export async function recordRestoreDrillOutcome(
result: RestoreDrillResult,
at: Date = new Date(),
): Promise<void> {
const reason = result.ok
? null
: (() => {
const reason =
result.ok !== false
? null
: (() => {
const text = redactSecrets((result.reason ?? "").trim());
return text.length > MAX_REASON_CHARS
? `${text.slice(0, MAX_REASON_CHARS)}…`
: text || null;
})();
const verdict = result.ok
? { lastSuccessAt: at }
: { lastFailureAt: at, lastFailureReason: reason };
const verdict =
result.ok === null
? {}
: result.ok
? { lastSuccessAt: at }
: { lastFailureAt: at, lastFailureReason: reason };
const counts = {
lastRunAt: at,
accountsChecked: result.accountsChecked,
+3 -1
View File
@@ -346,7 +346,9 @@ export async function handleRestoreDrill(
);
}
await keepOutcome(evt, {
ok: report.failed.length === 0,
// A run that stopped before its first object read nothing back: it
// is no pass, and it records none.
ok: report.accounts.length === 0 ? null : report.failed.length === 0,
accountsChecked: report.accounts.length,
accountsFailed: report.failed.length,
reason:
@@ -538,6 +538,19 @@ describe("restore drill", () => {
).toBe("failed_run");
});
// A run that stopped on its budget before the first object read nothing
// back. It used to be recorded as a pass and the tile said "passed".
it("does not call a run that read nothing back passed, whatever passed before", () => {
expect(
drill({
...run,
lastRunAt: ago(DAY),
lastSuccessAt: ago(40 * DAY),
accountsChecked: 0,
}),
).toMatchObject({ state: "attention", reason: "nothing_checked" });
});
it("does not apply without a bucket, and is ok before its first run", () => {
expect(drill(null, false).state).toBe("inactive");
expect(drill(null)).toMatchObject({ state: "ok", reason: "not_run_yet" });
+5
View File
@@ -420,6 +420,11 @@ export function evaluateRestoreDrill(
drill.lastFailureReason,
);
}
// The latest run read nothing back (it stopped on its budget before the
// first object): no proof that a copy restores, whatever passed before.
if (drill.accountsChecked === 0) {
return signal(id, "attention", "nothing_checked", drill.lastRunAt);
}
return signal(id, "ok", "passed", drill.lastSuccessAt ?? drill.lastRunAt, {
checked: drill.accountsChecked,
});
@@ -0,0 +1,102 @@
/**
* The restore drill's kept verdict, against Postgres, read back the way the
* readiness tile reads it. The unit tests of the drill stub this write; here
* it is the subject.
*
* The rule: a failure stays the verdict until a later drill passes, with no
* time window, and a run that checked nothing changes no verdict. Mutations
* that must turn this red: let a pass clear `lastFailureAt`, or let a failure
* clear `lastSuccessAt` (the next pass would then read as recovered from
* nothing); record a run that checked nothing as a pass.
*/
import { beforeEach, describe, expect, it } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import { recordRestoreDrillOutcome } from "@/lib/jobs/restore-drill-outcome";
import { evaluateRestoreDrill } from "@/lib/recovery/readiness-model";
const DAY = 86_400_000;
const T0 = new Date("2026-09-01T04:00:00Z");
const at = (days: number) => new Date(T0.getTime() + days * DAY);
async function verdict() {
const row = await getPrismaClient().restoreDrillOutcome.findUniqueOrThrow({
where: { id: "singleton" },
});
return {
row,
signal: evaluateRestoreDrill({
offhost: { configured: true, databaseLifecycle: null, accounts: [] },
drill: row,
}),
};
}
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
});
describe("the restore drill's kept verdict", () => {
it("keeps a pass and a failure apart, and a failure until a later pass", async () => {
await recordRestoreDrillOutcome(
{ ok: true, accountsChecked: 3, accountsFailed: 0 },
at(0),
);
expect((await verdict()).signal).toMatchObject({
state: "ok",
reason: "passed",
});
await recordRestoreDrillOutcome(
{
ok: false,
accountsChecked: 3,
accountsFailed: 1,
reason: "backup object 2026-09-30/user-x.json.enc does not decrypt",
},
at(30),
);
const failed = await verdict();
expect(failed.row.lastSuccessAt).toEqual(at(0));
expect(failed.row.lastFailureAt).toEqual(at(30));
expect(failed.signal).toMatchObject({
state: "failing",
reason: "failed",
detail: expect.stringMatching(/does not decrypt/),
});
// A run that stopped before its first object is no verdict: the failure
// stands, and the tile does not call it passed.
await recordRestoreDrillOutcome(
{ ok: null, accountsChecked: 0, accountsFailed: 0 },
at(60),
);
const nothing = await verdict();
expect(nothing.row.lastFailureAt).toEqual(at(30));
expect(nothing.row.lastSuccessAt).toEqual(at(0));
expect(nothing.row.lastRunAt).toEqual(at(60));
expect(nothing.signal.state).toBe("failing");
await recordRestoreDrillOutcome(
{ ok: true, accountsChecked: 4, accountsFailed: 0 },
at(90),
);
const recovered = await verdict();
expect(recovered.row.lastFailureAt).toEqual(at(30));
expect(recovered.row.lastFailureReason).toMatch(/does not decrypt/);
expect(recovered.signal).toMatchObject({ state: "ok", reason: "passed" });
});
it("does not record a run that checked nothing as a pass", async () => {
await recordRestoreDrillOutcome(
{ ok: null, accountsChecked: 0, accountsFailed: 0 },
at(0),
);
const { row, signal } = await verdict();
expect(row.lastSuccessAt).toBeNull();
expect(signal).toMatchObject({
state: "attention",
reason: "nothing_checked",
});
});
});