diff --git a/messages/de.json b/messages/de.json index 59abc14cf..28db12c84 100644 --- a/messages/de.json +++ b/messages/de.json @@ -8727,7 +8727,8 @@ "not_run_yet": "Die monatliche Wiederherstellungsprobe ist noch nicht gelaufen.", "failed": "Die letzte Wiederherstellungsprobe ist fehlgeschlagen: {failed} von {checked} Kopien ließen sich nicht zurücklesen.", "failed_run": "Die letzte Wiederherstellungsprobe konnte nicht zu Ende laufen.", - "passed": "Die letzte Wiederherstellungsprobe hat {checked} Kopien zurückgelesen." + "passed": "Die letzte Wiederherstellungsprobe hat {checked} Kopien zurückgelesen.", + "nothing_checked": "Die letzte Wiederherstellungsprobe hat ihr Zeitlimit erreicht, bevor sie eine einzige Kopie zurückgelesen hat, und hat damit nichts belegt." }, "offhost_setup": { "configured": "Kopien verlassen diesen Server jede Nacht.", diff --git a/messages/en.json b/messages/en.json index 2d683a8cc..6396403ef 100644 --- a/messages/en.json +++ b/messages/en.json @@ -8727,7 +8727,8 @@ "not_run_yet": "The monthly restore drill has not run yet.", "failed": "The last restore drill failed: {failed} of {checked} copies could not be read back.", "failed_run": "The last restore drill could not finish.", - "passed": "The last restore drill read back {checked} copies." + "passed": "The last restore drill read back {checked} copies.", + "nothing_checked": "The last restore drill stopped on its time limit before it read back a single copy, so it proved nothing." }, "offhost_setup": { "configured": "Copies leave this server every night.", diff --git a/messages/es.json b/messages/es.json index fcdf787e8..37505b5a7 100644 --- a/messages/es.json +++ b/messages/es.json @@ -8727,7 +8727,8 @@ "not_run_yet": "El simulacro de restauración mensual aún no se ha ejecutado.", "failed": "El último simulacro de restauración falló: no se pudieron volver a leer {failed} de {checked} copias.", "failed_run": "El último simulacro de restauración no pudo terminar.", - "passed": "El último simulacro de restauración volvió a leer {checked} copias." + "passed": "El último simulacro de restauración volvió a leer {checked} copias.", + "nothing_checked": "El último simulacro de restauración se detuvo por su límite de tiempo antes de leer una sola copia, así que no demostró nada." }, "offhost_setup": { "configured": "Las copias salen de este servidor cada noche.", diff --git a/messages/fr.json b/messages/fr.json index 5fffc02b5..4e29951c0 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -8727,7 +8727,8 @@ "not_run_yet": "Le test de restauration mensuel n'a pas encore été lancé.", "failed": "Le dernier test de restauration a échoué : {failed} copies sur {checked} n'ont pas pu être relues.", "failed_run": "Le dernier test de restauration n'a pas pu aller jusqu'au bout.", - "passed": "Le dernier test de restauration a relu {checked} copies." + "passed": "Le dernier test de restauration a relu {checked} copies.", + "nothing_checked": "Le dernier test de restauration s'est arrêté sur sa limite de temps avant de relire une seule copie : il n'a rien prouvé." }, "offhost_setup": { "configured": "Les copies quittent ce serveur chaque nuit.", diff --git a/messages/it.json b/messages/it.json index dcb73877f..b8bdcc59e 100644 --- a/messages/it.json +++ b/messages/it.json @@ -8727,7 +8727,8 @@ "not_run_yet": "La prova di ripristino mensile non è ancora stata eseguita.", "failed": "L'ultima prova di ripristino non è riuscita: {failed} copie su {checked} non si sono potute rileggere.", "failed_run": "L'ultima prova di ripristino non è riuscita ad arrivare in fondo.", - "passed": "L'ultima prova di ripristino ha riletto {checked} copie." + "passed": "L'ultima prova di ripristino ha riletto {checked} copie.", + "nothing_checked": "L'ultima prova di ripristino si è fermata al suo limite di tempo prima di rileggere una sola copia, quindi non ha dimostrato nulla." }, "offhost_setup": { "configured": "Le copie lasciano questo server ogni notte.", diff --git a/messages/ko.json b/messages/ko.json index 89dade95b..7948da54d 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -8727,7 +8727,8 @@ "not_run_yet": "월간 복원 훈련이 아직 실행되지 않았습니다.", "failed": "마지막 복원 훈련이 실패했습니다: 사본 {checked}개 중 {failed}개를 다시 읽지 못했습니다.", "failed_run": "마지막 복원 훈련을 끝까지 마치지 못했습니다.", - "passed": "마지막 복원 훈련에서 사본 {checked}개를 다시 읽었습니다." + "passed": "마지막 복원 훈련에서 사본 {checked}개를 다시 읽었습니다.", + "nothing_checked": "마지막 복원 훈련이 사본을 하나도 다시 읽기 전에 시간 제한으로 멈췄기 때문에 아무것도 확인하지 못했습니다." }, "offhost_setup": { "configured": "사본이 매일 밤 이 서버 밖으로 나갑니다.", diff --git a/messages/nl.json b/messages/nl.json index 085c84208..e0f68d755 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -8727,7 +8727,8 @@ "not_run_yet": "De maandelijkse hersteltest is nog niet uitgevoerd.", "failed": "De laatste hersteltest is mislukt: {failed} van {checked} kopieën konden niet worden teruggelezen.", "failed_run": "De laatste hersteltest kon niet worden afgerond.", - "passed": "De laatste hersteltest heeft {checked} kopieën teruggelezen." + "passed": "De laatste hersteltest heeft {checked} kopieën teruggelezen.", + "nothing_checked": "De laatste hersteltest stopte op zijn tijdslimiet voordat hij ook maar één kopie had teruggelezen, en heeft dus niets aangetoond." }, "offhost_setup": { "configured": "Kopieën verlaten deze server elke nacht.", diff --git a/messages/pl.json b/messages/pl.json index 659d8b920..2fc2ef566 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -8727,7 +8727,8 @@ "not_run_yet": "Comiesięczna próba przywracania jeszcze się nie odbyła.", "failed": "Ostatnia próba przywracania nie powiodła się: nie udało się odczytać {failed} z {checked} kopii.", "failed_run": "Ostatnia próba przywracania nie dobiegła końca.", - "passed": "Ostatnia próba przywracania odczytała kopie: {checked}." + "passed": "Ostatnia próba przywracania odczytała kopie: {checked}.", + "nothing_checked": "Ostatnia próba przywracania zatrzymała się na limicie czasu, zanim odczytała choć jedną kopię, więc niczego nie potwierdziła." }, "offhost_setup": { "configured": "Kopie opuszczają ten serwer każdej nocy.", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index 8c63e5948..8942a783d 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -8727,7 +8727,8 @@ "not_run_yet": "O teste mensal de restauração ainda não foi executado.", "failed": "O último teste de restauração falhou: {failed} de {checked} cópias não puderam ser lidas de volta.", "failed_run": "O último teste de restauração não pôde ser concluído.", - "passed": "O último teste de restauração leu de volta {checked} cópias." + "passed": "O último teste de restauração leu de volta {checked} cópias.", + "nothing_checked": "O último teste de restauração parou no limite de tempo antes de ler de volta uma única cópia, então não comprovou nada." }, "offhost_setup": { "configured": "As cópias saem deste servidor todas as noites.", diff --git a/src/lib/jobs/__tests__/restore-drill.test.ts b/src/lib/jobs/__tests__/restore-drill.test.ts index e6fe03ea3..abad40385 100644 --- a/src/lib/jobs/__tests__/restore-drill.test.ts +++ b/src/lib/jobs/__tests__/restore-drill.test.ts @@ -284,7 +284,24 @@ describe("handleRestoreDrill", () => { ); }); - it("keeps a failed drill's verdict even when keeping it fails", async () => { + it("records no pass for a run that stopped before its first object", async () => { + stubConfigEnv(); + const s3 = makeS3Mock({ + "2026-06-01/user-aaa.json.enc": backupObject({ userId: "user-aaa" }), + }); + const offhost = await import("../offhost-backup"); + vi.spyOn(offhost, "getS3Client").mockResolvedValue(s3 as never); + // The job's budget is already spent: pg-boss aborted it. + await handleRestoreDrill([ + { signal: { aborted: true }, expireInSeconds: 900 } as never, + ]); + expect(recordRestoreDrillOutcome).toHaveBeenCalledTimes(1); + expect(vi.mocked(recordRestoreDrillOutcome).mock.calls[0][0]).toMatchObject( + { ok: null, accountsChecked: 0 }, + ); + }); + + it("still reports the drill's own failure when its verdict cannot be kept", async () => { stubConfigEnv(); vi.stubEnv("BACKUP_ENCRYPTION_KEY", "not-a-key"); vi.mocked(recordRestoreDrillOutcome).mockRejectedValueOnce( diff --git a/src/lib/jobs/restore-drill-outcome.ts b/src/lib/jobs/restore-drill-outcome.ts index c6ab0bb69..c176e9d25 100644 --- a/src/lib/jobs/restore-drill-outcome.ts +++ b/src/lib/jobs/restore-drill-outcome.ts @@ -19,7 +19,12 @@ import { redactSecrets } from "@/lib/logging/redact"; const MAX_REASON_CHARS = 300; export interface RestoreDrillResult { - ok: boolean; + /** + * The verdict; null for a run that checked nothing (it stopped on its time + * budget before the first object), which is no verdict at all: the run and + * its counts are kept, the last pass or failure stands. + */ + ok: boolean | null; accountsChecked: number; accountsFailed: number; /** Why it failed; ignored when `ok`. */ @@ -30,17 +35,21 @@ export async function recordRestoreDrillOutcome( result: RestoreDrillResult, at: Date = new Date(), ): Promise { - const reason = result.ok - ? null - : (() => { + const reason = + result.ok !== false + ? null + : (() => { const text = redactSecrets((result.reason ?? "").trim()); return text.length > MAX_REASON_CHARS ? `${text.slice(0, MAX_REASON_CHARS)}…` : text || null; })(); - const verdict = result.ok - ? { lastSuccessAt: at } - : { lastFailureAt: at, lastFailureReason: reason }; + const verdict = + result.ok === null + ? {} + : result.ok + ? { lastSuccessAt: at } + : { lastFailureAt: at, lastFailureReason: reason }; const counts = { lastRunAt: at, accountsChecked: result.accountsChecked, diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index 89a6a49af..18097e3a9 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -346,7 +346,9 @@ export async function handleRestoreDrill( ); } await keepOutcome(evt, { - ok: report.failed.length === 0, + // A run that stopped before its first object read nothing back: it + // is no pass, and it records none. + ok: report.accounts.length === 0 ? null : report.failed.length === 0, accountsChecked: report.accounts.length, accountsFailed: report.failed.length, reason: diff --git a/src/lib/recovery/__tests__/readiness-model.test.ts b/src/lib/recovery/__tests__/readiness-model.test.ts index e8e92ab2c..35f0153e3 100644 --- a/src/lib/recovery/__tests__/readiness-model.test.ts +++ b/src/lib/recovery/__tests__/readiness-model.test.ts @@ -538,6 +538,19 @@ describe("restore drill", () => { ).toBe("failed_run"); }); + // A run that stopped on its budget before the first object read nothing + // back. It used to be recorded as a pass and the tile said "passed". + it("does not call a run that read nothing back passed, whatever passed before", () => { + expect( + drill({ + ...run, + lastRunAt: ago(DAY), + lastSuccessAt: ago(40 * DAY), + accountsChecked: 0, + }), + ).toMatchObject({ state: "attention", reason: "nothing_checked" }); + }); + it("does not apply without a bucket, and is ok before its first run", () => { expect(drill(null, false).state).toBe("inactive"); expect(drill(null)).toMatchObject({ state: "ok", reason: "not_run_yet" }); diff --git a/src/lib/recovery/readiness-model.ts b/src/lib/recovery/readiness-model.ts index 90b204f2f..7293923de 100644 --- a/src/lib/recovery/readiness-model.ts +++ b/src/lib/recovery/readiness-model.ts @@ -420,6 +420,11 @@ export function evaluateRestoreDrill( drill.lastFailureReason, ); } + // The latest run read nothing back (it stopped on its budget before the + // first object): no proof that a copy restores, whatever passed before. + if (drill.accountsChecked === 0) { + return signal(id, "attention", "nothing_checked", drill.lastRunAt); + } return signal(id, "ok", "passed", drill.lastSuccessAt ?? drill.lastRunAt, { checked: drill.accountsChecked, }); diff --git a/tests/integration/restore-drill-outcome.test.ts b/tests/integration/restore-drill-outcome.test.ts new file mode 100644 index 000000000..f92fb3b56 --- /dev/null +++ b/tests/integration/restore-drill-outcome.test.ts @@ -0,0 +1,102 @@ +/** + * The restore drill's kept verdict, against Postgres, read back the way the + * readiness tile reads it. The unit tests of the drill stub this write; here + * it is the subject. + * + * The rule: a failure stays the verdict until a later drill passes, with no + * time window, and a run that checked nothing changes no verdict. Mutations + * that must turn this red: let a pass clear `lastFailureAt`, or let a failure + * clear `lastSuccessAt` (the next pass would then read as recovered from + * nothing); record a run that checked nothing as a pass. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { recordRestoreDrillOutcome } from "@/lib/jobs/restore-drill-outcome"; +import { evaluateRestoreDrill } from "@/lib/recovery/readiness-model"; + +const DAY = 86_400_000; +const T0 = new Date("2026-09-01T04:00:00Z"); +const at = (days: number) => new Date(T0.getTime() + days * DAY); + +async function verdict() { + const row = await getPrismaClient().restoreDrillOutcome.findUniqueOrThrow({ + where: { id: "singleton" }, + }); + return { + row, + signal: evaluateRestoreDrill({ + offhost: { configured: true, databaseLifecycle: null, accounts: [] }, + drill: row, + }), + }; +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); +}); + +describe("the restore drill's kept verdict", () => { + it("keeps a pass and a failure apart, and a failure until a later pass", async () => { + await recordRestoreDrillOutcome( + { ok: true, accountsChecked: 3, accountsFailed: 0 }, + at(0), + ); + expect((await verdict()).signal).toMatchObject({ + state: "ok", + reason: "passed", + }); + + await recordRestoreDrillOutcome( + { + ok: false, + accountsChecked: 3, + accountsFailed: 1, + reason: "backup object 2026-09-30/user-x.json.enc does not decrypt", + }, + at(30), + ); + const failed = await verdict(); + expect(failed.row.lastSuccessAt).toEqual(at(0)); + expect(failed.row.lastFailureAt).toEqual(at(30)); + expect(failed.signal).toMatchObject({ + state: "failing", + reason: "failed", + detail: expect.stringMatching(/does not decrypt/), + }); + + // A run that stopped before its first object is no verdict: the failure + // stands, and the tile does not call it passed. + await recordRestoreDrillOutcome( + { ok: null, accountsChecked: 0, accountsFailed: 0 }, + at(60), + ); + const nothing = await verdict(); + expect(nothing.row.lastFailureAt).toEqual(at(30)); + expect(nothing.row.lastSuccessAt).toEqual(at(0)); + expect(nothing.row.lastRunAt).toEqual(at(60)); + expect(nothing.signal.state).toBe("failing"); + + await recordRestoreDrillOutcome( + { ok: true, accountsChecked: 4, accountsFailed: 0 }, + at(90), + ); + const recovered = await verdict(); + expect(recovered.row.lastFailureAt).toEqual(at(30)); + expect(recovered.row.lastFailureReason).toMatch(/does not decrypt/); + expect(recovered.signal).toMatchObject({ state: "ok", reason: "passed" }); + }); + + it("does not record a run that checked nothing as a pass", async () => { + await recordRestoreDrillOutcome( + { ok: null, accountsChecked: 0, accountsFailed: 0 }, + at(0), + ); + const { row, signal } = await verdict(); + expect(row.lastSuccessAt).toBeNull(); + expect(signal).toMatchObject({ + state: "attention", + reason: "nothing_checked", + }); + }); +});