mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-18 08:02:12 +00:00
fix(restore-drill): a drill that read nothing back is no pass
A drill that stopped on its time budget before the first object recorded ok with zero accounts checked, and the readiness tile said passed. Such a run now records its time and counts and no verdict, so the last pass or failure stands, and readiness calls it attention (nothing_checked) in every locale. The upsert that keeps a pass and a failure apart was stubbed in the drill's tests and tested nowhere; it now has an integration test read back through the readiness rule. The drill test that claimed to keep a failed verdict when keeping it fails is named for what it checks.
This commit is contained in:
1 parent
a857fd0264
commit
6051488b65
15 files changed
+175
-18
No files matched your search
+2
-1
@@ -8727,7 +8727,8 @@
|
||||
"not_run_yet": "Die monatliche Wiederherstellungsprobe ist noch nicht gelaufen.",
|
||||
"failed": "Die letzte Wiederherstellungsprobe ist fehlgeschlagen: {failed} von {checked} Kopien ließen sich nicht zurücklesen.",
|
||||
"failed_run": "Die letzte Wiederherstellungsprobe konnte nicht zu Ende laufen.",
|
||||
"passed": "Die letzte Wiederherstellungsprobe hat {checked} Kopien zurückgelesen."
|
||||
"passed": "Die letzte Wiederherstellungsprobe hat {checked} Kopien zurückgelesen.",
|
||||
"nothing_checked": "Die letzte Wiederherstellungsprobe hat ihr Zeitlimit erreicht, bevor sie eine einzige Kopie zurückgelesen hat, und hat damit nichts belegt."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Kopien verlassen diesen Server jede Nacht.",
|
||||
|
||||
+2
-1
@@ -8727,7 +8727,8 @@
|
||||
"not_run_yet": "The monthly restore drill has not run yet.",
|
||||
"failed": "The last restore drill failed: {failed} of {checked} copies could not be read back.",
|
||||
"failed_run": "The last restore drill could not finish.",
|
||||
"passed": "The last restore drill read back {checked} copies."
|
||||
"passed": "The last restore drill read back {checked} copies.",
|
||||
"nothing_checked": "The last restore drill stopped on its time limit before it read back a single copy, so it proved nothing."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Copies leave this server every night.",
|
||||
|
||||
+2
-1
@@ -8727,7 +8727,8 @@
|
||||
"not_run_yet": "El simulacro de restauración mensual aún no se ha ejecutado.",
|
||||
"failed": "El último simulacro de restauración falló: no se pudieron volver a leer {failed} de {checked} copias.",
|
||||
"failed_run": "El último simulacro de restauración no pudo terminar.",
|
||||
"passed": "El último simulacro de restauración volvió a leer {checked} copias."
|
||||
"passed": "El último simulacro de restauración volvió a leer {checked} copias.",
|
||||
"nothing_checked": "El último simulacro de restauración se detuvo por su límite de tiempo antes de leer una sola copia, así que no demostró nada."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Las copias salen de este servidor cada noche.",
|
||||
|
||||
+2
-1
@@ -8727,7 +8727,8 @@
|
||||
"not_run_yet": "Le test de restauration mensuel n'a pas encore été lancé.",
|
||||
"failed": "Le dernier test de restauration a échoué : {failed} copies sur {checked} n'ont pas pu être relues.",
|
||||
"failed_run": "Le dernier test de restauration n'a pas pu aller jusqu'au bout.",
|
||||
"passed": "Le dernier test de restauration a relu {checked} copies."
|
||||
"passed": "Le dernier test de restauration a relu {checked} copies.",
|
||||
"nothing_checked": "Le dernier test de restauration s'est arrêté sur sa limite de temps avant de relire une seule copie : il n'a rien prouvé."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Les copies quittent ce serveur chaque nuit.",
|
||||
|
||||
+2
-1
@@ -8727,7 +8727,8 @@
|
||||
"not_run_yet": "La prova di ripristino mensile non è ancora stata eseguita.",
|
||||
"failed": "L'ultima prova di ripristino non è riuscita: {failed} copie su {checked} non si sono potute rileggere.",
|
||||
"failed_run": "L'ultima prova di ripristino non è riuscita ad arrivare in fondo.",
|
||||
"passed": "L'ultima prova di ripristino ha riletto {checked} copie."
|
||||
"passed": "L'ultima prova di ripristino ha riletto {checked} copie.",
|
||||
"nothing_checked": "L'ultima prova di ripristino si è fermata al suo limite di tempo prima di rileggere una sola copia, quindi non ha dimostrato nulla."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Le copie lasciano questo server ogni notte.",
|
||||
|
||||
+2
-1
@@ -8727,7 +8727,8 @@
|
||||
"not_run_yet": "월간 복원 훈련이 아직 실행되지 않았습니다.",
|
||||
"failed": "마지막 복원 훈련이 실패했습니다: 사본 {checked}개 중 {failed}개를 다시 읽지 못했습니다.",
|
||||
"failed_run": "마지막 복원 훈련을 끝까지 마치지 못했습니다.",
|
||||
"passed": "마지막 복원 훈련에서 사본 {checked}개를 다시 읽었습니다."
|
||||
"passed": "마지막 복원 훈련에서 사본 {checked}개를 다시 읽었습니다.",
|
||||
"nothing_checked": "마지막 복원 훈련이 사본을 하나도 다시 읽기 전에 시간 제한으로 멈췄기 때문에 아무것도 확인하지 못했습니다."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "사본이 매일 밤 이 서버 밖으로 나갑니다.",
|
||||
|
||||
+2
-1
@@ -8727,7 +8727,8 @@
|
||||
"not_run_yet": "De maandelijkse hersteltest is nog niet uitgevoerd.",
|
||||
"failed": "De laatste hersteltest is mislukt: {failed} van {checked} kopieën konden niet worden teruggelezen.",
|
||||
"failed_run": "De laatste hersteltest kon niet worden afgerond.",
|
||||
"passed": "De laatste hersteltest heeft {checked} kopieën teruggelezen."
|
||||
"passed": "De laatste hersteltest heeft {checked} kopieën teruggelezen.",
|
||||
"nothing_checked": "De laatste hersteltest stopte op zijn tijdslimiet voordat hij ook maar één kopie had teruggelezen, en heeft dus niets aangetoond."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Kopieën verlaten deze server elke nacht.",
|
||||
|
||||
+2
-1
@@ -8727,7 +8727,8 @@
|
||||
"not_run_yet": "Comiesięczna próba przywracania jeszcze się nie odbyła.",
|
||||
"failed": "Ostatnia próba przywracania nie powiodła się: nie udało się odczytać {failed} z {checked} kopii.",
|
||||
"failed_run": "Ostatnia próba przywracania nie dobiegła końca.",
|
||||
"passed": "Ostatnia próba przywracania odczytała kopie: {checked}."
|
||||
"passed": "Ostatnia próba przywracania odczytała kopie: {checked}.",
|
||||
"nothing_checked": "Ostatnia próba przywracania zatrzymała się na limicie czasu, zanim odczytała choć jedną kopię, więc niczego nie potwierdziła."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "Kopie opuszczają ten serwer każdej nocy.",
|
||||
|
||||
+2
-1
@@ -8727,7 +8727,8 @@
|
||||
"not_run_yet": "O teste mensal de restauração ainda não foi executado.",
|
||||
"failed": "O último teste de restauração falhou: {failed} de {checked} cópias não puderam ser lidas de volta.",
|
||||
"failed_run": "O último teste de restauração não pôde ser concluído.",
|
||||
"passed": "O último teste de restauração leu de volta {checked} cópias."
|
||||
"passed": "O último teste de restauração leu de volta {checked} cópias.",
|
||||
"nothing_checked": "O último teste de restauração parou no limite de tempo antes de ler de volta uma única cópia, então não comprovou nada."
|
||||
},
|
||||
"offhost_setup": {
|
||||
"configured": "As cópias saem deste servidor todas as noites.",
|
||||
|
||||
@@ -284,7 +284,24 @@ describe("handleRestoreDrill", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps a failed drill's verdict even when keeping it fails", async () => {
|
||||
it("records no pass for a run that stopped before its first object", async () => {
|
||||
stubConfigEnv();
|
||||
const s3 = makeS3Mock({
|
||||
"2026-06-01/user-aaa.json.enc": backupObject({ userId: "user-aaa" }),
|
||||
});
|
||||
const offhost = await import("../offhost-backup");
|
||||
vi.spyOn(offhost, "getS3Client").mockResolvedValue(s3 as never);
|
||||
// The job's budget is already spent: pg-boss aborted it.
|
||||
await handleRestoreDrill([
|
||||
{ signal: { aborted: true }, expireInSeconds: 900 } as never,
|
||||
]);
|
||||
expect(recordRestoreDrillOutcome).toHaveBeenCalledTimes(1);
|
||||
expect(vi.mocked(recordRestoreDrillOutcome).mock.calls[0][0]).toMatchObject(
|
||||
{ ok: null, accountsChecked: 0 },
|
||||
);
|
||||
});
|
||||
|
||||
it("still reports the drill's own failure when its verdict cannot be kept", async () => {
|
||||
stubConfigEnv();
|
||||
vi.stubEnv("BACKUP_ENCRYPTION_KEY", "not-a-key");
|
||||
vi.mocked(recordRestoreDrillOutcome).mockRejectedValueOnce(
|
||||
|
||||
@@ -19,7 +19,12 @@ import { redactSecrets } from "@/lib/logging/redact";
|
||||
const MAX_REASON_CHARS = 300;
|
||||
|
||||
export interface RestoreDrillResult {
|
||||
ok: boolean;
|
||||
/**
|
||||
* The verdict; null for a run that checked nothing (it stopped on its time
|
||||
* budget before the first object), which is no verdict at all: the run and
|
||||
* its counts are kept, the last pass or failure stands.
|
||||
*/
|
||||
ok: boolean | null;
|
||||
accountsChecked: number;
|
||||
accountsFailed: number;
|
||||
/** Why it failed; ignored when `ok`. */
|
||||
@@ -30,17 +35,21 @@ export async function recordRestoreDrillOutcome(
|
||||
result: RestoreDrillResult,
|
||||
at: Date = new Date(),
|
||||
): Promise<void> {
|
||||
const reason = result.ok
|
||||
? null
|
||||
: (() => {
|
||||
const reason =
|
||||
result.ok !== false
|
||||
? null
|
||||
: (() => {
|
||||
const text = redactSecrets((result.reason ?? "").trim());
|
||||
return text.length > MAX_REASON_CHARS
|
||||
? `${text.slice(0, MAX_REASON_CHARS)}…`
|
||||
: text || null;
|
||||
})();
|
||||
const verdict = result.ok
|
||||
? { lastSuccessAt: at }
|
||||
: { lastFailureAt: at, lastFailureReason: reason };
|
||||
const verdict =
|
||||
result.ok === null
|
||||
? {}
|
||||
: result.ok
|
||||
? { lastSuccessAt: at }
|
||||
: { lastFailureAt: at, lastFailureReason: reason };
|
||||
const counts = {
|
||||
lastRunAt: at,
|
||||
accountsChecked: result.accountsChecked,
|
||||
|
||||
@@ -346,7 +346,9 @@ export async function handleRestoreDrill(
|
||||
);
|
||||
}
|
||||
await keepOutcome(evt, {
|
||||
ok: report.failed.length === 0,
|
||||
// A run that stopped before its first object read nothing back: it
|
||||
// is no pass, and it records none.
|
||||
ok: report.accounts.length === 0 ? null : report.failed.length === 0,
|
||||
accountsChecked: report.accounts.length,
|
||||
accountsFailed: report.failed.length,
|
||||
reason:
|
||||
|
||||
@@ -538,6 +538,19 @@ describe("restore drill", () => {
|
||||
).toBe("failed_run");
|
||||
});
|
||||
|
||||
// A run that stopped on its budget before the first object read nothing
|
||||
// back. It used to be recorded as a pass and the tile said "passed".
|
||||
it("does not call a run that read nothing back passed, whatever passed before", () => {
|
||||
expect(
|
||||
drill({
|
||||
...run,
|
||||
lastRunAt: ago(DAY),
|
||||
lastSuccessAt: ago(40 * DAY),
|
||||
accountsChecked: 0,
|
||||
}),
|
||||
).toMatchObject({ state: "attention", reason: "nothing_checked" });
|
||||
});
|
||||
|
||||
it("does not apply without a bucket, and is ok before its first run", () => {
|
||||
expect(drill(null, false).state).toBe("inactive");
|
||||
expect(drill(null)).toMatchObject({ state: "ok", reason: "not_run_yet" });
|
||||
|
||||
@@ -420,6 +420,11 @@ export function evaluateRestoreDrill(
|
||||
drill.lastFailureReason,
|
||||
);
|
||||
}
|
||||
// The latest run read nothing back (it stopped on its budget before the
|
||||
// first object): no proof that a copy restores, whatever passed before.
|
||||
if (drill.accountsChecked === 0) {
|
||||
return signal(id, "attention", "nothing_checked", drill.lastRunAt);
|
||||
}
|
||||
return signal(id, "ok", "passed", drill.lastSuccessAt ?? drill.lastRunAt, {
|
||||
checked: drill.accountsChecked,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* The restore drill's kept verdict, against Postgres, read back the way the
|
||||
* readiness tile reads it. The unit tests of the drill stub this write; here
|
||||
* it is the subject.
|
||||
*
|
||||
* The rule: a failure stays the verdict until a later drill passes, with no
|
||||
* time window, and a run that checked nothing changes no verdict. Mutations
|
||||
* that must turn this red: let a pass clear `lastFailureAt`, or let a failure
|
||||
* clear `lastSuccessAt` (the next pass would then read as recovered from
|
||||
* nothing); record a run that checked nothing as a pass.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import { recordRestoreDrillOutcome } from "@/lib/jobs/restore-drill-outcome";
|
||||
import { evaluateRestoreDrill } from "@/lib/recovery/readiness-model";
|
||||
|
||||
const DAY = 86_400_000;
|
||||
const T0 = new Date("2026-09-01T04:00:00Z");
|
||||
const at = (days: number) => new Date(T0.getTime() + days * DAY);
|
||||
|
||||
async function verdict() {
|
||||
const row = await getPrismaClient().restoreDrillOutcome.findUniqueOrThrow({
|
||||
where: { id: "singleton" },
|
||||
});
|
||||
return {
|
||||
row,
|
||||
signal: evaluateRestoreDrill({
|
||||
offhost: { configured: true, databaseLifecycle: null, accounts: [] },
|
||||
drill: row,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(getPrismaClient());
|
||||
});
|
||||
|
||||
describe("the restore drill's kept verdict", () => {
|
||||
it("keeps a pass and a failure apart, and a failure until a later pass", async () => {
|
||||
await recordRestoreDrillOutcome(
|
||||
{ ok: true, accountsChecked: 3, accountsFailed: 0 },
|
||||
at(0),
|
||||
);
|
||||
expect((await verdict()).signal).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "passed",
|
||||
});
|
||||
|
||||
await recordRestoreDrillOutcome(
|
||||
{
|
||||
ok: false,
|
||||
accountsChecked: 3,
|
||||
accountsFailed: 1,
|
||||
reason: "backup object 2026-09-30/user-x.json.enc does not decrypt",
|
||||
},
|
||||
at(30),
|
||||
);
|
||||
const failed = await verdict();
|
||||
expect(failed.row.lastSuccessAt).toEqual(at(0));
|
||||
expect(failed.row.lastFailureAt).toEqual(at(30));
|
||||
expect(failed.signal).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "failed",
|
||||
detail: expect.stringMatching(/does not decrypt/),
|
||||
});
|
||||
|
||||
// A run that stopped before its first object is no verdict: the failure
|
||||
// stands, and the tile does not call it passed.
|
||||
await recordRestoreDrillOutcome(
|
||||
{ ok: null, accountsChecked: 0, accountsFailed: 0 },
|
||||
at(60),
|
||||
);
|
||||
const nothing = await verdict();
|
||||
expect(nothing.row.lastFailureAt).toEqual(at(30));
|
||||
expect(nothing.row.lastSuccessAt).toEqual(at(0));
|
||||
expect(nothing.row.lastRunAt).toEqual(at(60));
|
||||
expect(nothing.signal.state).toBe("failing");
|
||||
|
||||
await recordRestoreDrillOutcome(
|
||||
{ ok: true, accountsChecked: 4, accountsFailed: 0 },
|
||||
at(90),
|
||||
);
|
||||
const recovered = await verdict();
|
||||
expect(recovered.row.lastFailureAt).toEqual(at(30));
|
||||
expect(recovered.row.lastFailureReason).toMatch(/does not decrypt/);
|
||||
expect(recovered.signal).toMatchObject({ state: "ok", reason: "passed" });
|
||||
});
|
||||
|
||||
it("does not record a run that checked nothing as a pass", async () => {
|
||||
await recordRestoreDrillOutcome(
|
||||
{ ok: null, accountsChecked: 0, accountsFailed: 0 },
|
||||
at(0),
|
||||
);
|
||||
const { row, signal } = await verdict();
|
||||
expect(row.lastSuccessAt).toBeNull();
|
||||
expect(signal).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "nothing_checked",
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user