mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 04:50:59 +00:00
feat(backups): tell an empty ledger apart from an empty bucket
On the release that adds the ledger the table is empty, so on a host that has been uploading nightly for months every account read Never, under a sentence saying no copy had ever reached the bucket. That sentence is untrue on exactly the morning an operator opens the new card for the first time, and it is a claim the host cannot make: what is empty is the record, not the bucket. The run now records every account it walks, not only the ones an object came of. A walk that produced nothing moves the attempt instant and leaves the success columns alone, so the two cases separate: no row at all is no record yet, a row without a success is never. Never now means what the word says — a run looked at this account and has never put anything there. The card names the last check on the never row, so the state is a fact with a timestamp rather than an assertion. Runbook and all seven bundles carry the same two sentences. The integration file proves both arms against real Postgres, including that a failed run leaves the previous good copy on the row.
This commit is contained in:
1 parent
cfc19a211c
commit
25f0310596
21 files changed
+340
-80
No files matched your search
@@ -225,8 +225,16 @@ failed` each time, which looks like weather.
|
||||
host last put that account's encrypted copy in the bucket, how big it was, and
|
||||
how that compares with the nightly schedule. **Fresh** is inside one run,
|
||||
**due** means one run produced nothing for that account, **stale** means two
|
||||
did not, and **never** means no object has ever landed. On a host without
|
||||
`BACKUP_S3_*` the card says off-host backup is not configured and stops there.
|
||||
did not, **never** means a run walked the account and no object has ever
|
||||
landed for it, and **no record yet** means no run has recorded the account at
|
||||
all. On a host without `BACKUP_S3_*` the card says off-host backup is not
|
||||
configured and stops there.
|
||||
|
||||
The ledger starts empty on the release that adds it, so on a host that has
|
||||
been uploading nightly for months every account reads **no record yet** until
|
||||
the first run after the upgrade. That is the ledger having no history, not the
|
||||
bucket being empty — which is why it is a separate word from **never**, and
|
||||
why the row says so rather than claiming nothing has ever reached the bucket.
|
||||
|
||||
Each verdict carries six hours of slack on top of the schedule's own 24, and
|
||||
that slack is load-bearing on two nights. The cron runs at 02:30 Europe/Berlin,
|
||||
|
||||
@@ -717,6 +717,7 @@ async function installA11yMocks(page: Page) {
|
||||
{
|
||||
userId: "a11y-user",
|
||||
username: "a11y-backup-user",
|
||||
lastAttemptAt: "2026-07-20T03:00:00.000Z",
|
||||
lastSuccessAt: "2026-07-20T03:00:00.000Z",
|
||||
sizeBytes: 4096,
|
||||
ageHours: 3,
|
||||
@@ -725,6 +726,7 @@ async function installA11yMocks(page: Page) {
|
||||
{
|
||||
userId: "a11y-user-2",
|
||||
username: "a11y-backup-user-2",
|
||||
lastAttemptAt: "2026-07-14T03:00:00.000Z",
|
||||
lastSuccessAt: "2026-07-14T03:00:00.000Z",
|
||||
sizeBytes: 2048,
|
||||
ageHours: 150,
|
||||
@@ -733,11 +735,21 @@ async function installA11yMocks(page: Page) {
|
||||
{
|
||||
userId: "a11y-user-3",
|
||||
username: "a11y-backup-user-3",
|
||||
lastAttemptAt: "2026-07-20T03:00:00.000Z",
|
||||
lastSuccessAt: null,
|
||||
sizeBytes: null,
|
||||
ageHours: null,
|
||||
freshness: "never",
|
||||
},
|
||||
{
|
||||
userId: "a11y-user-4",
|
||||
username: "a11y-backup-user-4",
|
||||
lastAttemptAt: null,
|
||||
lastSuccessAt: null,
|
||||
sizeBytes: null,
|
||||
ageHours: null,
|
||||
freshness: "unknown",
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
|
||||
+3
-1
@@ -7026,7 +7026,9 @@
|
||||
"stateDue": "Fällig",
|
||||
"stateStale": "Veraltet",
|
||||
"stateNever": "Nie",
|
||||
"neverDetail": "Für dieses Konto ist noch nie eine Kopie im Bucket angekommen.",
|
||||
"stateUnknown": "Noch kein Eintrag",
|
||||
"neverDetail": "Für dieses Konto ist nichts angekommen. Zuletzt geprüft {when}.",
|
||||
"unknownDetail": "Noch kein nächtlicher Lauf hat dieses Konto erfasst.",
|
||||
"age": "vor {hours} h",
|
||||
"scheduleNote": "Der Upload läuft alle {hours} h. Frisch heißt innerhalb eines Laufs, fällig heißt ein Lauf blieb ohne Ergebnis, veraltet heißt zwei."
|
||||
}
|
||||
|
||||
+3
-1
@@ -7026,7 +7026,9 @@
|
||||
"stateDue": "Due",
|
||||
"stateStale": "Stale",
|
||||
"stateNever": "Never",
|
||||
"neverDetail": "No copy has ever reached the bucket for this account.",
|
||||
"stateUnknown": "No record yet",
|
||||
"neverDetail": "Nothing has landed for this account. Last checked {when}.",
|
||||
"unknownDetail": "No nightly run has recorded this account yet.",
|
||||
"age": "{hours} h ago",
|
||||
"scheduleNote": "The upload runs every {hours} h. Fresh means inside one run, due means one run produced nothing, stale means two did not."
|
||||
}
|
||||
|
||||
+3
-1
@@ -7026,7 +7026,9 @@
|
||||
"stateDue": "Pendiente",
|
||||
"stateStale": "Obsoleta",
|
||||
"stateNever": "Nunca",
|
||||
"neverDetail": "Para esta cuenta nunca ha llegado una copia al bucket.",
|
||||
"stateUnknown": "Sin registro aún",
|
||||
"neverDetail": "No ha llegado nada para esta cuenta. Última comprobación {when}.",
|
||||
"unknownDetail": "Ninguna ejecución nocturna ha registrado esta cuenta todavía.",
|
||||
"age": "hace {hours} h",
|
||||
"scheduleNote": "La subida se ejecuta cada {hours} h. Reciente significa dentro de una ejecución, pendiente que una no produjo nada, obsoleta que fueron dos."
|
||||
}
|
||||
|
||||
+3
-1
@@ -7026,7 +7026,9 @@
|
||||
"stateDue": "Attendue",
|
||||
"stateStale": "Périmée",
|
||||
"stateNever": "Jamais",
|
||||
"neverDetail": "Aucune copie n'est jamais arrivée dans le bucket pour ce compte.",
|
||||
"stateUnknown": "Pas encore d'enregistrement",
|
||||
"neverDetail": "Rien n'est arrivé pour ce compte. Dernière vérification {when}.",
|
||||
"unknownDetail": "Aucun cycle nocturne n'a encore enregistré ce compte.",
|
||||
"age": "il y a {hours} h",
|
||||
"scheduleNote": "L'envoi tourne toutes les {hours} h. Récente signifie dans un cycle, attendue qu'un cycle n'a rien produit, périmée que deux n'ont rien produit."
|
||||
}
|
||||
|
||||
+3
-1
@@ -7026,7 +7026,9 @@
|
||||
"stateDue": "Attesa",
|
||||
"stateStale": "Obsoleta",
|
||||
"stateNever": "Mai",
|
||||
"neverDetail": "Per questo account nessuna copia è mai arrivata nel bucket.",
|
||||
"stateUnknown": "Nessuna registrazione",
|
||||
"neverDetail": "Per questo account non è arrivato nulla. Ultimo controllo {when}.",
|
||||
"unknownDetail": "Nessun giro notturno ha ancora registrato questo account.",
|
||||
"age": "{hours} h fa",
|
||||
"scheduleNote": "Il caricamento gira ogni {hours} h. Recente significa entro un giro, attesa che un giro non ha prodotto nulla, obsoleta che non lo hanno fatto due."
|
||||
}
|
||||
|
||||
+3
-1
@@ -7006,7 +7006,9 @@
|
||||
"stateDue": "지연",
|
||||
"stateStale": "오래됨",
|
||||
"stateNever": "없음",
|
||||
"neverDetail": "이 계정의 사본이 버킷에 도착한 적이 없습니다.",
|
||||
"stateUnknown": "기록 없음",
|
||||
"neverDetail": "이 계정으로 도착한 것이 없습니다. 마지막 확인 {when}.",
|
||||
"unknownDetail": "아직 어떤 야간 실행도 이 계정을 기록하지 않았습니다.",
|
||||
"age": "{hours}시간 전",
|
||||
"scheduleNote": "업로드는 {hours}시간마다 실행됩니다. 최신은 한 번의 실행 이내, 지연은 한 번이 아무것도 만들지 못한 상태, 오래됨은 두 번이 그런 상태입니다."
|
||||
}
|
||||
|
||||
+3
-1
@@ -7026,7 +7026,9 @@
|
||||
"stateDue": "Zaległa",
|
||||
"stateStale": "Przeterminowana",
|
||||
"stateNever": "Nigdy",
|
||||
"neverDetail": "Dla tego konta żadna kopia nigdy nie trafiła do bucketu.",
|
||||
"stateUnknown": "Brak wpisu",
|
||||
"neverDetail": "Dla tego konta nic nie dotarło. Ostatnie sprawdzenie {when}.",
|
||||
"unknownDetail": "Żaden nocny przebieg nie zarejestrował jeszcze tego konta.",
|
||||
"age": "{hours} h temu",
|
||||
"scheduleNote": "Wysyłka działa co {hours} h. Świeża znaczy w obrębie jednego przebiegu, zaległa że jeden nic nie dał, przeterminowana że dwa."
|
||||
}
|
||||
|
||||
@@ -4,11 +4,16 @@
|
||||
-- without the page ever talking to the bucket.
|
||||
CREATE TABLE "offhost_backup_state" (
|
||||
"user_id" TEXT NOT NULL,
|
||||
"last_success_at" TIMESTAMP(3) NOT NULL,
|
||||
-- Written on every account the run walks, success or not. A row's mere
|
||||
-- existence is what lets the console tell "no run has reached this
|
||||
-- account yet" from "a run reached it and nothing landed".
|
||||
"last_attempt_at" TIMESTAMP(3) NOT NULL,
|
||||
-- NULL until an object lands for this account.
|
||||
"last_success_at" TIMESTAMP(3),
|
||||
-- BIGINT, not INTEGER: a single object may occupy 8 MiB x 10 000 multipart
|
||||
-- parts, so the uploader permits 80 GB and a 32-bit column would refuse
|
||||
-- every account past 2.1 GB.
|
||||
"size_bytes" BIGINT NOT NULL,
|
||||
"size_bytes" BIGINT,
|
||||
|
||||
CONSTRAINT "offhost_backup_state_pkey" PRIMARY KEY ("user_id")
|
||||
);
|
||||
|
||||
+15
-4
@@ -4953,17 +4953,28 @@ model DataBackup {
|
||||
model OffhostBackupState {
|
||||
userId String @id @map("user_id")
|
||||
|
||||
/// When the newest object for this account finished uploading. Consumer:
|
||||
/// `classifyOffhostBackup()` behind `GET /api/admin/backups`.
|
||||
lastSuccessAt DateTime @map("last_success_at")
|
||||
/// When the last nightly run walked this account, whether or not an object
|
||||
/// came of it. It is what separates "no run has reached this account yet"
|
||||
/// from "a run reached it and put nothing there" — two facts that look
|
||||
/// identical in a ledger keyed only on success, and the first of which is
|
||||
/// what every account reads on the day this table ships. Consumer:
|
||||
/// `classifyOffhostBackup()` behind `GET /api/admin/backups`, and the
|
||||
/// card's "last checked" line.
|
||||
lastAttemptAt DateTime @map("last_attempt_at")
|
||||
|
||||
/// When the newest object for this account finished uploading. NULL means
|
||||
/// a run has looked at this account and nothing has ever landed for it.
|
||||
/// Consumer: `classifyOffhostBackup()` behind `GET /api/admin/backups`.
|
||||
lastSuccessAt DateTime? @map("last_success_at")
|
||||
|
||||
/// Size of that object in bytes — the same number the run reports as its
|
||||
/// largest, per account. `BigInt` because the uploader permits an object of
|
||||
/// up to 8 MiB × 10 000 parts = 80 GB and a 32-bit column stops at 2.1 GB:
|
||||
/// the biggest account on the host, the one an operator cares most about,
|
||||
/// is exactly the one whose row would refuse to be written.
|
||||
/// NULL alongside a NULL `lastSuccessAt`.
|
||||
/// Consumer: the off-host card's size column.
|
||||
sizeBytes BigInt @map("size_bytes")
|
||||
sizeBytes BigInt? @map("size_bytes")
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
id: "u1",
|
||||
username: "account-one",
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(now - 3 * HOUR),
|
||||
lastSuccessAt: new Date(now - 3 * HOUR),
|
||||
// The column is BigInt — an object may be 80 GB — and the wire type
|
||||
// is a plain number.
|
||||
@@ -73,6 +74,7 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
id: "u2",
|
||||
username: "account-two",
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(now - 31 * HOUR),
|
||||
lastSuccessAt: new Date(now - 31 * HOUR),
|
||||
sizeBytes: BigInt(8192),
|
||||
},
|
||||
@@ -81,11 +83,24 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
id: "u3",
|
||||
username: "account-three",
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(now - 100 * HOUR),
|
||||
lastSuccessAt: new Date(now - 100 * HOUR),
|
||||
sizeBytes: BigInt(128),
|
||||
},
|
||||
},
|
||||
{ id: "u4", username: "account-four", offhostBackupState: null },
|
||||
{
|
||||
id: "u4",
|
||||
username: "account-four",
|
||||
// Walked by a run, nothing landed.
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(now - 2 * HOUR),
|
||||
lastSuccessAt: null,
|
||||
sizeBytes: null,
|
||||
},
|
||||
},
|
||||
// No run has recorded this one at all — the state of every account on
|
||||
// the morning the ledger ships.
|
||||
{ id: "u5", username: "account-five", offhostBackupState: null },
|
||||
]);
|
||||
|
||||
const body = await read();
|
||||
@@ -99,12 +114,17 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
["account-two", "due"],
|
||||
["account-three", "stale"],
|
||||
["account-four", "never"],
|
||||
["account-five", "unknown"],
|
||||
]);
|
||||
expect(body.offhost.rows[0]?.sizeBytes).toBe(4096);
|
||||
expect(typeof body.offhost.rows[0]?.sizeBytes).toBe("number");
|
||||
expect(body.offhost.rows[0]?.ageHours).toBe(3);
|
||||
expect(body.offhost.rows[3]?.lastSuccessAt).toBeNull();
|
||||
expect(body.offhost.rows[3]?.sizeBytes).toBeNull();
|
||||
expect(body.offhost.rows[3]?.lastAttemptAt).not.toBeNull();
|
||||
// `unknown` is the honest one: the ledger has no history for this
|
||||
// account, which is not the same claim as "the bucket is empty".
|
||||
expect(body.offhost.rows[4]?.lastAttemptAt).toBeNull();
|
||||
});
|
||||
|
||||
it("says so and reads no account when off-host backup is not configured", async () => {
|
||||
@@ -128,6 +148,7 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
id: "u1",
|
||||
username: "account-one",
|
||||
offhostBackupState: {
|
||||
lastAttemptAt: new Date(),
|
||||
lastSuccessAt: new Date(),
|
||||
sizeBytes: BigInt(1),
|
||||
},
|
||||
@@ -139,6 +160,7 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
expect(Object.keys(body.offhost.rows[0] ?? {}).sort()).toEqual([
|
||||
"ageHours",
|
||||
"freshness",
|
||||
"lastAttemptAt",
|
||||
"lastSuccessAt",
|
||||
"sizeBytes",
|
||||
"userId",
|
||||
|
||||
@@ -95,7 +95,11 @@ export const GET = apiHandler(async () => {
|
||||
id: true,
|
||||
username: true,
|
||||
offhostBackupState: {
|
||||
select: { lastSuccessAt: true, sizeBytes: true },
|
||||
select: {
|
||||
lastAttemptAt: true,
|
||||
lastSuccessAt: true,
|
||||
sizeBytes: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { username: "asc" },
|
||||
@@ -105,17 +109,22 @@ export const GET = apiHandler(async () => {
|
||||
const offhostRows: OffhostAccountRow[] = offhostAccounts.map((account) => {
|
||||
const state = account.offhostBackupState;
|
||||
const verdict = classifyOffhostBackup({
|
||||
lastAttemptAt: state?.lastAttemptAt ?? null,
|
||||
lastSuccessAt: state?.lastSuccessAt ?? null,
|
||||
now,
|
||||
});
|
||||
return {
|
||||
userId: account.id,
|
||||
username: account.username,
|
||||
lastSuccessAt: state?.lastSuccessAt.toISOString() ?? null,
|
||||
lastAttemptAt: state?.lastAttemptAt.toISOString() ?? null,
|
||||
lastSuccessAt: state?.lastSuccessAt?.toISOString() ?? null,
|
||||
// BigInt column -> number on the wire. The value is bytes of one
|
||||
// object, capped by the uploader at 80 GB, so it is nowhere near
|
||||
// Number.MAX_SAFE_INTEGER and JSON has no BigInt.
|
||||
sizeBytes: state ? Number(state.sizeBytes) : null,
|
||||
sizeBytes:
|
||||
state?.sizeBytes === null || state?.sizeBytes === undefined
|
||||
? null
|
||||
: Number(state.sizeBytes),
|
||||
ageHours: verdict.ageHours,
|
||||
freshness: verdict.freshness,
|
||||
};
|
||||
|
||||
@@ -59,6 +59,7 @@ describe("<OffhostBackupsSection>", () => {
|
||||
{
|
||||
userId: "u1",
|
||||
username: "account-one",
|
||||
lastAttemptAt: "2026-09-10T06:00:00.000Z",
|
||||
lastSuccessAt: "2026-09-10T06:00:00.000Z",
|
||||
sizeBytes: 4096,
|
||||
ageHours: 3,
|
||||
@@ -67,6 +68,7 @@ describe("<OffhostBackupsSection>", () => {
|
||||
{
|
||||
userId: "u2",
|
||||
username: "account-two",
|
||||
lastAttemptAt: "2026-09-06T06:00:00.000Z",
|
||||
lastSuccessAt: "2026-09-06T06:00:00.000Z",
|
||||
sizeBytes: 2048,
|
||||
ageHours: 99,
|
||||
@@ -75,20 +77,35 @@ describe("<OffhostBackupsSection>", () => {
|
||||
{
|
||||
userId: "u3",
|
||||
username: "account-three",
|
||||
lastAttemptAt: "2026-09-10T02:30:00.000Z",
|
||||
lastSuccessAt: null,
|
||||
sizeBytes: null,
|
||||
ageHours: null,
|
||||
freshness: "never",
|
||||
},
|
||||
{
|
||||
userId: "u4",
|
||||
username: "account-four",
|
||||
lastAttemptAt: null,
|
||||
lastSuccessAt: null,
|
||||
sizeBytes: null,
|
||||
ageHours: null,
|
||||
freshness: "unknown",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(html).toContain('data-offhost-freshness="fresh"');
|
||||
expect(html).toContain('data-offhost-freshness="stale"');
|
||||
expect(html).toContain('data-offhost-freshness="never"');
|
||||
expect(html).toContain('data-offhost-freshness="unknown"');
|
||||
expect(html).toContain("account-two");
|
||||
// The account with no object says so instead of showing a blank cell.
|
||||
expect(html).toContain("No copy has ever reached the bucket");
|
||||
// The account a run walked and found nothing for says that, with the
|
||||
// instant it was checked.
|
||||
expect(html).toContain("Nothing has landed for this account");
|
||||
// The account no run has recorded says THAT — not "never", which would
|
||||
// claim the bucket is empty on the strength of an empty ledger.
|
||||
expect(html).toContain("No nightly run has recorded this account yet");
|
||||
// Never the bucket's own coordinates.
|
||||
expect(html).not.toMatch(/secret|access[- ]key/i);
|
||||
});
|
||||
|
||||
@@ -34,16 +34,17 @@ import type { BackupsList, OffhostAccountRow } from "@/types/backups";
|
||||
import { formatBytes } from "./backups-section";
|
||||
|
||||
/**
|
||||
* Status wash per verdict. `never` is deliberately neutral rather than
|
||||
* alarming: a host that turned the off-host job on this afternoon has not
|
||||
* failed at anything yet, and painting every row red on day one is how an
|
||||
* operator learns to stop reading the colour.
|
||||
* Status wash per verdict. `never` and `unknown` are deliberately neutral
|
||||
* rather than alarming: a host that turned the off-host job on this afternoon
|
||||
* has not failed at anything yet, and painting every row red on day one is how
|
||||
* an operator learns to stop reading the colour.
|
||||
*/
|
||||
const FRESHNESS_STYLE: Record<OffhostBackupFreshness, string> = {
|
||||
fresh: "border-success/30 bg-success/10 text-success",
|
||||
due: "border-warning/30 bg-warning/10 text-warning",
|
||||
stale: "border-destructive/30 bg-destructive/10 text-destructive",
|
||||
never: "border-border bg-muted text-muted-foreground",
|
||||
unknown: "border-border bg-muted text-muted-foreground",
|
||||
};
|
||||
|
||||
function freshnessLabel(
|
||||
@@ -61,6 +62,8 @@ function freshnessLabel(
|
||||
return t("admin.section.backups.offhost.stateStale");
|
||||
case "never":
|
||||
return t("admin.section.backups.offhost.stateNever");
|
||||
case "unknown":
|
||||
return t("admin.section.backups.offhost.stateUnknown");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,14 +90,22 @@ function AccountRow({ row }: { row: OffhostAccountRow }) {
|
||||
</Badge>
|
||||
</div>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{row.lastSuccessAt === null || row.sizeBytes === null
|
||||
? t("admin.section.backups.offhost.neverDetail")
|
||||
: `${fmt.dateTime(row.lastSuccessAt)} · ${formatBytes(
|
||||
row.sizeBytes,
|
||||
fmt,
|
||||
)} · ${t("admin.section.backups.offhost.age", {
|
||||
hours: row.ageHours ?? 0,
|
||||
})}`}
|
||||
{row.lastAttemptAt === null
|
||||
? // No run has recorded this account. Saying "never" here would
|
||||
// claim the bucket is empty when what is empty is the ledger,
|
||||
// which is exactly what every account looks like between the
|
||||
// upgrade that adds it and the first nightly run after that.
|
||||
t("admin.section.backups.offhost.unknownDetail")
|
||||
: row.lastSuccessAt === null || row.sizeBytes === null
|
||||
? t("admin.section.backups.offhost.neverDetail", {
|
||||
when: fmt.dateTime(row.lastAttemptAt),
|
||||
})
|
||||
: `${fmt.dateTime(row.lastSuccessAt)} · ${formatBytes(
|
||||
row.sizeBytes,
|
||||
fmt,
|
||||
)} · ${t("admin.section.backups.offhost.age", {
|
||||
hours: row.ageHours ?? 0,
|
||||
})}`}
|
||||
</p>
|
||||
</li>
|
||||
</ListRow>
|
||||
|
||||
@@ -16,20 +16,40 @@ function ago(ms: number): Date {
|
||||
}
|
||||
|
||||
describe("classifyOffhostBackup", () => {
|
||||
it("says never when the worker has put nothing there for this account", () => {
|
||||
expect(classifyOffhostBackup({ lastSuccessAt: null, now: NOW })).toEqual({
|
||||
freshness: "never",
|
||||
ageHours: null,
|
||||
});
|
||||
it("says unknown, not never, when no run has recorded this account", () => {
|
||||
// The ledger starts empty. On the first morning after the upgrade every
|
||||
// account on a perfectly healthy host looks like this, and `never` would
|
||||
// assert something about the bucket the host has not looked at.
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: null,
|
||||
lastSuccessAt: null,
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ freshness: "unknown", ageHours: null });
|
||||
});
|
||||
|
||||
it("says never once a run has walked the account and put nothing there", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(HOUR),
|
||||
lastSuccessAt: null,
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ freshness: "never", ageHours: null });
|
||||
});
|
||||
|
||||
it("is fresh right up to one period plus the grace and due one millisecond past it", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD + GRACE), now: NOW })
|
||||
.freshness,
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(PERIOD + GRACE),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("fresh");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(PERIOD + GRACE + 1),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
@@ -39,12 +59,14 @@ describe("classifyOffhostBackup", () => {
|
||||
it("is due right up to two periods plus the grace and stale one millisecond past it", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(PERIOD * 2 + GRACE),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("due");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(PERIOD * 2 + GRACE + 1),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
@@ -66,6 +88,7 @@ describe("classifyOffhostBackup", () => {
|
||||
// on every host, with nothing wrong anywhere.
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: beforeTheChange,
|
||||
now: afterTheChange,
|
||||
}).freshness,
|
||||
@@ -76,14 +99,21 @@ describe("classifyOffhostBackup", () => {
|
||||
// 31 hours: past one period even with six hours of slack, so a run that
|
||||
// produced nothing for this account still shows.
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(31 * HOUR), now: NOW })
|
||||
.freshness,
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(31 * HOUR),
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("due");
|
||||
});
|
||||
|
||||
it("reports whole hours of age", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(90 * 60_000), now: NOW }),
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: ago(90 * 60_000),
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ freshness: "fresh", ageHours: 1 });
|
||||
});
|
||||
|
||||
@@ -93,10 +123,15 @@ describe("classifyOffhostBackup", () => {
|
||||
// as zero so the thresholds under test are the caller's period alone.
|
||||
const twelveHours = ago(12 * HOUR);
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: twelveHours, now: NOW }).freshness,
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
}).freshness,
|
||||
).toBe("fresh");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
periodHours: 6,
|
||||
@@ -105,6 +140,7 @@ describe("classifyOffhostBackup", () => {
|
||||
).toBe("due");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
periodHours: 5,
|
||||
@@ -116,6 +152,7 @@ describe("classifyOffhostBackup", () => {
|
||||
it("does not call a copy stale because the host clock ran backwards", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: ago(0),
|
||||
lastSuccessAt: new Date(NOW.getTime() + 2 * HOUR),
|
||||
now: NOW,
|
||||
}),
|
||||
|
||||
@@ -334,16 +334,24 @@ describe("runOffhostBackup", () => {
|
||||
|
||||
expect(report.uploaded).toBe(1);
|
||||
expect(report.failed).toBe(1);
|
||||
// Both accounts are recorded as walked — that is what separates "a run
|
||||
// looked and found nothing" from "no run has looked at this account" —
|
||||
// but only the one whose object landed carries a success.
|
||||
type Upsert = {
|
||||
where: { userId: string };
|
||||
update: Record<string, unknown>;
|
||||
create: Record<string, unknown>;
|
||||
};
|
||||
const calls = upsert.mock.calls.map((call) => call[0] as Upsert);
|
||||
expect(calls.map((call) => call.where.userId)).toEqual(["u1", "u2"]);
|
||||
expect(calls[0]?.update).toHaveProperty("lastSuccessAt");
|
||||
// The guarantee an operator leans on: an account whose upload failed keeps
|
||||
// whatever `lastSuccessAt` it already had. A failed upload writes no
|
||||
// success into the ledger, so yesterday's good copy is still what the
|
||||
// console reports — the run does not overwrite it with today's failure.
|
||||
expect(upsert).toHaveBeenCalledTimes(1);
|
||||
expect(
|
||||
upsert.mock.calls.map(
|
||||
(call) => (call[0] as { where: { userId: string } }).where.userId,
|
||||
),
|
||||
).toEqual(["u1"]);
|
||||
// whatever `lastSuccessAt` it already had. The failed arm names only the
|
||||
// walk instant, so yesterday's good copy is still what the console
|
||||
// reports — the run does not overwrite it with today's failure.
|
||||
expect(calls[1]?.update).not.toHaveProperty("lastSuccessAt");
|
||||
expect(calls[1]?.update).not.toHaveProperty("sizeBytes");
|
||||
expect(calls[1]?.create).not.toHaveProperty("lastSuccessAt");
|
||||
});
|
||||
|
||||
it("still reports a landed object as uploaded when the ledger write fails", async () => {
|
||||
@@ -368,6 +376,7 @@ describe("runOffhostBackup", () => {
|
||||
expect(report.failed).toBe(0);
|
||||
expect(report.failures).toEqual([]);
|
||||
expect(s3.store.has("2026-05-08/user-u1.json.enc")).toBe(true);
|
||||
expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("refuses an account whose object outgrows one multipart upload", async () => {
|
||||
|
||||
@@ -14,10 +14,16 @@
|
||||
* same whether or not the app process holds a read grant on the bucket, and a
|
||||
* listing of somebody's backups never crosses the network to show a page.
|
||||
*
|
||||
* Four states, because the run is nightly and one missed night is not the same
|
||||
* event as four:
|
||||
* Five states, because the run is nightly and one missed night is not the same
|
||||
* event as four — and because "we have not looked yet" is not a verdict:
|
||||
*
|
||||
* never — the worker has never put an object there for this account.
|
||||
* unknown — no nightly run has recorded this account. The ledger starts
|
||||
* empty, so this is what every account on a perfectly healthy host
|
||||
* reads between the upgrade that adds the table and the first run
|
||||
* after it. Saying "never" there would assert something the host
|
||||
* cannot know.
|
||||
* never — a run walked this account and put no object there, and no earlier
|
||||
* run ever did either.
|
||||
* fresh — inside one schedule period plus the grace below. This is what
|
||||
* every account looks like on a host where the cron is doing its
|
||||
* job.
|
||||
@@ -54,7 +60,8 @@ export const OFFHOST_BACKUP_PERIOD_HOURS = 24;
|
||||
*/
|
||||
export const OFFHOST_BACKUP_GRACE_HOURS = 6;
|
||||
|
||||
export type OffhostBackupFreshness = "never" | "fresh" | "due" | "stale";
|
||||
export type OffhostBackupFreshness =
|
||||
"unknown" | "never" | "fresh" | "due" | "stale";
|
||||
|
||||
export interface OffhostBackupVerdict {
|
||||
freshness: OffhostBackupFreshness;
|
||||
@@ -63,6 +70,11 @@ export interface OffhostBackupVerdict {
|
||||
}
|
||||
|
||||
export function classifyOffhostBackup(args: {
|
||||
/**
|
||||
* When a run last walked this account, or null when none has. Null is the
|
||||
* only thing that separates `unknown` from `never`.
|
||||
*/
|
||||
lastAttemptAt: Date | null;
|
||||
lastSuccessAt: Date | null;
|
||||
now: Date;
|
||||
/** Defaults to the nightly schedule. Tests pass their own. */
|
||||
@@ -70,7 +82,8 @@ export function classifyOffhostBackup(args: {
|
||||
/** Defaults to `OFFHOST_BACKUP_GRACE_HOURS`. Tests pass their own. */
|
||||
graceHours?: number;
|
||||
}): OffhostBackupVerdict {
|
||||
const { lastSuccessAt, now } = args;
|
||||
const { lastAttemptAt, lastSuccessAt, now } = args;
|
||||
if (lastAttemptAt === null) return { freshness: "unknown", ageHours: null };
|
||||
if (lastSuccessAt === null) return { freshness: "never", ageHours: null };
|
||||
|
||||
const periodMs =
|
||||
|
||||
@@ -615,32 +615,49 @@ export async function runOffhostBackup(
|
||||
// by here the object is durably in the bucket and this account's backup
|
||||
// has succeeded whatever the database does next. A pool timeout on the
|
||||
// row would otherwise un-count a copy that exists — the run would report
|
||||
// one failure too many and the console would read `never` for an account
|
||||
// one failure too many and the console would read stale for an account
|
||||
// whose disaster-recovery object is fine, which is exactly the inversion
|
||||
// the ledger exists to remove.
|
||||
//
|
||||
// The row is written whether or not an object came of the walk, because a
|
||||
// ledger keyed only on success cannot tell "no run has reached this
|
||||
// account" from "a run reached it and nothing landed" — and the first of
|
||||
// those is what every account on a running host looks like the day this
|
||||
// table ships. A failed walk touches `lastAttemptAt` and leaves
|
||||
// `lastSuccessAt` exactly as it was, so yesterday's good copy still reads
|
||||
// as the copy this account has.
|
||||
//
|
||||
// `new Date()` rather than the run's `now`, because on a large cohort the
|
||||
// two are hours apart and the row is meant to say when this account's
|
||||
// object landed.
|
||||
if (objectBytes !== null) {
|
||||
const bytes = BigInt(objectBytes);
|
||||
try {
|
||||
await prisma.offhostBackupState.upsert({
|
||||
where: { userId: user.id },
|
||||
update: { lastSuccessAt: new Date(), sizeBytes: bytes },
|
||||
create: {
|
||||
userId: user.id,
|
||||
lastSuccessAt: new Date(),
|
||||
sizeBytes: bytes,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
ledgerWriteFailures++;
|
||||
const message = (err as Error).message ?? "unknown";
|
||||
evt?.addWarning(
|
||||
`offhost-backup ledger write failed for ${user.id}: ${message.slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
// two are hours apart and the row is meant to say when this account was
|
||||
// reached.
|
||||
const at = new Date();
|
||||
try {
|
||||
await prisma.offhostBackupState.upsert({
|
||||
where: { userId: user.id },
|
||||
update:
|
||||
objectBytes === null
|
||||
? { lastAttemptAt: at }
|
||||
: {
|
||||
lastAttemptAt: at,
|
||||
lastSuccessAt: at,
|
||||
sizeBytes: BigInt(objectBytes),
|
||||
},
|
||||
create:
|
||||
objectBytes === null
|
||||
? { userId: user.id, lastAttemptAt: at }
|
||||
: {
|
||||
userId: user.id,
|
||||
lastAttemptAt: at,
|
||||
lastSuccessAt: at,
|
||||
sizeBytes: BigInt(objectBytes),
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
ledgerWriteFailures++;
|
||||
const message = (err as Error).message ?? "unknown";
|
||||
evt?.addWarning(
|
||||
`offhost-backup ledger write failed for ${user.id}: ${message.slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (ledgerWriteFailures > 0) {
|
||||
|
||||
@@ -29,6 +29,12 @@ export interface BackupRow {
|
||||
export interface OffhostAccountRow {
|
||||
userId: string;
|
||||
username: string;
|
||||
/**
|
||||
* ISO instant a run last walked this account, or null when none has. Null
|
||||
* is what the whole cohort reads until the first run after the upgrade that
|
||||
* adds the ledger, and the card says so rather than claiming `never`.
|
||||
*/
|
||||
lastAttemptAt: string | null;
|
||||
/** ISO instant the newest object landed, or null when there is none. */
|
||||
lastSuccessAt: string | null;
|
||||
/** Size of that object in bytes, or null when there is none. */
|
||||
|
||||
@@ -109,6 +109,7 @@ describe("off-host backup ledger (real Postgres)", () => {
|
||||
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: row.lastAttemptAt,
|
||||
lastSuccessAt: row.lastSuccessAt,
|
||||
now: new Date(),
|
||||
}).freshness,
|
||||
@@ -144,12 +145,80 @@ describe("off-host backup ledger (real Postgres)", () => {
|
||||
});
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: row.lastAttemptAt,
|
||||
lastSuccessAt: row.lastSuccessAt,
|
||||
now: new Date(),
|
||||
}).freshness,
|
||||
).toBe("stale");
|
||||
});
|
||||
|
||||
it("separates an account no run has recorded from one a run found nothing for", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
await seedUser("account-one");
|
||||
|
||||
// Before any run: the ledger is empty, which is what every account on a
|
||||
// host that has been uploading for months looks like the moment this
|
||||
// table ships. It must not read as "nothing has ever reached the bucket".
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: null,
|
||||
lastSuccessAt: null,
|
||||
now: new Date(),
|
||||
}).freshness,
|
||||
).toBe("unknown");
|
||||
|
||||
// A run that reaches the account and produces no object records the walk
|
||||
// and leaves the success columns alone.
|
||||
const bucket = makeBucket();
|
||||
bucket.putStream = async () => {
|
||||
throw new Error("bucket refused the signature");
|
||||
};
|
||||
const report = await runOffhostBackup(prisma, bucket, new Date());
|
||||
expect(report.uploaded).toBe(0);
|
||||
expect(report.failed).toBe(1);
|
||||
|
||||
const row = await prisma.offhostBackupState.findUniqueOrThrow({
|
||||
where: { userId: "account-one" },
|
||||
});
|
||||
expect(row.lastSuccessAt).toBeNull();
|
||||
expect(row.sizeBytes).toBeNull();
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastAttemptAt: row.lastAttemptAt,
|
||||
lastSuccessAt: row.lastSuccessAt,
|
||||
now: new Date(),
|
||||
}).freshness,
|
||||
).toBe("never");
|
||||
});
|
||||
|
||||
it("keeps the last good copy on the row when a later run fails", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
await seedUser("account-one");
|
||||
|
||||
const bucket = makeBucket();
|
||||
await runOffhostBackup(prisma, bucket, new Date());
|
||||
const good = await prisma.offhostBackupState.findUniqueOrThrow({
|
||||
where: { userId: "account-one" },
|
||||
});
|
||||
expect(good.lastSuccessAt).not.toBeNull();
|
||||
|
||||
bucket.putStream = async () => {
|
||||
throw new Error("bucket refused the signature");
|
||||
};
|
||||
await runOffhostBackup(prisma, bucket, new Date());
|
||||
|
||||
const after = await prisma.offhostBackupState.findUniqueOrThrow({
|
||||
where: { userId: "account-one" },
|
||||
});
|
||||
// The failed run moved the walk instant and nothing else: yesterday's
|
||||
// copy is still the copy this account has off-host.
|
||||
expect(after.lastSuccessAt?.getTime()).toBe(good.lastSuccessAt?.getTime());
|
||||
expect(after.sizeBytes).toBe(good.sizeBytes);
|
||||
expect(after.lastAttemptAt.getTime()).toBeGreaterThanOrEqual(
|
||||
good.lastAttemptAt.getTime(),
|
||||
);
|
||||
});
|
||||
|
||||
it("goes with the account it describes", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
await seedUser("account-one");
|
||||
|
||||
Reference in new issue
Block a user