fix(db-backup): read the off-host leg of the dump on its own

A run that kept its dump on the volume and failed the upload is the newest
good dump and the newest failure at the same instant. Compared with ">", the
card showed no failure and readiness said fresh, night after night, while no
dump left the host.

The health read now carries the off-host leg separately: when the newest dump
reached the bucket, and the newest one that did not. One pure reading of it
(databaseDumpProblems) feeds both the backups card and the readiness model.
A configured bucket that missed the newest dump is attention (offhost_failed)
with the upload's error, and failing once the newest dump there is older than
36 hours (offhost_stale) or none ever arrived (offhost_never). The card shows
the refused upload and when the bucket last got a dump. A failure at the
instant of the newest dump for another reason, retention throwing, is now
shown too.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 16:53:24 +01:00
1 parent 4887ed8071
commit 664b46d2c6
18 files changed
+427 -37

No files matched your search

+56
View File
@@ -29,6 +29,7 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { prisma } from "@/lib/db";
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
import { evaluateDatabaseDump } from "@/lib/recovery/readiness-model";
import { runDatabaseBackup } from "@/lib/database-backup/run";
import {
connect,
@@ -391,6 +392,61 @@ describe("the nightly database dump", () => {
});
});
// A dump kept on the volume and refused by the bucket is the newest good
// dump and the newest failure at once. Read as only those two it looked
// healthy, night after night, while nothing left the host.
it("does not read a kept dump as healthy when the bucket refused it", async () => {
const { mkdirSync } = await import("node:fs");
const dir = join(work, "upload-refused");
mkdirSync(dir);
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid");
vi.stubEnv("BACKUP_S3_BUCKET", "b");
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY);
const refusing = {
putStream: async (_key: string, body: Readable) => {
for await (const _ of body) void _;
throw new Error("AccessDenied on database/");
},
};
const report = await runDatabaseBackup(prisma, {
now: new Date("2026-10-24T07:00:00Z"),
env: backupEnv({ DB_BACKUP_DIR: dir }),
s3: refusing as never,
});
vi.unstubAllEnvs();
expect(report).toMatchObject({
status: "failed",
local: "written",
offhost: "failed",
});
const health = await getDatabaseBackupHealth(
new Date(),
backupEnv({ DB_BACKUP_DIR: dir }),
);
expect(health.lastGood).toMatchObject({
fileName: "healthlog-20261024T070000Z-daily.dump",
local: true,
offhost: false,
});
expect(health.offhostConfigured).toBe(true);
expect(health.offhost.lastUploadAt).not.toBeNull();
expect(health.offhost.failure?.error).toMatch(/AccessDenied/);
expect(
evaluateDatabaseDump({
now: new Date(),
dump: health,
dumpWatchedSince: null,
}),
).toMatchObject({
state: "attention",
reason: "offhost_failed",
detail: expect.stringMatching(/AccessDenied/),
});
});
describe("restoring it", () => {
const target = `restore_${process.pid}_${Date.now()}`;
let targetUrl = "";
@@ -44,6 +44,10 @@ const dumpGoingOffhost = async (): Promise<DatabaseBackupHealth> => ({
offhost: true,
},
lastFailure: null,
offhost: {
lastUploadAt: new Date(NOW.getTime() - 3_600_000).toISOString(),
failure: null,
},
});
function useBucket(): void {