mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 23:33:29 +00:00
fix(db-backup): read the off-host leg of the dump on its own
A run that kept its dump on the volume and failed the upload is the newest good dump and the newest failure at the same instant. Compared with ">", the card showed no failure and readiness said fresh, night after night, while no dump left the host. The health read now carries the off-host leg separately: when the newest dump reached the bucket, and the newest one that did not. One pure reading of it (databaseDumpProblems) feeds both the backups card and the readiness model. A configured bucket that missed the newest dump is attention (offhost_failed) with the upload's error, and failing once the newest dump there is older than 36 hours (offhost_stale) or none ever arrived (offhost_never). The card shows the refused upload and when the bucket last got a dump. A failure at the instant of the newest dump for another reason, retention throwing, is now shown too.
This commit is contained in:
1 parent
4887ed8071
commit
664b46d2c6
18 files changed
+427
-37
No files matched your search
@@ -29,6 +29,7 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import { evaluateDatabaseDump } from "@/lib/recovery/readiness-model";
|
||||
import { runDatabaseBackup } from "@/lib/database-backup/run";
|
||||
import {
|
||||
connect,
|
||||
@@ -391,6 +392,61 @@ describe("the nightly database dump", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// A dump kept on the volume and refused by the bucket is the newest good
|
||||
// dump and the newest failure at once. Read as only those two it looked
|
||||
// healthy, night after night, while nothing left the host.
|
||||
it("does not read a kept dump as healthy when the bucket refused it", async () => {
|
||||
const { mkdirSync } = await import("node:fs");
|
||||
const dir = join(work, "upload-refused");
|
||||
mkdirSync(dir);
|
||||
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid");
|
||||
vi.stubEnv("BACKUP_S3_BUCKET", "b");
|
||||
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
|
||||
vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
|
||||
vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY);
|
||||
const refusing = {
|
||||
putStream: async (_key: string, body: Readable) => {
|
||||
for await (const _ of body) void _;
|
||||
throw new Error("AccessDenied on database/");
|
||||
},
|
||||
};
|
||||
const report = await runDatabaseBackup(prisma, {
|
||||
now: new Date("2026-10-24T07:00:00Z"),
|
||||
env: backupEnv({ DB_BACKUP_DIR: dir }),
|
||||
s3: refusing as never,
|
||||
});
|
||||
vi.unstubAllEnvs();
|
||||
expect(report).toMatchObject({
|
||||
status: "failed",
|
||||
local: "written",
|
||||
offhost: "failed",
|
||||
});
|
||||
|
||||
const health = await getDatabaseBackupHealth(
|
||||
new Date(),
|
||||
backupEnv({ DB_BACKUP_DIR: dir }),
|
||||
);
|
||||
expect(health.lastGood).toMatchObject({
|
||||
fileName: "healthlog-20261024T070000Z-daily.dump",
|
||||
local: true,
|
||||
offhost: false,
|
||||
});
|
||||
expect(health.offhostConfigured).toBe(true);
|
||||
expect(health.offhost.lastUploadAt).not.toBeNull();
|
||||
expect(health.offhost.failure?.error).toMatch(/AccessDenied/);
|
||||
expect(
|
||||
evaluateDatabaseDump({
|
||||
now: new Date(),
|
||||
dump: health,
|
||||
dumpWatchedSince: null,
|
||||
}),
|
||||
).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "offhost_failed",
|
||||
detail: expect.stringMatching(/AccessDenied/),
|
||||
});
|
||||
});
|
||||
|
||||
describe("restoring it", () => {
|
||||
const target = `restore_${process.pid}_${Date.now()}`;
|
||||
let targetUrl = "";
|
||||
|
||||
@@ -44,6 +44,10 @@ const dumpGoingOffhost = async (): Promise<DatabaseBackupHealth> => ({
|
||||
offhost: true,
|
||||
},
|
||||
lastFailure: null,
|
||||
offhost: {
|
||||
lastUploadAt: new Date(NOW.getTime() - 3_600_000).toISOString(),
|
||||
failure: null,
|
||||
},
|
||||
});
|
||||
|
||||
function useBucket(): void {
|
||||
|
||||
Reference in new issue
Block a user