mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 13:01:00 +00:00
feat(backup): take a nightly whole-database dump with a manifest
The per-account copies roll one account back on the same server; nothing the app made could rebuild a lost one. A scheduled job now runs pg_dump over the whole database (every schema, pg-boss included) into DB_BACKUP_DIR as healthlog-<UTC time>-daily.dump, with a manifest beside it: app version and build, newest migration, Postgres version, size and SHA-256, exact row counts of every table, and the fingerprints of the keys the database needs. The counts read the same exported snapshot pg_dump reads, so a dump taken while the app writes still matches its manifest. The archive is written as .partial, listed back with pg_restore --list, and renamed after its manifest; a failed run leaves nothing named like a good dump. Retention removes only files this job wrote, oldest first, and never the newest good one (DB_BACKUP_KEEP, default 7; two pre-migration dumps). With the off-host bucket configured, each dump and its manifest go up under database/ in the envelope the account copies use, streamed. Every run lands in the new database_backup_runs ledger (migration 0486), and getDatabaseBackupHealth() answers the last good dump, the last failure, whether the directory is writable and whether off-host is configured. The boot readiness summary says where dumps go, or that none can be kept. The dump, naming, retention and restore rules live in one plain ESM module so the image's own scripts can share them without tsx.
This commit is contained in:
Marc-André Bombeck committed
1 parent
27c8533f40
commit
6d3a1582fa
18 files changed
+3025
-9
No files matched your search
@@ -0,0 +1,30 @@
|
||||
-- The nightly whole-database dump records each run: whether a dump was
|
||||
-- written in DB_BACKUP_DIR and copied to the off-host bucket, and why not.
|
||||
-- Instance metadata with no account; the admin backups card and the backup
|
||||
-- alerts read the newest rows.
|
||||
--
|
||||
-- The table is new, so nothing waits on its lock in practice; the wait is
|
||||
-- bounded in a DO block anyway, and every statement is safe to run again, so
|
||||
-- the boot retries the file on a lock timeout.
|
||||
--
|
||||
-- @retry-on-lock-timeout
|
||||
|
||||
DO $migration$
|
||||
BEGIN
|
||||
PERFORM set_config('lock_timeout', '5s', true);
|
||||
CREATE TABLE IF NOT EXISTS "database_backup_runs" (
|
||||
"id" TEXT NOT NULL,
|
||||
"started_at" TIMESTAMP(3) NOT NULL,
|
||||
"finished_at" TIMESTAMP(3),
|
||||
"status" TEXT NOT NULL,
|
||||
"file_name" TEXT,
|
||||
"dump_bytes" BIGINT,
|
||||
"local_result" TEXT NOT NULL,
|
||||
"offhost_result" TEXT NOT NULL,
|
||||
"error" TEXT,
|
||||
CONSTRAINT "database_backup_runs_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS "database_backup_runs_started_at_idx"
|
||||
ON "database_backup_runs"("started_at");
|
||||
END
|
||||
$migration$;
|
||||
@@ -5929,6 +5929,35 @@ model OffhostPurgeRequest {
|
||||
@@map("offhost_purge_requests")
|
||||
}
|
||||
|
||||
/// One run of the nightly whole-database dump (the `database-backup` job):
|
||||
/// whether a dump was written on this server and copied off-host, and why not
|
||||
/// when it was not. Instance metadata, not an account's data. Written by
|
||||
/// `runDatabaseBackup`; read by `getDatabaseBackupHealth`, which feeds the
|
||||
/// database row of the admin backups card and the backup alerts. A row left
|
||||
/// `running` by a process that died is read as a failed run once the job's
|
||||
/// expiry has passed.
|
||||
model DatabaseBackupRun {
|
||||
id String @id @default(cuid())
|
||||
startedAt DateTime @map("started_at")
|
||||
finishedAt DateTime? @map("finished_at")
|
||||
/// `running`, `succeeded`, `failed` or `skipped`.
|
||||
status String
|
||||
/// The dump's file name, once one was written and checked.
|
||||
fileName String? @map("file_name")
|
||||
/// Size of the dump in bytes.
|
||||
dumpBytes BigInt? @map("dump_bytes")
|
||||
/// `written` (kept in DB_BACKUP_DIR), `no_directory` (no writable
|
||||
/// directory, the dump went off-host only), or `failed`.
|
||||
localResult String @map("local_result")
|
||||
/// `uploaded`, `not_configured` or `failed`.
|
||||
offhostResult String @map("offhost_result")
|
||||
/// Why the run failed or was skipped, trimmed. Never key material.
|
||||
error String?
|
||||
|
||||
@@index([startedAt])
|
||||
@@map("database_backup_runs")
|
||||
}
|
||||
|
||||
// ─── Apple Health export.zip import (v1.4.34) ───────────────
|
||||
//
|
||||
// Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the
|
||||
|
||||
@@ -149,3 +149,56 @@ describe("collectReadiness — retired AI grant", () => {
|
||||
expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("collectReadiness — the database dump", () => {
|
||||
const offhost = {
|
||||
BACKUP_S3_ENDPOINT: "https://s3.example.test",
|
||||
BACKUP_S3_BUCKET: "b",
|
||||
BACKUP_S3_ACCESS_KEY: "a",
|
||||
BACKUP_S3_SECRET_KEY: "s",
|
||||
BACKUP_ENCRYPTION_KEY: HEX32,
|
||||
};
|
||||
const dumpLine = (
|
||||
env: Record<string, string | undefined>,
|
||||
writable: boolean,
|
||||
) =>
|
||||
collectReadiness(env, () => writable).lines.find(
|
||||
(l) => l.label === "Database dump",
|
||||
);
|
||||
|
||||
it("names the schedule, the directory and the copies kept", () => {
|
||||
const line = dumpLine(baseEnv(), true);
|
||||
expect(line?.status).toBe("ok");
|
||||
expect(line?.detail).toBe(
|
||||
"nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7",
|
||||
);
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain(
|
||||
"copied off-host",
|
||||
);
|
||||
});
|
||||
|
||||
it("warns when nothing can be kept on the server", () => {
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({
|
||||
status: "warn",
|
||||
detail: expect.stringContaining("goes off-host only"),
|
||||
});
|
||||
expect(dumpLine(baseEnv(), false)).toMatchObject({
|
||||
status: "warn",
|
||||
detail: expect.stringContaining("no database dump can be taken"),
|
||||
});
|
||||
});
|
||||
|
||||
it("warns when it is switched off, and names a schedule it ignored", () => {
|
||||
expect(
|
||||
dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true),
|
||||
).toMatchObject({ status: "warn" });
|
||||
const report = collectReadiness(
|
||||
{ ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" },
|
||||
() => true,
|
||||
);
|
||||
expect(
|
||||
report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail,
|
||||
).toBe("not a five-field cron; using 30 1 * * *");
|
||||
expect(report.hasBlocker).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -19,6 +19,13 @@ import {
|
||||
LEGACY_ANY_HOST_WARNING,
|
||||
legacyAnyHostConfigured,
|
||||
} from "@/lib/ai/local-host-allowlist";
|
||||
import {
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
resolveBackupDir,
|
||||
resolveBackupSchedule,
|
||||
resolveDailyKeep,
|
||||
} from "@/lib/database-backup/dump-core.mjs";
|
||||
|
||||
export interface ReadinessLine {
|
||||
/** Short label, e.g. "ENCRYPTION_KEY". */
|
||||
@@ -60,8 +67,73 @@ function anySet(env: Env, keys: string[]): boolean {
|
||||
return keys.some((k) => (env[k] ?? "").trim().length > 0);
|
||||
}
|
||||
|
||||
/** Inspect the environment and build the readiness report. Pure + side-effect free. */
|
||||
export function collectReadiness(env: Env = process.env): ReadinessReport {
|
||||
/**
|
||||
* The nightly database dump's line. The dump is what rebuilds a lost server,
|
||||
* so a host where it cannot be kept anywhere is said out loud at every boot,
|
||||
* and so is one where it only goes off-host.
|
||||
*/
|
||||
export function databaseBackupReadiness(
|
||||
env: Env,
|
||||
writable: (dir: string) => boolean,
|
||||
): ReadinessLine[] {
|
||||
const label = "Database dump";
|
||||
if (!databaseBackupEnabled(env)) {
|
||||
return [
|
||||
{
|
||||
label,
|
||||
status: "warn",
|
||||
detail:
|
||||
"DB_BACKUP_ENABLED=false: no nightly database dump is taken; back the database up another way",
|
||||
},
|
||||
];
|
||||
}
|
||||
const lines: ReadinessLine[] = [];
|
||||
const schedule = resolveBackupSchedule(env);
|
||||
if (!schedule.valid) {
|
||||
lines.push({
|
||||
label: "DB_BACKUP_SCHEDULE",
|
||||
status: "warn",
|
||||
detail: `not a five-field cron; using ${schedule.cron}`,
|
||||
});
|
||||
}
|
||||
const dir = resolveBackupDir(env);
|
||||
const offhost = allSet(env, [
|
||||
"BACKUP_S3_ENDPOINT",
|
||||
"BACKUP_S3_BUCKET",
|
||||
"BACKUP_S3_ACCESS_KEY",
|
||||
"BACKUP_S3_SECRET_KEY",
|
||||
"BACKUP_ENCRYPTION_KEY",
|
||||
]);
|
||||
if (writable(dir)) {
|
||||
lines.push({
|
||||
label,
|
||||
status: "ok",
|
||||
detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`,
|
||||
});
|
||||
} else if (offhost) {
|
||||
lines.push({
|
||||
label,
|
||||
status: "warn",
|
||||
detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
|
||||
});
|
||||
} else {
|
||||
lines.push({
|
||||
label,
|
||||
status: "warn",
|
||||
detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
|
||||
});
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect the environment and build the readiness report. Pure apart from
|
||||
* `writable`, the probe of the dump directory, which tests replace.
|
||||
*/
|
||||
export function collectReadiness(
|
||||
env: Env = process.env,
|
||||
writable: (dir: string) => boolean = isWritableDirectory,
|
||||
): ReadinessReport {
|
||||
const lines: ReadinessLine[] = [];
|
||||
|
||||
// --- Core secrets (boot blockers) ---------------------------------------
|
||||
@@ -139,6 +211,8 @@ export function collectReadiness(env: Env = process.env): ReadinessReport {
|
||||
});
|
||||
}
|
||||
|
||||
lines.push(...databaseBackupReadiness(env, writable));
|
||||
|
||||
// --- Optional subsystems (informational) --------------------------------
|
||||
const optional: Array<[string, boolean]> = [
|
||||
["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])],
|
||||
|
||||
@@ -353,6 +353,8 @@ export const INSTANCE_SCOPED: Readonly<Record<string, string>> = {
|
||||
HostMetric: "instance-wide host telemetry, not attributable to an account",
|
||||
OffhostBackupKeyUse:
|
||||
"per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account",
|
||||
DatabaseBackupRun:
|
||||
"the instance's ledger of nightly whole-database dumps; describes the server, not any account",
|
||||
OffhostPurgeRequest:
|
||||
"the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account",
|
||||
CycleSymptomCategory:
|
||||
|
||||
@@ -0,0 +1,630 @@
|
||||
/**
|
||||
* The rules of the whole-database dump that need no database: names,
|
||||
* retention, the manifest, key fingerprints, row-count comparison, the
|
||||
* connection the client tools get, and the off-host envelope read back by the
|
||||
* restore's own reader.
|
||||
*
|
||||
* The fingerprints and the envelope are checked against the app's own code
|
||||
* (`src/lib/crypto.ts`, `src/lib/jobs/offhost-backup.ts`), not against a copy
|
||||
* of the rule: the restore script cannot import TypeScript, so these are the
|
||||
* tests that keep its plain-JS reading of both from drifting.
|
||||
*/
|
||||
import {
|
||||
mkdtempSync,
|
||||
createReadStream,
|
||||
rmSync,
|
||||
utimesSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
buildManifest,
|
||||
compareKeyFingerprints,
|
||||
compareRowCounts,
|
||||
connectionFromUrl,
|
||||
decodeKeyMaterial,
|
||||
decryptOffhostFile,
|
||||
dumpBaseName,
|
||||
formatDumpTimestamp,
|
||||
keyFingerprints,
|
||||
listDumpSets,
|
||||
offhostObjectKey,
|
||||
parseDumpFileName,
|
||||
parseManifest,
|
||||
resolveBackupSchedule,
|
||||
resolveDailyKeep,
|
||||
selectForDeletion,
|
||||
type DumpSet,
|
||||
} from "../dump-core.mjs";
|
||||
import type { S3Like } from "@/lib/jobs/offhost-backup";
|
||||
|
||||
/** Every byte of a stream, in order. */
|
||||
async function collect(stream: AsyncIterable<unknown>): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const c of stream) chunks.push(Buffer.from(c as Uint8Array));
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
const HEX_A = "a".repeat(64);
|
||||
const HEX_B = "b".repeat(64);
|
||||
const HEX_C = "c".repeat(64);
|
||||
const HEX_D = "d".repeat(64);
|
||||
|
||||
const dirs: string[] = [];
|
||||
function tempDir(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hl-dump-core-"));
|
||||
dirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0))
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe("dump names", () => {
|
||||
it("names a dump by its UTC instant and kind", () => {
|
||||
const at = new Date("2026-10-25T01:30:07.123Z");
|
||||
expect(formatDumpTimestamp(at)).toBe("20261025T013007Z");
|
||||
expect(dumpBaseName("daily", at)).toBe("healthlog-20261025T013007Z-daily");
|
||||
expect(dumpBaseName("pre-migration", at)).toBe(
|
||||
"healthlog-20261025T013007Z-pre-migration",
|
||||
);
|
||||
});
|
||||
|
||||
it("parses every file it writes, and back to the same instant", () => {
|
||||
const base = "healthlog-20261025T013007Z-daily";
|
||||
const cases: Array<[string, string, string]> = [
|
||||
[`${base}.dump`, "dump", "final"],
|
||||
[`${base}.manifest.json`, "manifest", "final"],
|
||||
[`${base}.dump.partial`, "dump", "partial"],
|
||||
[`${base}.manifest.json.partial`, "manifest", "partial"],
|
||||
[`${base}.dump.enc`, "dump", "encrypted"],
|
||||
[`${base}.manifest.json.enc`, "manifest", "encrypted"],
|
||||
];
|
||||
for (const [name, role, variant] of cases) {
|
||||
const parsed = parseDumpFileName(name);
|
||||
expect(parsed, name).not.toBeNull();
|
||||
expect(parsed?.base).toBe(base);
|
||||
expect(parsed?.kind).toBe("daily");
|
||||
expect(parsed?.role).toBe(role);
|
||||
expect(parsed?.variant).toBe(variant);
|
||||
expect(parsed?.timestamp.toISOString()).toBe("2026-10-25T01:30:07.000Z");
|
||||
}
|
||||
expect(
|
||||
parseDumpFileName("healthlog-20261025T013007Z-pre-migration.dump")?.kind,
|
||||
).toBe("pre-migration");
|
||||
});
|
||||
|
||||
it("refuses every name it did not write", () => {
|
||||
for (const name of [
|
||||
"healthlog.dump",
|
||||
"backup-20261025T013007Z-daily.dump",
|
||||
"healthlog-20261025T013007Z-weekly.dump",
|
||||
"healthlog-20261025-daily.dump",
|
||||
"healthlog-20261025T013007Z-daily.sql",
|
||||
"healthlog-20261025T013007Z-daily.dump.bak",
|
||||
"healthlog-20261399T013007Z-daily.dump",
|
||||
"notes.txt",
|
||||
]) {
|
||||
expect(parseDumpFileName(name), name).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("configuration", () => {
|
||||
it("keeps at least one daily dump and ignores what is not a number", () => {
|
||||
expect(resolveDailyKeep({})).toBe(7);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "14" })).toBe(14);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "0" })).toBe(1);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "-3" })).toBe(7);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "seven" })).toBe(7);
|
||||
});
|
||||
|
||||
it("takes a five-field cron and says when it ignored one", () => {
|
||||
expect(resolveBackupSchedule({})).toEqual({
|
||||
cron: "30 1 * * *",
|
||||
valid: true,
|
||||
configured: false,
|
||||
});
|
||||
expect(
|
||||
resolveBackupSchedule({ DB_BACKUP_SCHEDULE: " 0 2 * * 1-5 " }),
|
||||
).toEqual({
|
||||
cron: "0 2 * * 1-5",
|
||||
valid: true,
|
||||
configured: true,
|
||||
});
|
||||
for (const bad of ["daily", "0 2 * *", "0 2 * * * *", "0 2 * * mon; rm"]) {
|
||||
expect(resolveBackupSchedule({ DB_BACKUP_SCHEDULE: bad })).toEqual({
|
||||
cron: "30 1 * * *",
|
||||
valid: false,
|
||||
configured: true,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/** A dump set as `listDumpSets` reports it, for the pure retention rule. */
|
||||
function set(
|
||||
kind: "daily" | "pre-migration",
|
||||
iso: string,
|
||||
parts: { dump?: boolean; manifest?: boolean; partialAgeMs?: number } = {},
|
||||
now = Date.parse("2026-10-25T12:00:00Z"),
|
||||
): DumpSet {
|
||||
const base = dumpBaseName(kind, new Date(iso));
|
||||
return {
|
||||
base,
|
||||
kind,
|
||||
timestamp: new Date(iso),
|
||||
dump: parts.dump === false ? null : { path: `/b/${base}.dump`, bytes: 1 },
|
||||
manifest: parts.manifest === false ? null : `/b/${base}.manifest.json`,
|
||||
encryptedDump: null,
|
||||
encryptedManifest: null,
|
||||
leftovers:
|
||||
parts.partialAgeMs === undefined
|
||||
? []
|
||||
: [
|
||||
{
|
||||
path: `/b/${base}.dump.partial`,
|
||||
mtimeMs: now - parts.partialAgeMs,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("retention", () => {
|
||||
const now = Date.parse("2026-10-25T12:00:00Z");
|
||||
|
||||
it("keeps the newest N daily dumps and removes the rest, both files each", () => {
|
||||
const sets = [1, 2, 3, 4, 5].map((d) =>
|
||||
set("daily", `2026-10-2${d}T01:30:00Z`),
|
||||
);
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 3, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed.sort()).toEqual(
|
||||
[
|
||||
"/b/healthlog-20261021T013000Z-daily.dump",
|
||||
"/b/healthlog-20261021T013000Z-daily.manifest.json",
|
||||
"/b/healthlog-20261022T013000Z-daily.dump",
|
||||
"/b/healthlog-20261022T013000Z-daily.manifest.json",
|
||||
].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("never removes the newest good dump, whatever keep says", () => {
|
||||
const sets = [
|
||||
set("daily", "2026-10-24T01:30:00Z"),
|
||||
set("daily", "2026-10-23T01:30:00Z"),
|
||||
];
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 0, "pre-migration": 0 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).not.toContain("/b/healthlog-20261024T013000Z-daily.dump");
|
||||
expect(doomed).toContain("/b/healthlog-20261023T013000Z-daily.dump");
|
||||
});
|
||||
|
||||
it("counts only good dumps: an unfinished newer one does not push out the newest good one", () => {
|
||||
const sets = [
|
||||
set("daily", "2026-10-25T01:30:00Z", {
|
||||
dump: false,
|
||||
partialAgeMs: 60_000,
|
||||
}),
|
||||
set("daily", "2026-10-24T01:30:00Z"),
|
||||
];
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 1, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps two pre-migration dumps apart from the daily count", () => {
|
||||
const sets = [
|
||||
set("pre-migration", "2026-10-20T08:00:00Z"),
|
||||
set("pre-migration", "2026-10-10T08:00:00Z"),
|
||||
set("pre-migration", "2026-10-01T08:00:00Z"),
|
||||
set("daily", "2026-10-24T01:30:00Z"),
|
||||
];
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 1, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).toEqual([
|
||||
"/b/healthlog-20261001T080000Z-pre-migration.dump",
|
||||
"/b/healthlog-20261001T080000Z-pre-migration.manifest.json",
|
||||
]);
|
||||
});
|
||||
|
||||
it("removes a .partial only once it is older than twelve hours", () => {
|
||||
const fresh = set("daily", "2026-10-25T01:30:00Z", {
|
||||
dump: false,
|
||||
manifest: false,
|
||||
partialAgeMs: 60 * 60_000,
|
||||
});
|
||||
const stale = set("daily", "2026-10-23T01:30:00Z", {
|
||||
dump: false,
|
||||
manifest: false,
|
||||
partialAgeMs: 13 * 60 * 60_000,
|
||||
});
|
||||
const doomed = selectForDeletion([fresh, stale], {
|
||||
keep: { daily: 7, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).toEqual([
|
||||
"/b/healthlog-20261023T013000Z-daily.dump.partial",
|
||||
]);
|
||||
});
|
||||
|
||||
it("works on a real directory and leaves files it did not write alone", async () => {
|
||||
const dir = tempDir();
|
||||
const names: string[] = [];
|
||||
for (let day = 15; day <= 24; day++) {
|
||||
const base = `healthlog-202610${day}T013000Z-daily`;
|
||||
names.push(`${base}.dump`, `${base}.manifest.json`);
|
||||
}
|
||||
// Two that are not ours, and an off-host copy an operator put here.
|
||||
names.push(
|
||||
"my-own-dump.dump",
|
||||
"healthlog-20261001T013000Z-daily.dump.bak",
|
||||
"healthlog-20261002T013000Z-daily.dump.enc",
|
||||
);
|
||||
for (const name of names) writeFileSync(join(dir, name), "x");
|
||||
// A crash's leftovers: one old, one from a run that may still be going.
|
||||
const oldPartial = join(
|
||||
dir,
|
||||
"healthlog-20261003T013000Z-daily.dump.partial",
|
||||
);
|
||||
writeFileSync(oldPartial, "x");
|
||||
const day = 24 * 60 * 60;
|
||||
utimesSync(oldPartial, Date.now() / 1000 - day, Date.now() / 1000 - day);
|
||||
writeFileSync(
|
||||
join(dir, "healthlog-20261025T013000Z-daily.dump.partial"),
|
||||
"x",
|
||||
);
|
||||
|
||||
const removed = await applyRetention(dir, { dailyKeep: 7 });
|
||||
expect(removed).toHaveLength(7);
|
||||
const left = (await readdir(dir)).sort();
|
||||
expect(left).toContain("my-own-dump.dump");
|
||||
expect(left).toContain("healthlog-20261001T013000Z-daily.dump.bak");
|
||||
expect(left).toContain("healthlog-20261002T013000Z-daily.dump.enc");
|
||||
expect(left).toContain("healthlog-20261025T013000Z-daily.dump.partial");
|
||||
expect(left).not.toContain("healthlog-20261003T013000Z-daily.dump.partial");
|
||||
expect(left.filter((n) => n.endsWith("-daily.dump"))).toEqual(
|
||||
[18, 19, 20, 21, 22, 23, 24].map(
|
||||
(d) => `healthlog-202610${d}T013000Z-daily.dump`,
|
||||
),
|
||||
);
|
||||
expect(listDumpSets(dir)[1].base).toBe("healthlog-20261024T013000Z-daily");
|
||||
});
|
||||
});
|
||||
|
||||
describe("key fingerprints", () => {
|
||||
it("are the fingerprints the app's own key code gives, for the map and the single key", async () => {
|
||||
vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ v1: HEX_A, v2: HEX_B }));
|
||||
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "v2");
|
||||
vi.stubEnv("ENCRYPTION_KEY", "");
|
||||
const crypto = await import("@/lib/crypto");
|
||||
crypto._resetCryptoCacheForTests();
|
||||
const prints = keyFingerprints(process.env);
|
||||
expect(prints.encryption.activeKeyId).toBe("v2");
|
||||
expect(prints.encryption.keys).toEqual([
|
||||
{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") },
|
||||
{ id: "v2", fingerprint: crypto.getKeyFingerprint("v2") },
|
||||
]);
|
||||
|
||||
vi.stubEnv("ENCRYPTION_KEYS", "");
|
||||
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "");
|
||||
const base64 = Buffer.from(HEX_C, "hex").toString("base64");
|
||||
vi.stubEnv("ENCRYPTION_KEY", base64);
|
||||
crypto._resetCryptoCacheForTests();
|
||||
const single = keyFingerprints(process.env);
|
||||
expect(single.encryption).toEqual({
|
||||
activeKeyId: "v1",
|
||||
keys: [{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }],
|
||||
});
|
||||
crypto._resetCryptoCacheForTests();
|
||||
});
|
||||
|
||||
it("names the off-host key the way an off-host object records it", async () => {
|
||||
const { offhostKeyId } = await import("@/lib/jobs/offhost-backup");
|
||||
const prints = keyFingerprints({
|
||||
ENCRYPTION_KEY: HEX_A,
|
||||
BACKUP_ENCRYPTION_KEY: HEX_D,
|
||||
BACKUP_ENCRYPTION_PREVIOUS_KEYS: ` ${HEX_C} ,`,
|
||||
API_TOKEN_HMAC_KEY: "hmac-secret-of-some-length-0123456789",
|
||||
});
|
||||
expect(prints.backupEncryptionKey).toBe(
|
||||
offhostKeyId(Buffer.from(HEX_D, "hex")),
|
||||
);
|
||||
expect(prints.backupEncryptionPreviousKeys).toEqual([
|
||||
offhostKeyId(Buffer.from(HEX_C, "hex")),
|
||||
]);
|
||||
expect(prints.apiTokenHmacKey).toMatch(/^[0-9a-f]{12}$/);
|
||||
});
|
||||
|
||||
it("never carries key material", () => {
|
||||
const env = {
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A }),
|
||||
BACKUP_ENCRYPTION_KEY: HEX_B,
|
||||
API_TOKEN_HMAC_KEY: HEX_C,
|
||||
};
|
||||
const text = JSON.stringify(keyFingerprints(env));
|
||||
for (const secret of [HEX_A, HEX_B, HEX_C]) {
|
||||
expect(text).not.toContain(secret.slice(0, 16));
|
||||
}
|
||||
});
|
||||
|
||||
it("decodes keys the way the app does, and nothing that is not a key", () => {
|
||||
expect(decodeKeyMaterial(HEX_A)?.length).toBe(32);
|
||||
expect(
|
||||
decodeKeyMaterial(Buffer.alloc(32, 7).toString("base64"))?.length,
|
||||
).toBe(32);
|
||||
expect(decodeKeyMaterial("")).toBeNull();
|
||||
expect(decodeKeyMaterial("not a key!")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("comparing a dump's keys with the environment's", () => {
|
||||
const written = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_B }),
|
||||
ENCRYPTION_ACTIVE_KEY_ID: "v2",
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-one",
|
||||
});
|
||||
|
||||
it("finds nothing when every key is there", () => {
|
||||
const same = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B, v1: HEX_A }),
|
||||
ENCRYPTION_ACTIVE_KEY_ID: "v2",
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-one",
|
||||
});
|
||||
expect(compareKeyFingerprints(written, same)).toEqual({
|
||||
findings: [],
|
||||
refuse: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses when the active key is different, and says which", () => {
|
||||
const wrong = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_D }),
|
||||
ENCRYPTION_ACTIVE_KEY_ID: "v2",
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-one",
|
||||
});
|
||||
const result = compareKeyFingerprints(written, wrong);
|
||||
expect(result.refuse).toBe(true);
|
||||
expect(result.findings).toHaveLength(1);
|
||||
expect(result.findings[0]).toMatchObject({
|
||||
key: "ENCRYPTION_KEYS[v2]",
|
||||
state: "different",
|
||||
severity: "refuse",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses when the active key is missing, warns for a retired one", () => {
|
||||
const onlyOld = keyFingerprints({ ENCRYPTION_KEY: HEX_A });
|
||||
const result = compareKeyFingerprints(written, onlyOld);
|
||||
expect(result.refuse).toBe(true);
|
||||
const byKey = Object.fromEntries(result.findings.map((f) => [f.key, f]));
|
||||
expect(byKey["ENCRYPTION_KEYS[v2]"]).toMatchObject({
|
||||
state: "missing",
|
||||
severity: "refuse",
|
||||
});
|
||||
expect(byKey["ENCRYPTION_KEYS[v1]"]).toBeUndefined();
|
||||
expect(byKey.BACKUP_ENCRYPTION_KEY).toMatchObject({
|
||||
state: "missing",
|
||||
severity: "warn",
|
||||
});
|
||||
expect(byKey.API_TOKEN_HMAC_KEY).toMatchObject({
|
||||
state: "missing",
|
||||
severity: "warn",
|
||||
});
|
||||
|
||||
const noRetired = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B }),
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-two",
|
||||
});
|
||||
const second = compareKeyFingerprints(written, noRetired);
|
||||
expect(second.refuse).toBe(false);
|
||||
expect(second.findings.map((f) => [f.key, f.severity])).toEqual([
|
||||
["ENCRYPTION_KEYS[v1]", "warn"],
|
||||
["API_TOKEN_HMAC_KEY", "warn"],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the manifest", () => {
|
||||
const manifest = buildManifest({
|
||||
kind: "daily",
|
||||
file: "healthlog-20261025T013000Z-daily.dump",
|
||||
createdAt: new Date("2026-10-25T01:30:00Z"),
|
||||
appVersion: "1.46.0",
|
||||
buildSha: "abc1234",
|
||||
latestMigration: "0486_database_backup_runs",
|
||||
server: { version: "18.6", versionNum: 180006 },
|
||||
pgDump: "pg_dump (PostgreSQL) 18.6",
|
||||
archive: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
|
||||
rowCounts: { "public.users": 3, "pgboss.job": 12, "public.a": 0 },
|
||||
env: { ENCRYPTION_KEY: HEX_A, API_TOKEN_HMAC_KEY: "h" },
|
||||
});
|
||||
|
||||
it("carries what a restore and an operator need", () => {
|
||||
expect(manifest).toMatchObject({
|
||||
format: "healthlog-database-dump",
|
||||
formatVersion: 1,
|
||||
kind: "daily",
|
||||
file: "healthlog-20261025T013000Z-daily.dump",
|
||||
createdAt: "2026-10-25T01:30:00.000Z",
|
||||
appVersion: "1.46.0",
|
||||
buildSha: "abc1234",
|
||||
latestMigration: "0486_database_backup_runs",
|
||||
postgres: { serverVersion: "18.6", serverVersionNum: 180006 },
|
||||
dump: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
|
||||
});
|
||||
expect(Object.keys(manifest.rowCounts)).toEqual([
|
||||
"pgboss.job",
|
||||
"public.a",
|
||||
"public.users",
|
||||
]);
|
||||
expect(manifest.keys.encryption.keys[0].id).toBe("v1");
|
||||
});
|
||||
|
||||
it("reads back, and refuses anything else", () => {
|
||||
expect(parseManifest(JSON.parse(JSON.stringify(manifest)))).toEqual(
|
||||
manifest,
|
||||
);
|
||||
expect(() => parseManifest({ format: "something-else" })).toThrow(
|
||||
/not a HealthLog/,
|
||||
);
|
||||
expect(() => parseManifest({ ...manifest, formatVersion: 2 })).toThrow();
|
||||
expect(() => parseManifest(null)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("row counts after a restore", () => {
|
||||
it("names every table that is missing, extra or different", () => {
|
||||
expect(
|
||||
compareRowCounts(
|
||||
{ "public.a": 1, "public.b": 2 },
|
||||
{ "public.a": 1, "public.b": 2 },
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
compareRowCounts(
|
||||
{ "public.a": 1, "public.b": 2, "public.c": 5 },
|
||||
{ "public.a": 1, "public.b": 0, "public.d": 1 },
|
||||
),
|
||||
).toEqual([
|
||||
"public.b: 2 rows in the dump, 0 restored",
|
||||
"public.c: in the dump with 5 rows, missing now",
|
||||
"public.d: restored but not in the dump's manifest",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the connection the client tools get", () => {
|
||||
it("drops Prisma's parameters and keeps the password out of the arguments", () => {
|
||||
const c = connectionFromUrl(
|
||||
"postgresql://health%40log:p%40ss%2Fw@db:6543/hl?schema=app&connection_limit=20&pool_timeout=20&sslmode=require",
|
||||
);
|
||||
expect(c.pgEnv).toEqual({
|
||||
PGHOST: "db",
|
||||
PGPORT: "6543",
|
||||
PGUSER: "health@log",
|
||||
PGPASSWORD: "p@ss/w",
|
||||
PGDATABASE: "hl",
|
||||
PGSSLMODE: "require",
|
||||
});
|
||||
expect(c.schema).toBe("app");
|
||||
const cs = String(c.clientConfig.connectionString);
|
||||
expect(cs).not.toContain("connection_limit");
|
||||
expect(cs).not.toContain("schema=");
|
||||
expect(cs).toContain("sslmode=require");
|
||||
});
|
||||
});
|
||||
|
||||
describe("an off-host copy read back by the restore", () => {
|
||||
/** The uploader the job uses, into memory. */
|
||||
async function upload(objectKey: string, key: Buffer, body: Buffer) {
|
||||
const { uploadEncryptedBackup } = await import("@/lib/jobs/offhost-backup");
|
||||
const stored = new Map<string, Buffer>();
|
||||
const s3: S3Like = {
|
||||
putStream: async (k: string, stream: Readable) => {
|
||||
stored.set(k, await collect(stream));
|
||||
},
|
||||
putObject: async () => {},
|
||||
getObject: async () => Buffer.alloc(0),
|
||||
headObject: async () => false,
|
||||
listObjects: async () => [],
|
||||
deleteObject: async () => {},
|
||||
};
|
||||
await uploadEncryptedBackup(s3, objectKey, key, async (write) => {
|
||||
// Several pieces, as the job streams the file.
|
||||
for (let i = 0; i < body.length; i += 1000) {
|
||||
await write(body.subarray(i, i + 1000));
|
||||
}
|
||||
});
|
||||
return stored.get(objectKey) as Buffer;
|
||||
}
|
||||
|
||||
it("opens what the job's uploader sealed, byte for byte", async () => {
|
||||
const dir = tempDir();
|
||||
const key = Buffer.from(HEX_D, "hex");
|
||||
const body = Buffer.from(Array.from({ length: 50_000 }, (_, i) => i % 251));
|
||||
const name = "healthlog-20261025T013000Z-daily.dump.enc";
|
||||
const sealed = await upload(offhostObjectKey(name), key, body);
|
||||
writeFileSync(join(dir, name), sealed);
|
||||
|
||||
await decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: join(dir, "opened.dump"),
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys: [Buffer.from(HEX_A, "hex"), key],
|
||||
});
|
||||
expect(
|
||||
(await collect(createReadStream(join(dir, "opened.dump")))).equals(body),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("refuses a copy renamed, altered or under a key it does not hold, and leaves nothing", async () => {
|
||||
const dir = tempDir();
|
||||
const key = Buffer.from(HEX_D, "hex");
|
||||
const body = Buffer.from("PGDMP fake archive bytes".repeat(100));
|
||||
const name = "healthlog-20261025T013000Z-daily.dump.enc";
|
||||
const sealed = await upload(offhostObjectKey(name), key, body);
|
||||
writeFileSync(join(dir, name), sealed);
|
||||
const out = join(dir, "opened.dump");
|
||||
|
||||
// Renamed: the object key is bound as associated data.
|
||||
await expect(
|
||||
decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: out,
|
||||
objectKey: offhostObjectKey(
|
||||
"healthlog-20261024T013000Z-daily.dump.enc",
|
||||
),
|
||||
keys: [key],
|
||||
}),
|
||||
).rejects.toThrow(/does not open/);
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
|
||||
// Altered.
|
||||
const altered = Buffer.from(sealed);
|
||||
altered[altered.length - 40] ^= 1;
|
||||
writeFileSync(join(dir, name), altered);
|
||||
await expect(
|
||||
decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: out,
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys: [key],
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
|
||||
// A key it was not written under.
|
||||
writeFileSync(join(dir, name), sealed);
|
||||
await expect(
|
||||
decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: out,
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys: [Buffer.from(HEX_A, "hex")],
|
||||
}),
|
||||
).rejects.toThrow(/neither BACKUP_ENCRYPTION_KEY nor/);
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Whether the nightly whole-database dump is happening, as facts a page or an
|
||||
* alert can act on.
|
||||
*
|
||||
* Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of
|
||||
* this process: whether `DB_BACKUP_DIR` is writable here, and whether the
|
||||
* off-host bucket is configured. A ledger row that says a copy went off-host
|
||||
* outranks this process's environment, for the reason the off-host card gives
|
||||
* (a split install can hand the bucket variables to the worker only).
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
|
||||
import {
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
resolveBackupDir,
|
||||
} from "./dump-core.mjs";
|
||||
|
||||
export interface DatabaseBackupHealth {
|
||||
/** `DB_BACKUP_ENABLED` is not off. */
|
||||
enabled: boolean;
|
||||
/** The directory dumps are kept in (`DB_BACKUP_DIR`). */
|
||||
directory: string;
|
||||
/** This process can write there. False means dumps only go off-host. */
|
||||
localDirWritable: boolean;
|
||||
/** The off-host bucket is configured, or a dump has been uploaded to it. */
|
||||
offhostConfigured: boolean;
|
||||
/** The newest dump that was kept on the server or reached the bucket. */
|
||||
lastGood: {
|
||||
at: string;
|
||||
fileName: string;
|
||||
sizeBytes: number | null;
|
||||
local: boolean;
|
||||
offhost: boolean;
|
||||
} | null;
|
||||
/** The newest run that failed, or that a dying process left unfinished. */
|
||||
lastFailure: { at: string; error: string } | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A run still `running` this long after it started was cut off by its
|
||||
* process: the job's expiry, past which pg-boss has failed it anyway.
|
||||
*/
|
||||
const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000;
|
||||
|
||||
export async function getDatabaseBackupHealth(
|
||||
now: Date = new Date(),
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<DatabaseBackupHealth> {
|
||||
const directory = resolveBackupDir(env);
|
||||
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
|
||||
[
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: {
|
||||
fileName: { not: null },
|
||||
OR: [{ localResult: "written" }, { offhostResult: "uploaded" }],
|
||||
},
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: {
|
||||
startedAt: true,
|
||||
finishedAt: true,
|
||||
fileName: true,
|
||||
dumpBytes: true,
|
||||
localResult: true,
|
||||
offhostResult: true,
|
||||
},
|
||||
}),
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: { status: "failed" },
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true, finishedAt: true, error: true },
|
||||
}),
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: {
|
||||
status: "running",
|
||||
startedAt: { lt: new Date(now.getTime() - ABANDONED_AFTER_MS) },
|
||||
},
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true },
|
||||
}),
|
||||
prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }),
|
||||
],
|
||||
);
|
||||
|
||||
const failures = [
|
||||
lastFailed && {
|
||||
at: (lastFailed.finishedAt ?? lastFailed.startedAt).toISOString(),
|
||||
error: lastFailed.error ?? "failed without a reason",
|
||||
},
|
||||
lastAbandoned && {
|
||||
at: lastAbandoned.startedAt.toISOString(),
|
||||
error: "the run did not finish (the process stopped under it)",
|
||||
},
|
||||
]
|
||||
.filter((f): f is { at: string; error: string } => Boolean(f))
|
||||
.sort((a, b) => b.at.localeCompare(a.at));
|
||||
|
||||
return {
|
||||
enabled: databaseBackupEnabled(env),
|
||||
directory,
|
||||
localDirWritable: isWritableDirectory(directory),
|
||||
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
|
||||
lastGood: lastGood
|
||||
? {
|
||||
at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(),
|
||||
fileName: lastGood.fileName ?? "",
|
||||
sizeBytes:
|
||||
lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes),
|
||||
local: lastGood.localResult === "written",
|
||||
offhost: lastGood.offhostResult === "uploaded",
|
||||
}
|
||||
: null,
|
||||
lastFailure: failures[0] ?? null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* One run of the nightly whole-database dump.
|
||||
*
|
||||
* The dump is the artifact that rebuilds a lost server: every account, every
|
||||
* login, every grant and managed profile, the job queue and the instance
|
||||
* settings, in one `pg_dump --format=custom` file with a manifest beside it
|
||||
* (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the
|
||||
* bundled compose file) and, when the off-host bucket is configured, uploaded
|
||||
* under `database/` in the same envelope the per-account copies use, sealed
|
||||
* with `BACKUP_ENCRYPTION_KEY`.
|
||||
*
|
||||
* Without a writable directory (an install that runs the image without the
|
||||
* volume) the dump is written to a temporary directory, uploaded and removed:
|
||||
* the off-host copy still happens, nothing is kept on the server, and the
|
||||
* ledger and the boot summary say so. With neither, the run is skipped and
|
||||
* recorded as such.
|
||||
*
|
||||
* Every run leaves one `DatabaseBackupRun` row: started, then finished with
|
||||
* what happened on each leg. `getDatabaseBackupHealth` reads them.
|
||||
*/
|
||||
import { createReadStream } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, join } from "node:path";
|
||||
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import {
|
||||
getS3Client,
|
||||
loadOffhostConfig,
|
||||
uploadEncryptedBackup,
|
||||
type S3Like,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import packageJson from "../../../package.json";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
createDatabaseDump,
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
offhostObjectKey,
|
||||
resolveBackupDir,
|
||||
resolveDailyKeep,
|
||||
} from "./dump-core.mjs";
|
||||
|
||||
/** How long ledger rows are kept. A run a day; a year and a bit of history. */
|
||||
const LEDGER_RETENTION_DAYS = 400;
|
||||
|
||||
export type DatabaseBackupLocalResult =
|
||||
"written" | "no_directory" | "failed" | "skipped";
|
||||
export type DatabaseBackupOffhostResult =
|
||||
"uploaded" | "not_configured" | "failed" | "skipped";
|
||||
|
||||
export interface DatabaseBackupReport {
|
||||
status: "succeeded" | "failed" | "skipped";
|
||||
fileName: string | null;
|
||||
dumpBytes: number | null;
|
||||
local: DatabaseBackupLocalResult;
|
||||
offhost: DatabaseBackupOffhostResult;
|
||||
/** Dumps and manifests retention removed. */
|
||||
removed: number;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface RunDatabaseBackupOptions {
|
||||
now?: Date;
|
||||
/** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */
|
||||
deadline?: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** A bucket double for tests; the configured bucket otherwise. */
|
||||
s3?: S3Like;
|
||||
}
|
||||
|
||||
/** The running build's version, as `/api/version` answers it. */
|
||||
function appVersion(env: NodeJS.ProcessEnv): string {
|
||||
return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version;
|
||||
}
|
||||
|
||||
/** An error as one line for the ledger and the admin page. */
|
||||
function reason(err: unknown): string {
|
||||
const text = err instanceof Error ? err.message : String(err);
|
||||
return text.replace(/\s+/g, " ").trim().slice(0, 500);
|
||||
}
|
||||
|
||||
/** Upload one local file as an off-host object, streamed. */
|
||||
async function uploadFile(
|
||||
s3: S3Like,
|
||||
key: Buffer,
|
||||
path: string,
|
||||
): Promise<number> {
|
||||
return uploadEncryptedBackup(
|
||||
s3,
|
||||
offhostObjectKey(`${basename(path)}.enc`),
|
||||
key,
|
||||
async (write) => {
|
||||
for await (const chunk of createReadStream(path)) {
|
||||
await write(chunk as Buffer);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Take tonight's dump, keep it, copy it off-host, prune the old ones, and
|
||||
* record what happened. Never throws: a failure is the report's `status`.
|
||||
*/
|
||||
export async function runDatabaseBackup(
|
||||
prisma: PrismaClient,
|
||||
options: RunDatabaseBackupOptions = {},
|
||||
): Promise<DatabaseBackupReport> {
|
||||
const env = options.env ?? process.env;
|
||||
const now = options.now ?? new Date();
|
||||
const dir = resolveBackupDir(env);
|
||||
const localWritable = isWritableDirectory(dir);
|
||||
const offhostConfig = (() => {
|
||||
try {
|
||||
return loadOffhostConfig();
|
||||
} catch {
|
||||
// A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the
|
||||
// same value with their own message; here it reads as an upload error.
|
||||
return "invalid" as const;
|
||||
}
|
||||
})();
|
||||
|
||||
// Each leg reads "failed" until it has succeeded, so a run the process died
|
||||
// under says so once its expiry has passed.
|
||||
const report: DatabaseBackupReport = {
|
||||
status: "failed",
|
||||
fileName: null,
|
||||
dumpBytes: null,
|
||||
local: localWritable ? "failed" : "no_directory",
|
||||
offhost: offhostConfig ? "failed" : "not_configured",
|
||||
removed: 0,
|
||||
error: null,
|
||||
};
|
||||
const run = await prisma.databaseBackupRun.create({
|
||||
data: {
|
||||
startedAt: now,
|
||||
status: "running",
|
||||
localResult: report.local,
|
||||
offhostResult: report.offhost,
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
let scratch: string | null = null;
|
||||
try {
|
||||
if (!databaseBackupEnabled(env)) {
|
||||
report.status = "skipped";
|
||||
report.local = "skipped";
|
||||
report.offhost = "skipped";
|
||||
report.error = "DB_BACKUP_ENABLED is false";
|
||||
} else if (!localWritable && !offhostConfig) {
|
||||
report.status = "skipped";
|
||||
report.error = `no writable backup directory (${dir}) and no off-host bucket configured`;
|
||||
} else {
|
||||
const target = localWritable
|
||||
? dir
|
||||
: (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-")));
|
||||
const dump = await createDatabaseDump({
|
||||
databaseUrl: env.DATABASE_URL ?? "",
|
||||
dir: target,
|
||||
kind: "daily",
|
||||
now,
|
||||
env,
|
||||
appVersion: appVersion(env),
|
||||
buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null,
|
||||
deadline: options.deadline,
|
||||
});
|
||||
if (localWritable) report.local = "written";
|
||||
report.fileName = basename(dump.dumpPath);
|
||||
report.dumpBytes = dump.manifest.dump.bytes;
|
||||
|
||||
let uploadError: string | null = null;
|
||||
if (offhostConfig === "invalid") {
|
||||
uploadError =
|
||||
"BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64";
|
||||
} else if (offhostConfig) {
|
||||
try {
|
||||
const s3 = options.s3 ?? (await getS3Client(offhostConfig));
|
||||
// The manifest goes up first: a dump object in the bucket always
|
||||
// has the manifest a restore checks it against.
|
||||
await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath);
|
||||
await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath);
|
||||
report.offhost = "uploaded";
|
||||
} catch (err) {
|
||||
uploadError = `off-host upload failed: ${reason(err)}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (localWritable) {
|
||||
const removed = await applyRetention(dir, {
|
||||
dailyKeep: resolveDailyKeep(env),
|
||||
now: now.getTime(),
|
||||
});
|
||||
report.removed = removed.length;
|
||||
}
|
||||
|
||||
// Good when the dump exists somewhere it survives: on the volume, or in
|
||||
// the bucket. An upload that failed beside a kept dump still fails the
|
||||
// run, because the operator asked for both.
|
||||
report.status = uploadError ? "failed" : "succeeded";
|
||||
report.error = uploadError;
|
||||
}
|
||||
} catch (err) {
|
||||
report.status = "failed";
|
||||
report.error = reason(err);
|
||||
} finally {
|
||||
if (scratch) await rm(scratch, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
await prisma.databaseBackupRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
finishedAt: new Date(),
|
||||
status: report.status,
|
||||
fileName: report.fileName,
|
||||
dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes),
|
||||
localResult: report.local,
|
||||
offhostResult: report.offhost,
|
||||
error: report.error,
|
||||
},
|
||||
});
|
||||
await prisma.databaseBackupRun.deleteMany({
|
||||
where: {
|
||||
startedAt: {
|
||||
lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
annotate({
|
||||
action: { name: "database.backup.run" },
|
||||
meta: {
|
||||
status: report.status,
|
||||
local: report.local,
|
||||
offhost: report.offhost,
|
||||
dump_bytes: report.dumpBytes ?? 0,
|
||||
removed: report.removed,
|
||||
},
|
||||
});
|
||||
return report;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* The queue facts of the nightly whole-database dump: its name, its schedule
|
||||
* and its send options. Apart from the job so the worker registry can read
|
||||
* them without loading it (`job-registry-graph.test.ts`).
|
||||
*/
|
||||
import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy";
|
||||
import { resolveBackupSchedule } from "@/lib/database-backup/dump-core.mjs";
|
||||
|
||||
export const DATABASE_BACKUP_QUEUE = "database-backup";
|
||||
|
||||
/**
|
||||
* `DB_BACKUP_SCHEDULE`, read once when the worker loads its registry, in the
|
||||
* zone every cron runs in (`createAndSchedule`, Europe/Berlin). Default 01:30,
|
||||
* an hour before the off-host account copies so the two uploads do not meet.
|
||||
* A value that is not a five-field cron keeps the default, and the boot
|
||||
* readiness summary says so.
|
||||
*/
|
||||
export const DATABASE_BACKUP_CRON = resolveBackupSchedule(process.env).cron;
|
||||
|
||||
/**
|
||||
* Four hours. `pg_dump` of a large instance takes minutes, the upload of the
|
||||
* dump about as long as the dump; the window is there to be impossible to hit
|
||||
* for a working run. The handler kills `pg_dump` at three quarters of it
|
||||
* (`jobDeadline`), so a hung dump ends as a failed run with a reason rather
|
||||
* than an expired job.
|
||||
*/
|
||||
export const DATABASE_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60;
|
||||
|
||||
/**
|
||||
* What the nightly schedule sends with: the expiry, the backups' heartbeat,
|
||||
* and one retry. A dump that failed for a passing reason (a lock it waited
|
||||
* out, a bucket that answered 503) gets a second attempt; one that fails
|
||||
* again waits for the next night, and the failure stays on the admin page.
|
||||
*/
|
||||
export const DATABASE_BACKUP_SEND_OPTIONS = {
|
||||
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
|
||||
heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS,
|
||||
retryLimit: 1,
|
||||
retryDelay: 15 * 60,
|
||||
} as const;
|
||||
@@ -78,6 +78,12 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"continued",
|
||||
"conversation_titles_migrated",
|
||||
"created",
|
||||
// The nightly whole-database dump: where it went and how big.
|
||||
"database_backup_bytes",
|
||||
"database_backup_local",
|
||||
"database_backup_offhost",
|
||||
"database_backup_removed",
|
||||
"database_backup_status",
|
||||
"daily_rows_retired",
|
||||
"days_consolidated",
|
||||
"days_rebuilt",
|
||||
|
||||
@@ -494,9 +494,9 @@ function createEnvelopeStream(
|
||||
return { stream, bytes: () => written };
|
||||
}
|
||||
|
||||
/** Produces the backup JSON in pieces. Every piece is written in order. */
|
||||
/** Produces the plaintext (JSON text, or a dump's bytes) in ordered pieces. */
|
||||
export type BackupJsonProducer = (
|
||||
write: (chunk: string) => Promise<void>,
|
||||
write: (chunk: string | Buffer) => Promise<void>,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export interface UploadBackupOptions {
|
||||
@@ -572,9 +572,9 @@ export async function uploadEncryptedBackup(
|
||||
},
|
||||
);
|
||||
|
||||
const write = async (chunk: string): Promise<void> => {
|
||||
const write = async (chunk: string | Buffer): Promise<void> => {
|
||||
if (failure) throw failure;
|
||||
if (gzip.write(chunk, "utf8")) return;
|
||||
if (gzip.write(chunk, "utf8")) return; // a Buffer ignores the encoding
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = (): void => {
|
||||
gzip.off("error", onError);
|
||||
|
||||
@@ -42,6 +42,7 @@ import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key-
|
||||
import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill";
|
||||
import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run";
|
||||
import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy";
|
||||
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
|
||||
import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue";
|
||||
import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore";
|
||||
import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue";
|
||||
@@ -332,6 +333,17 @@ export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
|
||||
exclusive: "lockedPass",
|
||||
why: "Serialises every account's whole record.",
|
||||
},
|
||||
"database-backup": {
|
||||
runtime: "long",
|
||||
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
|
||||
expiryVia: ["DATABASE_BACKUP_SEND_OPTIONS"],
|
||||
stop: {
|
||||
file: "lib/jobs/reminder/backup-handlers.ts",
|
||||
fn: "handleDatabaseBackup",
|
||||
},
|
||||
exclusive: "lockedPass",
|
||||
why: "Runs pg_dump over the whole database and uploads the dump off-host.",
|
||||
},
|
||||
"data-backup-offhost": {
|
||||
runtime: "long",
|
||||
expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* The housekeeping queues: backups (weekly in-database, nightly off-host),
|
||||
* The housekeeping queues: backups (the nightly whole-database dump, weekly
|
||||
* in-database account copies, nightly off-host),
|
||||
* the restore drill and restores, every ledger and retention cleanup, the
|
||||
* medication maintenance passes, the imports (Apple Health, Health Connect,
|
||||
* medication intake), the converging encryption backfills, the document vault
|
||||
@@ -47,6 +48,11 @@ import {
|
||||
DATA_BACKUP_QUEUE,
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/data-backup-policy";
|
||||
import {
|
||||
DATABASE_BACKUP_CRON,
|
||||
DATABASE_BACKUP_QUEUE,
|
||||
DATABASE_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/database-backup-queue";
|
||||
import {
|
||||
DAWARICH_SYNC_CRON,
|
||||
DAWARICH_SYNC_QUEUE,
|
||||
@@ -300,6 +306,16 @@ const jobs: readonly JobDefinition[] = [
|
||||
lane: true,
|
||||
load: async () => (await backups()).handleDataBackup,
|
||||
}),
|
||||
// The whole-database dump: the copy that rebuilds a lost server. Locked, so
|
||||
// a retry that starts while a run is still dumping waits its turn.
|
||||
defineJob({
|
||||
queue: DATABASE_BACKUP_QUEUE,
|
||||
cron: DATABASE_BACKUP_CRON,
|
||||
scheduleOptions: DATABASE_BACKUP_SEND_OPTIONS,
|
||||
work: { localConcurrency: 1 },
|
||||
lock: () => WHOLE_PASS,
|
||||
load: async () => (await backups()).handleDatabaseBackup,
|
||||
}),
|
||||
prune(
|
||||
RATE_LIMIT_CLEANUP_QUEUE,
|
||||
RATE_LIMIT_CLEANUP_CRON,
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* schedules, and boss.work registrations.
|
||||
*/
|
||||
import { type Job, type JobWithMetadata } from "pg-boss";
|
||||
import { jobBudget } from "@/lib/jobs/job-budget";
|
||||
import { jobBudget, jobDeadline } from "@/lib/jobs/job-budget";
|
||||
import { recordError } from "@/lib/jobs/worker-status";
|
||||
import {
|
||||
BackupBlobTooLargeError,
|
||||
@@ -31,6 +31,37 @@ import {
|
||||
} from "@/lib/jobs/backup-pass-attempts";
|
||||
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
||||
|
||||
/**
|
||||
* The nightly whole-database dump (`database-backup`).
|
||||
*
|
||||
* One `pg_dump` in a child process, so it is not in the heavy lane: the heap
|
||||
* holds a part of the upload at a time, not the database. `pg_dump` is stopped
|
||||
* at three quarters of the job's expiry (`jobDeadline`), so a dump that hangs
|
||||
* on a lock ends as a failed run with its reason rather than an expired job.
|
||||
* A skipped run (switched off, nowhere to put it) completes; the admin page
|
||||
* and the boot summary carry why.
|
||||
*/
|
||||
export async function handleDatabaseBackup(
|
||||
jobs: Job<Record<string, never>>[],
|
||||
): Promise<JobOutcome> {
|
||||
return withBackgroundEvent("job.database_backup", async () => {
|
||||
const { runDatabaseBackup } = await import("@/lib/database-backup/run");
|
||||
const report = await runDatabaseBackup(getWorkerPrisma(), {
|
||||
deadline: jobs[0] ? jobDeadline(jobs[0]) : undefined,
|
||||
});
|
||||
const did = {
|
||||
database_backup_status: report.status,
|
||||
database_backup_local: report.local,
|
||||
database_backup_offhost: report.offhost,
|
||||
database_backup_bytes: report.dumpBytes ?? 0,
|
||||
database_backup_removed: report.removed,
|
||||
};
|
||||
return report.status === "failed"
|
||||
? jobFailed("database dump failed", report.error ?? undefined, did)
|
||||
: jobDone(did);
|
||||
});
|
||||
}
|
||||
|
||||
export interface DataBackupPayload {
|
||||
triggeredAt: string;
|
||||
}
|
||||
|
||||
@@ -99,6 +99,7 @@ const subsystemSurface = {
|
||||
"audit-log-cleanup": { audience: "system" },
|
||||
"step-up-elevation-cleanup": { audience: "system" },
|
||||
"data-backup-offhost": { audience: "system" },
|
||||
"database-backup": { audience: "system" },
|
||||
"data-restore-drill": { audience: "system" },
|
||||
"offhost-backup-purge": { audience: "system" },
|
||||
"backup-restore": { audience: "system" },
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"registeredQueueCount": 122,
|
||||
"registeredQueueCount": 123,
|
||||
"registeredQueues": [
|
||||
"achievement-unlock-sweep",
|
||||
"apple-health-import",
|
||||
@@ -23,6 +23,7 @@
|
||||
"data-backup",
|
||||
"data-backup-offhost",
|
||||
"data-restore-drill",
|
||||
"database-backup",
|
||||
"dawarich-sync",
|
||||
"dense-intraday-hourly-rebuild",
|
||||
"dense-intraday-retention",
|
||||
@@ -455,6 +456,21 @@
|
||||
"retry_delay_max": null,
|
||||
"retry_limit": 2
|
||||
},
|
||||
{
|
||||
"dead_letter": null,
|
||||
"deletion_seconds": 604800,
|
||||
"expire_seconds": 900,
|
||||
"heartbeat_seconds": null,
|
||||
"name": "database-backup",
|
||||
"notify": false,
|
||||
"partition": false,
|
||||
"policy": "standard",
|
||||
"retention_seconds": 1209600,
|
||||
"retry_backoff": false,
|
||||
"retry_delay": 0,
|
||||
"retry_delay_max": null,
|
||||
"retry_limit": 2
|
||||
},
|
||||
{
|
||||
"dead_letter": null,
|
||||
"deletion_seconds": 604800,
|
||||
@@ -2137,6 +2153,21 @@
|
||||
},
|
||||
"timezone": "Europe/Berlin"
|
||||
},
|
||||
{
|
||||
"cron": "30 1 * * *",
|
||||
"data": {},
|
||||
"key": "",
|
||||
"kind": "cron",
|
||||
"name": "database-backup",
|
||||
"options": {
|
||||
"expireInSeconds": 14400,
|
||||
"heartbeatSeconds": 300,
|
||||
"retryDelay": 900,
|
||||
"retryLimit": 1,
|
||||
"tz": "Europe/Berlin"
|
||||
},
|
||||
"timezone": "Europe/Berlin"
|
||||
},
|
||||
{
|
||||
"cron": "20 3 * * *",
|
||||
"data": {},
|
||||
@@ -3022,6 +3053,13 @@
|
||||
"pollingIntervalSeconds": 30
|
||||
}
|
||||
},
|
||||
{
|
||||
"queue": "database-backup",
|
||||
"options": {
|
||||
"localConcurrency": 1,
|
||||
"pollingIntervalSeconds": 30
|
||||
}
|
||||
},
|
||||
{
|
||||
"queue": "dawarich-sync",
|
||||
"options": {
|
||||
|
||||
@@ -0,0 +1,511 @@
|
||||
/**
|
||||
* The nightly whole-database dump against a real Postgres: the job writes a
|
||||
* dump and its manifest, records the run, copies both off-host in the
|
||||
* envelope the restore opens, keeps only what retention allows; and the
|
||||
* restore puts the dump into an empty database and refuses one with tables.
|
||||
*
|
||||
* `pg_dump` and `pg_restore` are the test container's own. A workstation may
|
||||
* have none, or one of another major than the server (which `pg_dump`
|
||||
* refuses), so the test puts two small forwarders on PATH that run the tool
|
||||
* inside the database container with `docker exec`. Everything else, the
|
||||
* snapshot, the counts, the manifest, the ledger and the restore's checks,
|
||||
* runs exactly as in the image.
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Readable } from "node:stream";
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import { runDatabaseBackup } from "@/lib/database-backup/run";
|
||||
import {
|
||||
connect,
|
||||
countRows,
|
||||
decryptOffhostFile,
|
||||
offhostObjectKey,
|
||||
readManifestFile,
|
||||
} from "@/lib/database-backup/dump-core.mjs";
|
||||
|
||||
import { main as restoreMain } from "../../scripts/db-restore.mjs";
|
||||
import { shippedMigrations } from "../../scripts/pending-migrations.mjs";
|
||||
|
||||
const OFFHOST_KEY = "e".repeat(64);
|
||||
const work = mkdtempSync(join(tmpdir(), "hl-db-backup-"));
|
||||
const backups = join(work, "backups");
|
||||
const shims = join(work, "bin");
|
||||
const originalPath = process.env.PATH;
|
||||
|
||||
/** The id of the container that serves DATABASE_URL. */
|
||||
function containerId(): string {
|
||||
const port = new URL(process.env.DATABASE_URL ?? "").port;
|
||||
const rows = execFileSync(
|
||||
"docker",
|
||||
["ps", "--format", "{{.ID}} {{.Ports}}"],
|
||||
{
|
||||
encoding: "utf8",
|
||||
},
|
||||
).split("\n");
|
||||
const row = rows.find((r) => r.includes(`:${port}->5432/tcp`));
|
||||
if (!row) throw new Error(`no container publishes port ${port}`);
|
||||
return row.split(" ")[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* `pg_dump` / `pg_restore` that run inside the database container. The
|
||||
* connection comes from the PG* variables the core sets; host and port become
|
||||
* the container's own. A file argument that exists on this machine is sent on
|
||||
* standard input, which both tools read when no file is named.
|
||||
*/
|
||||
function writeShims(container: string): void {
|
||||
for (const tool of ["pg_dump", "pg_restore"]) {
|
||||
const path = join(shims, tool);
|
||||
writeFileSync(
|
||||
path,
|
||||
`#!/usr/bin/env node
|
||||
const { spawn } = require("node:child_process");
|
||||
const { createReadStream, existsSync } = require("node:fs");
|
||||
const args = process.argv.slice(2);
|
||||
let input = null;
|
||||
if (args.length && !args[args.length - 1].startsWith("-") && existsSync(args[args.length - 1])) {
|
||||
input = args.pop();
|
||||
}
|
||||
const env = [];
|
||||
for (const name of ["PGUSER", "PGPASSWORD", "PGDATABASE"]) {
|
||||
if (process.env[name] !== undefined) env.push("-e", name + "=" + process.env[name]);
|
||||
}
|
||||
env.push("-e", "PGHOST=127.0.0.1", "-e", "PGPORT=5432");
|
||||
const child = spawn("docker", ["exec", "-i", ...env, ${JSON.stringify(container)}, ${JSON.stringify(tool)}, ...args], {
|
||||
stdio: ["pipe", "inherit", "inherit"],
|
||||
});
|
||||
(input ? createReadStream(input) : process.stdin).pipe(child.stdin);
|
||||
child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0)));
|
||||
`,
|
||||
);
|
||||
chmodSync(path, 0o755);
|
||||
}
|
||||
}
|
||||
|
||||
/** A bucket in memory, with the one call the job's uploader uses. */
|
||||
function memoryBucket() {
|
||||
const objects = new Map<string, Buffer>();
|
||||
return {
|
||||
objects,
|
||||
putStream: async (key: string, body: Readable) => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const c of body) chunks.push(c as Buffer);
|
||||
objects.set(key, Buffer.concat(chunks));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function backupEnv(extra: Record<string, string> = {}): NodeJS.ProcessEnv {
|
||||
return { ...process.env, DB_BACKUP_DIR: backups, ...extra };
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
const { mkdirSync } = await import("node:fs");
|
||||
mkdirSync(shims, { recursive: true });
|
||||
mkdirSync(backups, { recursive: true });
|
||||
writeShims(containerId());
|
||||
process.env.PATH = `${shims}:${originalPath}`;
|
||||
for (const name of ["dump-a", "dump-b", "dump-c"]) {
|
||||
await prisma.user.create({
|
||||
data: { username: `${name}-${Date.now()}` },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
process.env.PATH = originalPath;
|
||||
vi.unstubAllEnvs();
|
||||
rmSync(work, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("the nightly database dump", () => {
|
||||
let dumpFile = "";
|
||||
|
||||
it("writes a checked dump and manifest, records the run and copies both off-host", async () => {
|
||||
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid");
|
||||
vi.stubEnv("BACKUP_S3_BUCKET", "b");
|
||||
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
|
||||
vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
|
||||
vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY);
|
||||
const bucket = memoryBucket();
|
||||
const now = new Date("2026-10-20T01:30:00Z");
|
||||
|
||||
const report = await runDatabaseBackup(prisma, {
|
||||
now,
|
||||
env: backupEnv(),
|
||||
s3: bucket as never,
|
||||
});
|
||||
expect(report).toMatchObject({
|
||||
status: "succeeded",
|
||||
local: "written",
|
||||
offhost: "uploaded",
|
||||
error: null,
|
||||
fileName: "healthlog-20261020T013000Z-daily.dump",
|
||||
});
|
||||
dumpFile = join(backups, report.fileName ?? "");
|
||||
|
||||
// Both files, under their final names, and nothing half-written.
|
||||
expect(readdirSync(backups).sort()).toEqual([
|
||||
"healthlog-20261020T013000Z-daily.dump",
|
||||
"healthlog-20261020T013000Z-daily.manifest.json",
|
||||
]);
|
||||
expect(readFileSync(dumpFile).subarray(0, 5).toString("latin1")).toBe(
|
||||
"PGDMP",
|
||||
);
|
||||
|
||||
const manifest = await readManifestFile(
|
||||
join(backups, "healthlog-20261020T013000Z-daily.manifest.json"),
|
||||
);
|
||||
expect(manifest.kind).toBe("daily");
|
||||
expect(manifest.dump.bytes).toBe(report.dumpBytes);
|
||||
expect(manifest.latestMigration).toBe(
|
||||
shippedMigrations(join(process.cwd(), "prisma", "migrations")).at(-1),
|
||||
);
|
||||
expect(manifest.postgres.serverVersionNum).toBeGreaterThanOrEqual(160000);
|
||||
expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count());
|
||||
expect(Object.keys(manifest.rowCounts).length).toBeGreaterThan(100);
|
||||
expect(manifest.keys.encryption.keys.length).toBeGreaterThan(0);
|
||||
expect(manifest.keys.backupEncryptionKey).toMatch(/^[0-9a-f]{12}$/);
|
||||
|
||||
// The ledger row and what the admin page reads from it.
|
||||
const runs = await prisma.databaseBackupRun.findMany({
|
||||
where: { startedAt: now },
|
||||
});
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(runs[0]).toMatchObject({
|
||||
status: "succeeded",
|
||||
fileName: "healthlog-20261020T013000Z-daily.dump",
|
||||
localResult: "written",
|
||||
offhostResult: "uploaded",
|
||||
error: null,
|
||||
});
|
||||
expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes);
|
||||
const health = await getDatabaseBackupHealth(
|
||||
new Date("2026-10-20T02:00:00Z"),
|
||||
backupEnv(),
|
||||
);
|
||||
expect(health).toMatchObject({
|
||||
enabled: true,
|
||||
localDirWritable: true,
|
||||
offhostConfigured: true,
|
||||
lastGood: {
|
||||
fileName: "healthlog-20261020T013000Z-daily.dump",
|
||||
sizeBytes: manifest.dump.bytes,
|
||||
local: true,
|
||||
offhost: true,
|
||||
},
|
||||
});
|
||||
|
||||
// The off-host pair opens, with the restore's own reader, to the very
|
||||
// bytes on disk.
|
||||
expect([...bucket.objects.keys()].sort()).toEqual([
|
||||
"database/healthlog-20261020T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261020T013000Z-daily.manifest.json.enc",
|
||||
]);
|
||||
const opened = join(work, "opened");
|
||||
const { mkdirSync } = await import("node:fs");
|
||||
mkdirSync(opened);
|
||||
for (const name of [
|
||||
"healthlog-20261020T013000Z-daily.dump",
|
||||
"healthlog-20261020T013000Z-daily.manifest.json",
|
||||
]) {
|
||||
writeFileSync(
|
||||
join(opened, `${name}.enc`),
|
||||
bucket.objects.get(offhostObjectKey(`${name}.enc`)) as Buffer,
|
||||
);
|
||||
await decryptOffhostFile({
|
||||
inPath: join(opened, `${name}.enc`),
|
||||
outPath: join(opened, name),
|
||||
objectKey: offhostObjectKey(`${name}.enc`),
|
||||
keys: [Buffer.from(OFFHOST_KEY, "hex")],
|
||||
});
|
||||
expect(
|
||||
readFileSync(join(opened, name)).equals(
|
||||
readFileSync(join(backups, name)),
|
||||
),
|
||||
).toBe(true);
|
||||
}
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("keeps DB_BACKUP_KEEP nightly dumps and removes the oldest", async () => {
|
||||
for (const day of ["21", "22", "23"]) {
|
||||
const report = await runDatabaseBackup(prisma, {
|
||||
now: new Date(`2026-10-${day}T01:30:00Z`),
|
||||
env: backupEnv({ DB_BACKUP_KEEP: "2" }),
|
||||
});
|
||||
expect(report.status).toBe("succeeded");
|
||||
expect(report.offhost).toBe("not_configured");
|
||||
}
|
||||
expect(readdirSync(backups).filter((n) => n.endsWith(".dump"))).toEqual([
|
||||
"healthlog-20261022T013000Z-daily.dump",
|
||||
"healthlog-20261023T013000Z-daily.dump",
|
||||
]);
|
||||
});
|
||||
|
||||
it("records a failed dump, leaves nothing that looks good, and the page says so", async () => {
|
||||
const before = readdirSync(backups).sort();
|
||||
const url = new URL(process.env.DATABASE_URL ?? "");
|
||||
url.password = "not-the-password";
|
||||
const now = new Date("2026-10-24T01:30:00Z");
|
||||
const report = await runDatabaseBackup(prisma, {
|
||||
now,
|
||||
env: backupEnv({ DATABASE_URL: url.toString() }),
|
||||
});
|
||||
expect(report.status).toBe("failed");
|
||||
expect(report.local).toBe("failed");
|
||||
expect(report.error).toMatch(/password authentication failed/);
|
||||
expect(readdirSync(backups).sort()).toEqual(before);
|
||||
const health = await getDatabaseBackupHealth(
|
||||
new Date("2026-10-24T02:00:00Z"),
|
||||
backupEnv(),
|
||||
);
|
||||
expect(health.lastFailure?.error).toMatch(/password authentication/);
|
||||
expect(health.lastGood?.fileName).toBe(
|
||||
"healthlog-20261023T013000Z-daily.dump",
|
||||
);
|
||||
});
|
||||
|
||||
it("skips, and says why, with nowhere to put a dump", async () => {
|
||||
const report = await runDatabaseBackup(prisma, {
|
||||
now: new Date("2026-10-24T03:00:00Z"),
|
||||
env: backupEnv({ DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here" }),
|
||||
});
|
||||
expect(report).toMatchObject({
|
||||
status: "skipped",
|
||||
local: "no_directory",
|
||||
offhost: "not_configured",
|
||||
});
|
||||
expect(report.error).toMatch(/no writable backup directory/);
|
||||
const off = await runDatabaseBackup(prisma, {
|
||||
now: new Date("2026-10-24T04:00:00Z"),
|
||||
env: backupEnv({ DB_BACKUP_ENABLED: "false" }),
|
||||
});
|
||||
expect(off).toMatchObject({
|
||||
status: "skipped",
|
||||
local: "skipped",
|
||||
offhost: "skipped",
|
||||
});
|
||||
});
|
||||
|
||||
describe("restoring it", () => {
|
||||
const target = `restore_${process.pid}_${Date.now()}`;
|
||||
let targetUrl = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
const admin = await connect(process.env.DATABASE_URL ?? "");
|
||||
await admin.query(`CREATE DATABASE ${target}`);
|
||||
await admin.end();
|
||||
const url = new URL(process.env.DATABASE_URL ?? "");
|
||||
url.pathname = `/${target}`;
|
||||
targetUrl = url.toString();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
const admin = await connect(process.env.DATABASE_URL ?? "");
|
||||
await admin.query(`DROP DATABASE IF EXISTS ${target} WITH (FORCE)`);
|
||||
await admin.end();
|
||||
});
|
||||
|
||||
it("refuses the database the dump came from, which has tables", async () => {
|
||||
const errors: string[] = [];
|
||||
const spy = vi
|
||||
.spyOn(console, "error")
|
||||
.mockImplementation((l: string) => void errors.push(String(l)));
|
||||
const code = await restoreMain(["latest"], backupEnv());
|
||||
spy.mockRestore();
|
||||
expect(code).toBe(1);
|
||||
expect(errors.join("\n")).toMatch(/only into an empty database/);
|
||||
});
|
||||
|
||||
it("restores the newest dump into an empty database with every row count", async () => {
|
||||
const lines: string[] = [];
|
||||
const spy = vi
|
||||
.spyOn(console, "log")
|
||||
.mockImplementation((l: string) => void lines.push(String(l)));
|
||||
const code = await restoreMain(
|
||||
["latest"],
|
||||
backupEnv({ DATABASE_URL: targetUrl }),
|
||||
);
|
||||
spy.mockRestore();
|
||||
expect(code, lines.join("\n")).toBe(0);
|
||||
expect(lines.join("\n")).toMatch(
|
||||
/all \d+ tables hold the row counts the manifest names/,
|
||||
);
|
||||
|
||||
const manifest = await readManifestFile(
|
||||
join(backups, "healthlog-20261023T013000Z-daily.manifest.json"),
|
||||
);
|
||||
const client = await connect(targetUrl);
|
||||
try {
|
||||
expect(await countRows(client)).toEqual(manifest.rowCounts);
|
||||
const { rows } = await client.query(
|
||||
"SELECT count(*)::int AS n FROM users WHERE username LIKE 'dump-%'",
|
||||
);
|
||||
expect(rows[0].n).toBeGreaterThanOrEqual(3);
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
// Restoring again into the now full database is refused.
|
||||
const again = await restoreMain(
|
||||
["latest"],
|
||||
backupEnv({ DATABASE_URL: targetUrl }),
|
||||
);
|
||||
expect(again).toBe(1);
|
||||
expect(existsSync(dumpFile)).toBe(false); // the first dump aged out
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("the dump before migrating", () => {
|
||||
const preDir = join(work, "pre");
|
||||
const appDir = join(work, "app");
|
||||
const failing = join(work, "failing-bin");
|
||||
let cwd = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
const { cpSync, mkdirSync } = await import("node:fs");
|
||||
// An image that ships one migration the database has not applied.
|
||||
mkdirSync(join(appDir, "prisma"), { recursive: true });
|
||||
cpSync(
|
||||
join(process.cwd(), "prisma", "migrations"),
|
||||
join(appDir, "prisma", "migrations"),
|
||||
{ recursive: true },
|
||||
);
|
||||
mkdirSync(join(appDir, "prisma", "migrations", "9999_not_applied_yet"));
|
||||
writeFileSync(
|
||||
join(
|
||||
appDir,
|
||||
"prisma",
|
||||
"migrations",
|
||||
"9999_not_applied_yet",
|
||||
"migration.sql",
|
||||
),
|
||||
"",
|
||||
);
|
||||
mkdirSync(preDir, { recursive: true });
|
||||
mkdirSync(failing, { recursive: true });
|
||||
writeFileSync(
|
||||
join(failing, "pg_dump"),
|
||||
"#!/bin/sh\necho 'pg_dump: error: could not write to output file: No space left on device' >&2\nexit 1\n",
|
||||
);
|
||||
chmodSync(join(failing, "pg_dump"), 0o755);
|
||||
cwd = process.cwd();
|
||||
process.chdir(appDir);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
process.chdir(cwd);
|
||||
process.env.PATH = `${shims}:${originalPath}`;
|
||||
});
|
||||
|
||||
async function preMigration(env: Record<string, string>) {
|
||||
const { preMigrationDump } = await import("../../scripts/db-dump.mjs");
|
||||
const errors: string[] = [];
|
||||
const spyErr = vi
|
||||
.spyOn(console, "error")
|
||||
.mockImplementation((l: string) => void errors.push(String(l)));
|
||||
const spyLog = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
try {
|
||||
const code = await preMigrationDump({ ...process.env, ...env });
|
||||
return { code, errors: errors.join("\n") };
|
||||
} finally {
|
||||
spyErr.mockRestore();
|
||||
spyLog.mockRestore();
|
||||
}
|
||||
}
|
||||
|
||||
it("writes a checked dump with its manifest, and keeps the last two", async () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
if (i > 0) await new Promise((r) => setTimeout(r, 1100));
|
||||
const { code } = await preMigration({ DB_BACKUP_DIR: preDir });
|
||||
expect(code).toBe(0);
|
||||
}
|
||||
const names = readdirSync(preDir).sort();
|
||||
expect(names).toHaveLength(4);
|
||||
expect(names.filter((n) => n.endsWith("-pre-migration.dump"))).toHaveLength(
|
||||
2,
|
||||
);
|
||||
const manifest = await readManifestFile(
|
||||
join(preDir, names.find((n) => n.endsWith(".manifest.json")) ?? ""),
|
||||
);
|
||||
expect(manifest.kind).toBe("pre-migration");
|
||||
expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count());
|
||||
});
|
||||
|
||||
it("does nothing when switched off", async () => {
|
||||
const before = readdirSync(preDir);
|
||||
const { code } = await preMigration({
|
||||
DB_BACKUP_DIR: preDir,
|
||||
DB_BACKUP_BEFORE_MIGRATE: "false",
|
||||
});
|
||||
expect(code).toBe(0);
|
||||
expect(readdirSync(preDir)).toEqual(before);
|
||||
});
|
||||
|
||||
it("warns and lets the boot go on without a writable directory", async () => {
|
||||
const { code, errors } = await preMigration({
|
||||
DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here",
|
||||
});
|
||||
expect(code).toBe(0);
|
||||
expect(errors).toMatch(
|
||||
/is not writable, so no dump is taken before the 1 pending migration/,
|
||||
);
|
||||
});
|
||||
|
||||
it("stops the boot when the dump fails on a writable directory, and leaves no dump behind", async () => {
|
||||
// A second of its own, so the name cannot meet the last dump's.
|
||||
await new Promise((r) => setTimeout(r, 1100));
|
||||
const before = readdirSync(preDir).sort();
|
||||
process.env.PATH = `${failing}:${originalPath}`;
|
||||
const { code, errors } = await preMigration({ DB_BACKUP_DIR: preDir });
|
||||
process.env.PATH = `${shims}:${originalPath}`;
|
||||
expect(code).toBe(1);
|
||||
expect(errors).toMatch(
|
||||
/the database dump before migrating failed: .*No space left/,
|
||||
);
|
||||
expect(readdirSync(preDir).sort()).toEqual(before);
|
||||
});
|
||||
});
|
||||
|
||||
describe("two dumps in the same second", () => {
|
||||
it("refuses the second and leaves the first whole", async () => {
|
||||
const { createDatabaseDump } =
|
||||
await import("@/lib/database-backup/dump-core.mjs");
|
||||
const dir = join(work, "same-second");
|
||||
const { mkdirSync } = await import("node:fs");
|
||||
mkdirSync(dir);
|
||||
const now = new Date("2026-10-26T01:30:00Z");
|
||||
const options = {
|
||||
databaseUrl: process.env.DATABASE_URL ?? "",
|
||||
dir,
|
||||
kind: "daily" as const,
|
||||
now,
|
||||
};
|
||||
await createDatabaseDump(options);
|
||||
const before = readdirSync(dir)
|
||||
.sort()
|
||||
.map((n) => [n, readFileSync(join(dir, n)).length]);
|
||||
await expect(createDatabaseDump(options)).rejects.toThrow(/already exists/);
|
||||
expect(
|
||||
readdirSync(dir)
|
||||
.sort()
|
||||
.map((n) => [n, readFileSync(join(dir, n)).length]),
|
||||
).toEqual(before);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user