/** * One run of the nightly whole-database dump. * * The dump is the artifact that rebuilds a lost server: every account, every * login, every grant and managed profile, the job queue and the instance * settings, in one `pg_dump --format=custom` file with a manifest beside it * (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the * bundled compose file) and, when the off-host bucket is configured, uploaded * under `database/` in the same envelope the per-account copies use, sealed * with `BACKUP_ENCRYPTION_KEY`. * * Without a directory that survives the container (an install that runs the * image without the volume: the image's own directory is writable but sits in * the container's layer, which the next update throws away) the dump is * written to a temporary directory, uploaded and removed: the off-host copy * still happens, nothing is kept on the server, and the ledger and the boot * summary say so. A dump left in the container's layer would be counted as * kept and lost with the container, so none is. With neither, the run is * skipped and recorded as such. * * Every run leaves one `DatabaseBackupRun` row: started, then finished with * what happened on each leg. `getDatabaseBackupHealth` reads them. */ import { createReadStream } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import type { PrismaClient } from "@/generated/prisma/client"; import { getS3Client, loadOffhostConfig, uploadEncryptedBackup, type S3Like, } from "@/lib/jobs/offhost-backup"; import { annotate } from "@/lib/logging/context"; import packageJson from "../../../package.json"; import { applyRetention, backupDirectoryStatus, createDatabaseDump, databaseBackupEnabled, instanceIdFromEnv, offhostObjectKey, resolveBackupDir, resolveDailyKeep, type BackupDirectoryStatus, } from "./dump-core.mjs"; /** How long ledger rows are kept. A run a day; a year and a bit of history. */ const LEDGER_RETENTION_DAYS = 400; export type DatabaseBackupLocalResult = "written" | "no_directory" | "not_persistent" | "failed" | "skipped"; export type DatabaseBackupOffhostResult = "uploaded" | "not_configured" | "failed" | "skipped"; export interface DatabaseBackupReport { status: "succeeded" | "failed" | "skipped"; fileName: string | null; dumpBytes: number | null; local: DatabaseBackupLocalResult; offhost: DatabaseBackupOffhostResult; /** Dumps and manifests retention removed. */ removed: number; error: string | null; } export interface RunDatabaseBackupOptions { now?: Date; /** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */ deadline?: number; env?: NodeJS.ProcessEnv; /** A bucket double for tests; the configured bucket otherwise. */ s3?: S3Like; /** Where `DB_BACKUP_DIR` stands; the file system's answer otherwise. */ directoryStatus?: (dir: string) => BackupDirectoryStatus; } /** The running build's version, as `/api/version` answers it. */ function appVersion(env: NodeJS.ProcessEnv): string { return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version; } /** An error as one line for the ledger and the admin page. */ function reason(err: unknown): string { const text = err instanceof Error ? err.message : String(err); return text.replace(/\s+/g, " ").trim().slice(0, 500); } /** Upload one local file as an off-host object, streamed. */ async function uploadFile( s3: S3Like, key: Buffer, path: string, instanceId: string | null, ): Promise { return uploadEncryptedBackup( s3, offhostObjectKey(`${basename(path)}.enc`, instanceId), key, async (write) => { for await (const chunk of createReadStream(path)) { await write(chunk as Buffer); } }, ); } /** * Take tonight's dump, keep it, copy it off-host, prune the old ones, and * record what happened. Never throws: a failure is the report's `status`. */ export async function runDatabaseBackup( prisma: PrismaClient, options: RunDatabaseBackupOptions = {}, ): Promise { const env = options.env ?? process.env; const now = options.now ?? new Date(); const dir = resolveBackupDir(env); const directory = (options.directoryStatus ?? backupDirectoryStatus)(dir); // Kept here only where it survives the container being replaced. const keepLocal = directory.persistent; const offhostConfig = (() => { try { return loadOffhostConfig(); } catch { // A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the // same value with their own message; here it reads as an upload error. return "invalid" as const; } })(); // Each leg reads "failed" until it has succeeded, so a run the process died // under says so once its expiry has passed. const report: DatabaseBackupReport = { status: "failed", fileName: null, dumpBytes: null, local: keepLocal ? "failed" : directory.writable ? "not_persistent" : "no_directory", offhost: offhostConfig ? "failed" : "not_configured", removed: 0, error: null, }; const run = await prisma.databaseBackupRun.create({ data: { startedAt: now, status: "running", localResult: report.local, offhostResult: report.offhost, }, select: { id: true }, }); let scratch: string | null = null; let activeKeyId: string | null = null; try { if (!databaseBackupEnabled(env)) { report.status = "skipped"; report.local = "skipped"; report.offhost = "skipped"; report.error = "DB_BACKUP_ENABLED is false"; } else if (!keepLocal && !offhostConfig) { report.status = "skipped"; report.error = directory.writable ? `the backup directory (${dir}) is not on persistent storage, so a dump there would be lost with the container, and no off-host bucket is configured` : `no writable backup directory (${dir}) and no off-host bucket configured`; } else { const target = keepLocal ? dir : (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-"))); const dump = await createDatabaseDump({ databaseUrl: env.DATABASE_URL ?? "", dir: target, kind: "daily", now, env, appVersion: appVersion(env), buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null, deadline: options.deadline, }); if (keepLocal) report.local = "written"; activeKeyId = dump.manifest.keys.encryption.activeKeyId; report.fileName = basename(dump.dumpPath); report.dumpBytes = dump.manifest.dump.bytes; let uploadError: string | null = null; if (offhostConfig === "invalid") { uploadError = "BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64"; } else if (offhostConfig) { try { const s3 = options.s3 ?? (await getS3Client(offhostConfig)); // The manifest goes up first: a dump object in the bucket always // has the manifest a restore checks it against. // Under this instance's own folder: a bucket may hold the copies // of more than one instance (`instanceIdFromEnv`). const instanceId = instanceIdFromEnv(env); await uploadFile( s3, offhostConfig.encryptionKey, dump.manifestPath, instanceId, ); await uploadFile( s3, offhostConfig.encryptionKey, dump.dumpPath, instanceId, ); report.offhost = "uploaded"; } catch (err) { uploadError = `off-host upload failed: ${reason(err)}`; } } if (keepLocal) { const removed = await applyRetention(dir, { dailyKeep: resolveDailyKeep(env), now: now.getTime(), }); report.removed = removed.length; } // Good when the dump exists somewhere it survives: on the volume, or in // the bucket. An upload that failed beside a kept dump still fails the // run, because the operator asked for both. report.status = uploadError ? "failed" : "succeeded"; report.error = uploadError; } } catch (err) { report.status = "failed"; report.error = reason(err); } finally { if (scratch) await rm(scratch, { recursive: true, force: true }); } await prisma.databaseBackupRun.update({ where: { id: run.id }, data: { finishedAt: new Date(), status: report.status, fileName: report.fileName, dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes), localResult: report.local, offhostResult: report.offhost, error: report.error, activeKeyId, }, }); await prisma.databaseBackupRun.deleteMany({ where: { startedAt: { lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000), }, }, }); annotate({ action: { name: "database.backup.run" }, meta: { status: report.status, local: report.local, offhost: report.offhost, dump_bytes: report.dumpBytes ?? 0, removed: report.removed, }, }); return report; }