HealthLog/src/lib/database-backup/run.ts

/**
 * One run of the nightly whole-database dump.
 *
 * The dump is the artifact that rebuilds a lost server: every account, every
 * login, every grant and managed profile, the job queue and the instance
 * settings, in one `pg_dump --format=custom` file with a manifest beside it
 * (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the
 * bundled compose file) and, when the off-host bucket is configured, uploaded
 * under `database/` in the same envelope the per-account copies use, sealed
 * with `BACKUP_ENCRYPTION_KEY`.
 *
 * Without a directory that survives the container (an install that runs the
 * image without the volume: the image's own directory is writable but sits in
 * the container's layer, which the next update throws away) the dump is
 * written to a temporary directory, uploaded and removed: the off-host copy
 * still happens, nothing is kept on the server, and the ledger and the boot
 * summary say so. A dump left in the container's layer would be counted as
 * kept and lost with the container, so none is. With neither, the run is
 * skipped and recorded as such.
 *
 * Every run leaves one `DatabaseBackupRun` row: started, then finished with
 * what happened on each leg. `getDatabaseBackupHealth` reads them.
 */
import { createReadStream } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";

import type { PrismaClient } from "@/generated/prisma/client";
import {
  getS3Client,
  loadOffhostConfig,
  uploadEncryptedBackup,
  type S3Like,
} from "@/lib/jobs/offhost-backup";
import { annotate } from "@/lib/logging/context";
import packageJson from "../../../package.json";

import {
  applyRetention,
  backupDirectoryStatus,
  createDatabaseDump,
  databaseBackupEnabled,
  instanceIdFromEnv,
  offhostObjectKey,
  resolveBackupDir,
  resolveDailyKeep,
  type BackupDirectoryStatus,
} from "./dump-core.mjs";

/** How long ledger rows are kept. A run a day; a year and a bit of history. */
const LEDGER_RETENTION_DAYS = 400;

export type DatabaseBackupLocalResult =
  "written" | "no_directory" | "not_persistent" | "failed" | "skipped";
export type DatabaseBackupOffhostResult =
  "uploaded" | "not_configured" | "failed" | "skipped";

export interface DatabaseBackupReport {
  status: "succeeded" | "failed" | "skipped";
  fileName: string | null;
  dumpBytes: number | null;
  local: DatabaseBackupLocalResult;
  offhost: DatabaseBackupOffhostResult;
  /** Dumps and manifests retention removed. */
  removed: number;
  error: string | null;
}

export interface RunDatabaseBackupOptions {
  now?: Date;
  /** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */
  deadline?: number;
  env?: NodeJS.ProcessEnv;
  /** A bucket double for tests; the configured bucket otherwise. */
  s3?: S3Like;
  /** Where `DB_BACKUP_DIR` stands; the file system's answer otherwise. */
  directoryStatus?: (dir: string) => BackupDirectoryStatus;
}

/** The running build's version, as `/api/version` answers it. */
function appVersion(env: NodeJS.ProcessEnv): string {
  return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version;
}

/** An error as one line for the ledger and the admin page. */
function reason(err: unknown): string {
  const text = err instanceof Error ? err.message : String(err);
  return text.replace(/\s+/g, " ").trim().slice(0, 500);
}

/** Upload one local file as an off-host object, streamed. */
async function uploadFile(
  s3: S3Like,
  key: Buffer,
  path: string,
  instanceId: string | null,
): Promise<number> {
  return uploadEncryptedBackup(
    s3,
    offhostObjectKey(`${basename(path)}.enc`, instanceId),
    key,
    async (write) => {
      for await (const chunk of createReadStream(path)) {
        await write(chunk as Buffer);
      }
    },
  );
}

/**
 * Take tonight's dump, keep it, copy it off-host, prune the old ones, and
 * record what happened. Never throws: a failure is the report's `status`.
 */
export async function runDatabaseBackup(
  prisma: PrismaClient,
  options: RunDatabaseBackupOptions = {},
): Promise<DatabaseBackupReport> {
  const env = options.env ?? process.env;
  const now = options.now ?? new Date();
  const dir = resolveBackupDir(env);
  const directory = (options.directoryStatus ?? backupDirectoryStatus)(dir);
  // Kept here only where it survives the container being replaced.
  const keepLocal = directory.persistent;
  const offhostConfig = (() => {
    try {
      return loadOffhostConfig();
    } catch {
      // A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the
      // same value with their own message; here it reads as an upload error.
      return "invalid" as const;
    }
  })();

  // Each leg reads "failed" until it has succeeded, so a run the process died
  // under says so once its expiry has passed.
  const report: DatabaseBackupReport = {
    status: "failed",
    fileName: null,
    dumpBytes: null,
    local: keepLocal
      ? "failed"
      : directory.writable
        ? "not_persistent"
        : "no_directory",
    offhost: offhostConfig ? "failed" : "not_configured",
    removed: 0,
    error: null,
  };
  const run = await prisma.databaseBackupRun.create({
    data: {
      startedAt: now,
      status: "running",
      localResult: report.local,
      offhostResult: report.offhost,
    },
    select: { id: true },
  });

  let scratch: string | null = null;
  let activeKeyId: string | null = null;
  try {
    if (!databaseBackupEnabled(env)) {
      report.status = "skipped";
      report.local = "skipped";
      report.offhost = "skipped";
      report.error = "DB_BACKUP_ENABLED is false";
    } else if (!keepLocal && !offhostConfig) {
      report.status = "skipped";
      report.error = directory.writable
        ? `the backup directory (${dir}) is not on persistent storage, so a dump there would be lost with the container, and no off-host bucket is configured`
        : `no writable backup directory (${dir}) and no off-host bucket configured`;
    } else {
      const target = keepLocal
        ? dir
        : (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-")));
      const dump = await createDatabaseDump({
        databaseUrl: env.DATABASE_URL ?? "",
        dir: target,
        kind: "daily",
        now,
        env,
        appVersion: appVersion(env),
        buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null,
        deadline: options.deadline,
      });
      if (keepLocal) report.local = "written";
      activeKeyId = dump.manifest.keys.encryption.activeKeyId;
      report.fileName = basename(dump.dumpPath);
      report.dumpBytes = dump.manifest.dump.bytes;

      let uploadError: string | null = null;
      if (offhostConfig === "invalid") {
        uploadError =
          "BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64";
      } else if (offhostConfig) {
        try {
          const s3 = options.s3 ?? (await getS3Client(offhostConfig));
          // The manifest goes up first: a dump object in the bucket always
          // has the manifest a restore checks it against.
          // Under this instance's own folder: a bucket may hold the copies
          // of more than one instance (`instanceIdFromEnv`).
          const instanceId = instanceIdFromEnv(env);
          await uploadFile(
            s3,
            offhostConfig.encryptionKey,
            dump.manifestPath,
            instanceId,
          );
          await uploadFile(
            s3,
            offhostConfig.encryptionKey,
            dump.dumpPath,
            instanceId,
          );
          report.offhost = "uploaded";
        } catch (err) {
          uploadError = `off-host upload failed: ${reason(err)}`;
        }
      }

      if (keepLocal) {
        const removed = await applyRetention(dir, {
          dailyKeep: resolveDailyKeep(env),
          now: now.getTime(),
        });
        report.removed = removed.length;
      }

      // Good when the dump exists somewhere it survives: on the volume, or in
      // the bucket. An upload that failed beside a kept dump still fails the
      // run, because the operator asked for both.
      report.status = uploadError ? "failed" : "succeeded";
      report.error = uploadError;
    }
  } catch (err) {
    report.status = "failed";
    report.error = reason(err);
  } finally {
    if (scratch) await rm(scratch, { recursive: true, force: true });
  }

  await prisma.databaseBackupRun.update({
    where: { id: run.id },
    data: {
      finishedAt: new Date(),
      status: report.status,
      fileName: report.fileName,
      dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes),
      localResult: report.local,
      offhostResult: report.offhost,
      error: report.error,
      activeKeyId,
    },
  });
  await prisma.databaseBackupRun.deleteMany({
    where: {
      startedAt: {
        lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000),
      },
    },
  });

  annotate({
    action: { name: "database.backup.run" },
    meta: {
      status: report.status,
      local: report.local,
      offhost: report.offhost,
      dump_bytes: report.dumpBytes ?? 0,
      removed: report.removed,
    },
  });
  return report;
}