mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-07 04:12:04 +00:00
Marc-André Bombeck
d7046c9737
The migration before-and-after files, the dump directory, the geo backfill's three passes, the settings trip, the backup parity steps, the lab conversion record and the pulse method's dry run before its rebuild are each one sequence by design; they now say so and keep their order with shuffle: false. The CSV import's zone and crash cases wrote into the account the lab and medication walks count, and now use accounts of their own.
705 lines
24 KiB
TypeScript
705 lines
24 KiB
TypeScript
/**
|
|
* The nightly whole-database dump against a real Postgres: the job writes a
|
|
* dump and its manifest, records the run, copies both off-host in the
|
|
* envelope the restore opens, keeps only what retention allows; and the
|
|
* restore puts the dump into an empty database and refuses one with tables.
|
|
*
|
|
* `pg_dump` and `pg_restore` are the test container's own. A workstation may
|
|
* have none, or one of another major than the server (which `pg_dump`
|
|
* refuses), so the test puts two small forwarders on PATH that run the tool
|
|
* inside the database container with `docker exec`. Everything else, the
|
|
* snapshot, the counts, the manifest, the ledger and the restore's checks,
|
|
* runs exactly as in the image.
|
|
*/
|
|
import { execFileSync } from "node:child_process";
|
|
import {
|
|
chmodSync,
|
|
existsSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
readdirSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { Readable } from "node:stream";
|
|
|
|
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
|
|
|
import { prisma } from "@/lib/db";
|
|
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
|
|
import { evaluateDatabaseDump } from "@/lib/recovery/readiness-model";
|
|
import { runDatabaseBackup } from "@/lib/database-backup/run";
|
|
import {
|
|
connect,
|
|
countRows,
|
|
decryptOffhostFile,
|
|
offhostObjectKey,
|
|
readManifestFile,
|
|
} from "@/lib/database-backup/dump-core.mjs";
|
|
|
|
import { main as restoreMain } from "../../scripts/db-restore.mjs";
|
|
import { shippedMigrations } from "../../scripts/pending-migrations.mjs";
|
|
|
|
const OFFHOST_KEY = "e".repeat(64);
|
|
const work = mkdtempSync(join(tmpdir(), "hl-db-backup-"));
|
|
const backups = join(work, "backups");
|
|
const shims = join(work, "bin");
|
|
const originalPath = process.env.PATH;
|
|
|
|
/** The id of the container that serves DATABASE_URL. */
|
|
function containerId(): string {
|
|
const port = new URL(process.env.DATABASE_URL ?? "").port;
|
|
const rows = execFileSync(
|
|
"docker",
|
|
["ps", "--format", "{{.ID}} {{.Ports}}"],
|
|
{
|
|
encoding: "utf8",
|
|
},
|
|
).split("\n");
|
|
const row = rows.find((r) => r.includes(`:${port}->5432/tcp`));
|
|
if (!row) throw new Error(`no container publishes port ${port}`);
|
|
return row.split(" ")[0];
|
|
}
|
|
|
|
/**
|
|
* `pg_dump` / `pg_restore` that run inside the database container. The
|
|
* connection comes from the PG* variables the core sets; host and port become
|
|
* the container's own. A file argument that exists on this machine is sent on
|
|
* standard input, which both tools read when no file is named.
|
|
*/
|
|
function writeShims(container: string): void {
|
|
for (const tool of ["pg_dump", "pg_restore"]) {
|
|
const path = join(shims, tool);
|
|
writeFileSync(
|
|
path,
|
|
`#!/usr/bin/env node
|
|
const { spawn } = require("node:child_process");
|
|
const { createReadStream, existsSync } = require("node:fs");
|
|
const args = process.argv.slice(2);
|
|
let input = null;
|
|
if (args.length && !args[args.length - 1].startsWith("-") && existsSync(args[args.length - 1])) {
|
|
input = args.pop();
|
|
}
|
|
const env = [];
|
|
for (const name of ["PGUSER", "PGPASSWORD", "PGDATABASE"]) {
|
|
if (process.env[name] !== undefined) env.push("-e", name + "=" + process.env[name]);
|
|
}
|
|
env.push("-e", "PGHOST=127.0.0.1", "-e", "PGPORT=5432");
|
|
// Standard input is attached only when a file is fed: with \`-i\` and an
|
|
// input that is already at its end, the docker client can close the stream
|
|
// before all of the tool's output has arrived.
|
|
const child = spawn("docker", ["exec", ...(input ? ["-i"] : []), ...env, ${JSON.stringify(container)}, ${JSON.stringify(tool)}, ...args], {
|
|
stdio: [input ? "pipe" : "ignore", "inherit", "inherit"],
|
|
});
|
|
// A docker client that exits before it has read all of the input closes the
|
|
// pipe under us; that is the tool's failure to report, not a crash of this
|
|
// wrapper, so the exit code below carries it and the reason goes to stderr.
|
|
child.on("error", (err) => {
|
|
process.stderr.write("shim: " + err.message + "\\n");
|
|
process.exit(127);
|
|
});
|
|
if (input) {
|
|
child.stdin.on("error", (err) => process.stderr.write("shim stdin: " + err.code + "\\n"));
|
|
createReadStream(input).pipe(child.stdin);
|
|
}
|
|
child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0)));
|
|
`,
|
|
);
|
|
chmodSync(path, 0o755);
|
|
}
|
|
}
|
|
|
|
/** A bucket in memory, with the one call the job's uploader uses. */
|
|
function memoryBucket() {
|
|
const objects = new Map<string, Buffer>();
|
|
return {
|
|
objects,
|
|
putStream: async (key: string, body: Readable) => {
|
|
const chunks: Buffer[] = [];
|
|
for await (const c of body) chunks.push(c as Buffer);
|
|
objects.set(key, Buffer.concat(chunks));
|
|
},
|
|
};
|
|
}
|
|
|
|
function backupEnv(extra: Record<string, string> = {}): NodeJS.ProcessEnv {
|
|
return { ...process.env, DB_BACKUP_DIR: backups, ...extra };
|
|
}
|
|
|
|
beforeAll(async () => {
|
|
const { mkdirSync } = await import("node:fs");
|
|
mkdirSync(shims, { recursive: true });
|
|
mkdirSync(backups, { recursive: true });
|
|
writeShims(containerId());
|
|
process.env.PATH = `${shims}:${originalPath}`;
|
|
for (const name of ["dump-a", "dump-b", "dump-c"]) {
|
|
await prisma.user.create({
|
|
data: { username: `${name}-${Date.now()}` },
|
|
});
|
|
}
|
|
});
|
|
|
|
afterAll(() => {
|
|
process.env.PATH = originalPath;
|
|
vi.unstubAllEnvs();
|
|
rmSync(work, { recursive: true, force: true });
|
|
});
|
|
|
|
// One backup directory through the cases in order: the first dump is the one
|
|
// the retention, failure and restore cases count and restore. The order holds
|
|
// under a shuffled run.
|
|
describe("the nightly database dump", { shuffle: false }, () => {
|
|
let dumpFile = "";
|
|
|
|
it("writes a checked dump and manifest, records the run and copies both off-host", async () => {
|
|
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid");
|
|
vi.stubEnv("BACKUP_S3_BUCKET", "b");
|
|
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
|
|
vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
|
|
vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY);
|
|
const bucket = memoryBucket();
|
|
const now = new Date("2026-10-20T01:30:00Z");
|
|
|
|
const report = await runDatabaseBackup(prisma, {
|
|
now,
|
|
env: backupEnv(),
|
|
s3: bucket as never,
|
|
});
|
|
expect(report).toMatchObject({
|
|
status: "succeeded",
|
|
local: "written",
|
|
offhost: "uploaded",
|
|
error: null,
|
|
fileName: "healthlog-20261020T013000Z-daily.dump",
|
|
});
|
|
dumpFile = join(backups, report.fileName ?? "");
|
|
|
|
// Both files, under their final names, and nothing half-written.
|
|
expect(readdirSync(backups).sort()).toEqual([
|
|
"healthlog-20261020T013000Z-daily.dump",
|
|
"healthlog-20261020T013000Z-daily.manifest.json",
|
|
]);
|
|
expect(readFileSync(dumpFile).subarray(0, 5).toString("latin1")).toBe(
|
|
"PGDMP",
|
|
);
|
|
|
|
const manifest = await readManifestFile(
|
|
join(backups, "healthlog-20261020T013000Z-daily.manifest.json"),
|
|
);
|
|
expect(manifest.kind).toBe("daily");
|
|
expect(manifest.dump.bytes).toBe(report.dumpBytes);
|
|
expect(manifest.latestMigration).toBe(
|
|
shippedMigrations(join(process.cwd(), "prisma", "migrations")).at(-1),
|
|
);
|
|
expect(manifest.postgres.serverVersionNum).toBeGreaterThanOrEqual(160000);
|
|
expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count());
|
|
expect(Object.keys(manifest.rowCounts).length).toBeGreaterThan(100);
|
|
expect(manifest.keys.encryption.keys.length).toBeGreaterThan(0);
|
|
expect(manifest.keys.backupEncryptionKey).toMatch(/^[0-9a-f]{12}$/);
|
|
|
|
// The ledger row and what the admin page reads from it.
|
|
const runs = await prisma.databaseBackupRun.findMany({
|
|
where: { startedAt: now },
|
|
});
|
|
expect(runs).toHaveLength(1);
|
|
expect(runs[0]).toMatchObject({
|
|
status: "succeeded",
|
|
fileName: "healthlog-20261020T013000Z-daily.dump",
|
|
localResult: "written",
|
|
offhostResult: "uploaded",
|
|
error: null,
|
|
});
|
|
expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes);
|
|
// The key a restore of this dump needs, for the recovery kit.
|
|
expect(runs[0].activeKeyId).toBe(manifest.keys.encryption.activeKeyId);
|
|
expect(runs[0].activeKeyId).not.toBeNull();
|
|
const health = await getDatabaseBackupHealth(
|
|
new Date("2026-10-20T02:00:00Z"),
|
|
backupEnv(),
|
|
);
|
|
expect(health).toMatchObject({
|
|
enabled: true,
|
|
localDirWritable: true,
|
|
offhostConfigured: true,
|
|
lastGood: {
|
|
fileName: "healthlog-20261020T013000Z-daily.dump",
|
|
sizeBytes: manifest.dump.bytes,
|
|
local: true,
|
|
offhost: true,
|
|
},
|
|
});
|
|
|
|
// The off-host pair opens, with the restore's own reader, to the very
|
|
// bytes on disk.
|
|
expect([...bucket.objects.keys()].sort()).toEqual([
|
|
"database/healthlog-20261020T013000Z-daily.dump.enc",
|
|
"database/healthlog-20261020T013000Z-daily.manifest.json.enc",
|
|
]);
|
|
const opened = join(work, "opened");
|
|
const { mkdirSync } = await import("node:fs");
|
|
mkdirSync(opened);
|
|
for (const name of [
|
|
"healthlog-20261020T013000Z-daily.dump",
|
|
"healthlog-20261020T013000Z-daily.manifest.json",
|
|
]) {
|
|
writeFileSync(
|
|
join(opened, `${name}.enc`),
|
|
bucket.objects.get(offhostObjectKey(`${name}.enc`)) as Buffer,
|
|
);
|
|
await decryptOffhostFile({
|
|
inPath: join(opened, `${name}.enc`),
|
|
outPath: join(opened, name),
|
|
objectKey: offhostObjectKey(`${name}.enc`),
|
|
keys: [Buffer.from(OFFHOST_KEY, "hex")],
|
|
});
|
|
expect(
|
|
readFileSync(join(opened, name)).equals(
|
|
readFileSync(join(backups, name)),
|
|
),
|
|
).toBe(true);
|
|
}
|
|
vi.unstubAllEnvs();
|
|
});
|
|
|
|
it("keeps DB_BACKUP_KEEP nightly dumps and removes the oldest", async () => {
|
|
for (const day of ["21", "22", "23"]) {
|
|
const report = await runDatabaseBackup(prisma, {
|
|
now: new Date(`2026-10-${day}T01:30:00Z`),
|
|
env: backupEnv({ DB_BACKUP_KEEP: "2" }),
|
|
});
|
|
expect(report.status, report.error ?? "").toBe("succeeded");
|
|
expect(report.offhost).toBe("not_configured");
|
|
}
|
|
expect(readdirSync(backups).filter((n) => n.endsWith(".dump"))).toEqual([
|
|
"healthlog-20261022T013000Z-daily.dump",
|
|
"healthlog-20261023T013000Z-daily.dump",
|
|
]);
|
|
});
|
|
|
|
it("records a failed dump, leaves nothing that looks good, and the page says so", async () => {
|
|
const before = readdirSync(backups).sort();
|
|
const url = new URL(process.env.DATABASE_URL ?? "");
|
|
url.password = "not-the-password";
|
|
const now = new Date("2026-10-24T01:30:00Z");
|
|
const report = await runDatabaseBackup(prisma, {
|
|
now,
|
|
env: backupEnv({ DATABASE_URL: url.toString() }),
|
|
});
|
|
expect(report.status).toBe("failed");
|
|
expect(report.local).toBe("failed");
|
|
expect(report.error).toMatch(/password authentication failed/);
|
|
expect(readdirSync(backups).sort()).toEqual(before);
|
|
const health = await getDatabaseBackupHealth(
|
|
new Date("2026-10-24T02:00:00Z"),
|
|
backupEnv(),
|
|
);
|
|
expect(health.lastFailure?.error).toMatch(/password authentication/);
|
|
expect(health.lastGood?.fileName).toBe(
|
|
"healthlog-20261023T013000Z-daily.dump",
|
|
);
|
|
});
|
|
|
|
it("skips, and says why, with nowhere to put a dump", async () => {
|
|
const report = await runDatabaseBackup(prisma, {
|
|
now: new Date("2026-10-24T03:00:00Z"),
|
|
env: backupEnv({ DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here" }),
|
|
});
|
|
expect(report).toMatchObject({
|
|
status: "skipped",
|
|
local: "no_directory",
|
|
offhost: "not_configured",
|
|
});
|
|
expect(report.error).toMatch(/no writable backup directory/);
|
|
const off = await runDatabaseBackup(prisma, {
|
|
now: new Date("2026-10-24T04:00:00Z"),
|
|
env: backupEnv({ DB_BACKUP_ENABLED: "false" }),
|
|
});
|
|
expect(off).toMatchObject({
|
|
status: "skipped",
|
|
local: "skipped",
|
|
offhost: "skipped",
|
|
});
|
|
});
|
|
|
|
it("keeps nothing in a directory inside the container, and stops counting what was written there", async () => {
|
|
const { mkdirSync } = await import("node:fs");
|
|
const layerDir = join(work, "container-layer");
|
|
mkdirSync(layerDir);
|
|
// The image's own directory without the volume: writable, and thrown
|
|
// away with the container at the next update.
|
|
const layer = () =>
|
|
({
|
|
writable: true,
|
|
persistent: false,
|
|
reason: "container_layer",
|
|
}) as const;
|
|
const env = backupEnv({ DB_BACKUP_DIR: layerDir });
|
|
|
|
const skipped = await runDatabaseBackup(prisma, {
|
|
now: new Date("2026-10-24T05:00:00Z"),
|
|
env,
|
|
directoryStatus: layer,
|
|
});
|
|
expect(skipped).toMatchObject({
|
|
status: "skipped",
|
|
local: "not_persistent",
|
|
offhost: "not_configured",
|
|
});
|
|
expect(skipped.error).toMatch(/not on persistent storage/);
|
|
expect(readdirSync(layerDir)).toEqual([]);
|
|
|
|
// Read as keeping nothing, the newest dump that still exists is the one
|
|
// that reached the bucket on the 20th, not the local-only one of the
|
|
// 23rd; read as a volume, it is the 23rd's.
|
|
const asLayer = await getDatabaseBackupHealth(
|
|
new Date("2026-10-24T05:30:00Z"),
|
|
env,
|
|
layer,
|
|
);
|
|
expect(asLayer).toMatchObject({
|
|
localDirWritable: true,
|
|
localPersistent: false,
|
|
localReason: "container_layer",
|
|
lastGood: {
|
|
fileName: "healthlog-20261020T013000Z-daily.dump",
|
|
local: false,
|
|
offhost: true,
|
|
},
|
|
});
|
|
const asVolume = await getDatabaseBackupHealth(
|
|
new Date("2026-10-24T05:30:00Z"),
|
|
backupEnv(),
|
|
);
|
|
expect(asVolume.localPersistent).toBe(true);
|
|
expect(asVolume.lastGood).toMatchObject({
|
|
fileName: "healthlog-20261023T013000Z-daily.dump",
|
|
local: true,
|
|
});
|
|
|
|
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid");
|
|
vi.stubEnv("BACKUP_S3_BUCKET", "b");
|
|
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
|
|
vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
|
|
vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY);
|
|
const bucket = memoryBucket();
|
|
const uploaded = await runDatabaseBackup(prisma, {
|
|
now: new Date("2026-10-24T06:00:00Z"),
|
|
env: backupEnv({ DB_BACKUP_DIR: layerDir }),
|
|
s3: bucket as never,
|
|
directoryStatus: layer,
|
|
});
|
|
vi.unstubAllEnvs();
|
|
expect(uploaded).toMatchObject({
|
|
status: "succeeded",
|
|
local: "not_persistent",
|
|
offhost: "uploaded",
|
|
removed: 0,
|
|
});
|
|
expect(readdirSync(layerDir)).toEqual([]);
|
|
expect([...bucket.objects.keys()].sort()).toEqual([
|
|
"database/healthlog-20261024T060000Z-daily.dump.enc",
|
|
"database/healthlog-20261024T060000Z-daily.manifest.json.enc",
|
|
]);
|
|
const row = await prisma.databaseBackupRun.findFirst({
|
|
where: { startedAt: new Date("2026-10-24T06:00:00Z") },
|
|
});
|
|
expect(row).toMatchObject({
|
|
localResult: "not_persistent",
|
|
offhostResult: "uploaded",
|
|
});
|
|
});
|
|
|
|
// A dump kept on the volume and refused by the bucket is the newest good
|
|
// dump and the newest failure at once. Read as only those two it looked
|
|
// healthy, night after night, while nothing left the host.
|
|
it("does not read a kept dump as healthy when the bucket refused it", async () => {
|
|
const { mkdirSync } = await import("node:fs");
|
|
const dir = join(work, "upload-refused");
|
|
mkdirSync(dir);
|
|
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid");
|
|
vi.stubEnv("BACKUP_S3_BUCKET", "b");
|
|
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
|
|
vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
|
|
vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY);
|
|
const refusing = {
|
|
putStream: async (_key: string, body: Readable) => {
|
|
for await (const _ of body) void _;
|
|
throw new Error("AccessDenied on database/");
|
|
},
|
|
};
|
|
const report = await runDatabaseBackup(prisma, {
|
|
now: new Date("2026-10-24T07:00:00Z"),
|
|
env: backupEnv({ DB_BACKUP_DIR: dir }),
|
|
s3: refusing as never,
|
|
});
|
|
vi.unstubAllEnvs();
|
|
expect(report).toMatchObject({
|
|
status: "failed",
|
|
local: "written",
|
|
offhost: "failed",
|
|
});
|
|
|
|
const health = await getDatabaseBackupHealth(
|
|
new Date(),
|
|
backupEnv({ DB_BACKUP_DIR: dir }),
|
|
);
|
|
expect(health.lastGood).toMatchObject({
|
|
fileName: "healthlog-20261024T070000Z-daily.dump",
|
|
local: true,
|
|
offhost: false,
|
|
});
|
|
expect(health.offhostConfigured).toBe(true);
|
|
expect(health.offhost.lastUploadAt).not.toBeNull();
|
|
expect(health.offhost.failure?.error).toMatch(/AccessDenied/);
|
|
expect(
|
|
evaluateDatabaseDump({
|
|
now: new Date(),
|
|
dump: health,
|
|
dumpWatchedSince: null,
|
|
}),
|
|
).toMatchObject({
|
|
state: "attention",
|
|
reason: "offhost_failed",
|
|
detail: expect.stringMatching(/AccessDenied/),
|
|
});
|
|
});
|
|
|
|
describe("restoring it", () => {
|
|
const target = `restore_${process.pid}_${Date.now()}`;
|
|
let targetUrl = "";
|
|
|
|
beforeAll(async () => {
|
|
const admin = await connect(process.env.DATABASE_URL ?? "");
|
|
await admin.query(`CREATE DATABASE ${target}`);
|
|
await admin.end();
|
|
const url = new URL(process.env.DATABASE_URL ?? "");
|
|
url.pathname = `/${target}`;
|
|
targetUrl = url.toString();
|
|
});
|
|
|
|
afterAll(async () => {
|
|
const admin = await connect(process.env.DATABASE_URL ?? "");
|
|
await admin.query(`DROP DATABASE IF EXISTS ${target} WITH (FORCE)`);
|
|
await admin.end();
|
|
});
|
|
|
|
it("refuses the database the dump came from, which has tables", async () => {
|
|
const errors: string[] = [];
|
|
const spy = vi
|
|
.spyOn(console, "error")
|
|
.mockImplementation((l: string) => void errors.push(String(l)));
|
|
const code = await restoreMain(["latest"], backupEnv());
|
|
spy.mockRestore();
|
|
expect(code).toBe(1);
|
|
expect(errors.join("\n")).toMatch(/only into an empty database/);
|
|
});
|
|
|
|
it("restores the newest dump into an empty database with every row count", async () => {
|
|
const lines: string[] = [];
|
|
const spy = vi
|
|
.spyOn(console, "log")
|
|
.mockImplementation((l: string) => void lines.push(String(l)));
|
|
const code = await restoreMain(
|
|
["latest"],
|
|
backupEnv({ DATABASE_URL: targetUrl }),
|
|
);
|
|
spy.mockRestore();
|
|
expect(code, lines.join("\n")).toBe(0);
|
|
expect(lines.join("\n")).toMatch(
|
|
/all \d+ tables hold the row counts the manifest names/,
|
|
);
|
|
|
|
const manifest = await readManifestFile(
|
|
join(backups, "healthlog-20261023T013000Z-daily.manifest.json"),
|
|
);
|
|
const client = await connect(targetUrl);
|
|
try {
|
|
expect(await countRows(client)).toEqual(manifest.rowCounts);
|
|
const { rows } = await client.query(
|
|
"SELECT count(*)::int AS n FROM users WHERE username LIKE 'dump-%'",
|
|
);
|
|
expect(rows[0].n).toBeGreaterThanOrEqual(3);
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
// Restoring again into the now full database is refused.
|
|
const again = await restoreMain(
|
|
["latest"],
|
|
backupEnv({ DATABASE_URL: targetUrl }),
|
|
);
|
|
expect(again).toBe(1);
|
|
expect(existsSync(dumpFile)).toBe(false); // the first dump aged out
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("the dump before migrating", () => {
|
|
const preDir = join(work, "pre");
|
|
const appDir = join(work, "app");
|
|
const failing = join(work, "failing-bin");
|
|
let cwd = "";
|
|
|
|
beforeAll(async () => {
|
|
const { cpSync, mkdirSync } = await import("node:fs");
|
|
// An image that ships one migration the database has not applied.
|
|
mkdirSync(join(appDir, "prisma"), { recursive: true });
|
|
cpSync(
|
|
join(process.cwd(), "prisma", "migrations"),
|
|
join(appDir, "prisma", "migrations"),
|
|
{ recursive: true },
|
|
);
|
|
mkdirSync(join(appDir, "prisma", "migrations", "9999_not_applied_yet"));
|
|
writeFileSync(
|
|
join(
|
|
appDir,
|
|
"prisma",
|
|
"migrations",
|
|
"9999_not_applied_yet",
|
|
"migration.sql",
|
|
),
|
|
"",
|
|
);
|
|
mkdirSync(preDir, { recursive: true });
|
|
mkdirSync(failing, { recursive: true });
|
|
writeFileSync(
|
|
join(failing, "pg_dump"),
|
|
"#!/bin/sh\necho 'pg_dump: error: could not write to output file: No space left on device' >&2\nexit 1\n",
|
|
);
|
|
chmodSync(join(failing, "pg_dump"), 0o755);
|
|
cwd = process.cwd();
|
|
process.chdir(appDir);
|
|
});
|
|
|
|
afterAll(() => {
|
|
process.chdir(cwd);
|
|
process.env.PATH = `${shims}:${originalPath}`;
|
|
});
|
|
|
|
async function preMigration(env: Record<string, string>) {
|
|
const { preMigrationDump } = await import("../../scripts/db-dump.mjs");
|
|
const errors: string[] = [];
|
|
const spyErr = vi
|
|
.spyOn(console, "error")
|
|
.mockImplementation((l: string) => void errors.push(String(l)));
|
|
const spyLog = vi.spyOn(console, "log").mockImplementation(() => {});
|
|
try {
|
|
const code = await preMigrationDump({ ...process.env, ...env });
|
|
return { code, errors: errors.join("\n") };
|
|
} finally {
|
|
spyErr.mockRestore();
|
|
spyLog.mockRestore();
|
|
}
|
|
}
|
|
|
|
it("writes a checked dump with its manifest, and keeps the last two", async () => {
|
|
for (let i = 0; i < 3; i++) {
|
|
if (i > 0) await new Promise((r) => setTimeout(r, 1100));
|
|
const { code } = await preMigration({ DB_BACKUP_DIR: preDir });
|
|
expect(code).toBe(0);
|
|
}
|
|
const names = readdirSync(preDir).sort();
|
|
expect(names).toHaveLength(4);
|
|
expect(names.filter((n) => n.endsWith("-pre-migration.dump"))).toHaveLength(
|
|
2,
|
|
);
|
|
const manifest = await readManifestFile(
|
|
join(preDir, names.find((n) => n.endsWith(".manifest.json")) ?? ""),
|
|
);
|
|
expect(manifest.kind).toBe("pre-migration");
|
|
expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count());
|
|
});
|
|
|
|
it("does nothing when switched off", async () => {
|
|
const before = readdirSync(preDir);
|
|
const { code } = await preMigration({
|
|
DB_BACKUP_DIR: preDir,
|
|
DB_BACKUP_BEFORE_MIGRATE: "false",
|
|
});
|
|
expect(code).toBe(0);
|
|
expect(readdirSync(preDir)).toEqual(before);
|
|
});
|
|
|
|
it("warns and lets the boot go on without a writable directory", async () => {
|
|
const { code, errors } = await preMigration({
|
|
DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here",
|
|
});
|
|
expect(code).toBe(0);
|
|
expect(errors).toMatch(
|
|
/is not writable, so no dump is taken before the 1 pending migration/,
|
|
);
|
|
});
|
|
|
|
it("takes no dump into a directory inside the container, says why, and lets the boot go on", async () => {
|
|
const { preMigrationDump } = await import("../../scripts/db-dump.mjs");
|
|
const before = readdirSync(preDir).sort();
|
|
const errors: string[] = [];
|
|
const spyErr = vi
|
|
.spyOn(console, "error")
|
|
.mockImplementation((l: string) => void errors.push(String(l)));
|
|
const spyLog = vi.spyOn(console, "log").mockImplementation(() => {});
|
|
try {
|
|
const code = await preMigrationDump(
|
|
{ ...process.env, DB_BACKUP_DIR: preDir },
|
|
{
|
|
directoryStatus: () => ({
|
|
writable: true,
|
|
persistent: false,
|
|
reason: "container_layer",
|
|
}),
|
|
},
|
|
);
|
|
expect(code).toBe(0);
|
|
} finally {
|
|
spyErr.mockRestore();
|
|
spyLog.mockRestore();
|
|
}
|
|
expect(errors.join("\n")).toMatch(
|
|
/is inside the container, not on a mounted volume, and would not keep a dump past this update, so no dump is taken before the 1 pending migration/,
|
|
);
|
|
expect(readdirSync(preDir).sort()).toEqual(before);
|
|
});
|
|
|
|
it("stops the boot when the dump fails on a writable directory, and leaves no dump behind", async () => {
|
|
// A second of its own, so the name cannot meet the last dump's.
|
|
await new Promise((r) => setTimeout(r, 1100));
|
|
const before = readdirSync(preDir).sort();
|
|
process.env.PATH = `${failing}:${originalPath}`;
|
|
const { code, errors } = await preMigration({ DB_BACKUP_DIR: preDir });
|
|
process.env.PATH = `${shims}:${originalPath}`;
|
|
expect(code).toBe(1);
|
|
expect(errors).toMatch(
|
|
/the database dump before migrating failed: .*No space left/,
|
|
);
|
|
expect(readdirSync(preDir).sort()).toEqual(before);
|
|
});
|
|
});
|
|
|
|
describe("two dumps in the same second", () => {
|
|
it("refuses the second and leaves the first whole", async () => {
|
|
const { createDatabaseDump } =
|
|
await import("@/lib/database-backup/dump-core.mjs");
|
|
const dir = join(work, "same-second");
|
|
const { mkdirSync } = await import("node:fs");
|
|
mkdirSync(dir);
|
|
const now = new Date("2026-10-26T01:30:00Z");
|
|
const options = {
|
|
databaseUrl: process.env.DATABASE_URL ?? "",
|
|
dir,
|
|
kind: "daily" as const,
|
|
now,
|
|
};
|
|
await createDatabaseDump(options);
|
|
const before = readdirSync(dir)
|
|
.sort()
|
|
.map((n) => [n, readFileSync(join(dir, n)).length]);
|
|
await expect(createDatabaseDump(options)).rejects.toThrow(/already exists/);
|
|
expect(
|
|
readdirSync(dir)
|
|
.sort()
|
|
.map((n) => [n, readFileSync(join(dir, n)).length]),
|
|
).toEqual(before);
|
|
});
|
|
});
|