Merge remote-tracking branch 'origin/release/v1.46.0' into fix/v146-security

This commit is contained in:
Marc-André Bombeck committed 2026-10-25 14:07:01 +01:00
commit e79ea08b13
55 files changed
+5523 -91

No files matched your search

@@ -58,7 +58,9 @@ describe("the Postgres 16 -> 18 move in docker-compose.yml", () => {
it("keeps the new data where the Postgres 18 image expects it", () => {
// Mounted at .../data, as 16 had it, the 18 image refuses to start.
expect(db.volumes).toEqual(["postgres-data:/var/lib/postgresql"]);
// `backups` holds the app's database dumps, not database files.
expect(Object.keys(compose.volumes).sort()).toEqual([
"backups",
"pgdata",
"postgres-data",
]);
@@ -54,9 +54,29 @@ vi.mock("@/lib/jobs/offhost-backup", () => ({
probeOffhostLifecycle: async () => ({
state: "missing",
expirationDays: null,
database: { state: "missing", expirationDays: null },
}),
}));
const databaseHealth = {
enabled: true,
directory: "/var/lib/healthlog/backups",
keepNightly: 7,
localDirWritable: true,
offhostConfigured: true,
lastGood: {
at: "2026-09-21T23:31:40.000Z",
fileName: "healthlog-20260921T233000Z-daily.dump",
sizeBytes: 52_428_800,
local: true,
offhost: true,
},
lastFailure: null,
};
vi.mock("@/lib/database-backup/health", () => ({
getDatabaseBackupHealth: async () => databaseHealth,
}));
import { GET } from "../route";
import type { BackupsList } from "@/types/backups";
@@ -166,7 +186,11 @@ describe("GET /api/admin/backups — off-host freshness", () => {
periodHours: 24,
rows: [],
// No bucket to ask on a host that is not configured.
lifecycle: { state: "unknown", expirationDays: null },
lifecycle: {
state: "unknown",
expirationDays: null,
database: { state: "unknown", expirationDays: null },
},
});
});
@@ -229,6 +253,7 @@ describe("GET /api/admin/backups — off-host freshness", () => {
expect(body.offhost.lifecycle).toEqual({
state: "missing",
expirationDays: null,
database: { state: "missing", expirationDays: null },
});
expect(body.offhost.pendingDeletions).toEqual({
count: 2,
@@ -285,3 +310,10 @@ describe("GET /api/admin/backups — a run in progress and one a process died un
});
});
});
describe("GET /api/admin/backups — the database dump", () => {
it("carries the dump's health beside the account copies", async () => {
const body = await read();
expect(body.database).toEqual(databaseHealth);
});
});
+10 -4
View File
@@ -14,6 +14,10 @@
* stopped six weeks ago — every row it lists has a perfectly ordinary
* timestamp either way.
*
* The database row comes first in what an operator needs and last in this
* file: the nightly whole-database dump (`getDatabaseBackupHealth`), the
* one copy that rebuilds a lost server, read from its own ledger.
*
* And alongside both, the off-host leg: one row per account saying when this
* host last put that account's encrypted copy in the operator's bucket and how
* stale that is against the nightly schedule. It comes from the ledger the
@@ -31,11 +35,13 @@ import {
} from "@/lib/jobs/job-failures";
import { readBackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
import {
offhostBackupConfigured,
probeOffhostLifecycle,
} from "@/lib/jobs/offhost-backup";
import { OFFHOST_BACKUP_QUEUE } from "@/lib/jobs/offhost-backup-queue";
import { LIFECYCLE_UNKNOWN } from "@/lib/jobs/offhost-lifecycle";
import {
classifyOffhostBackup,
OFFHOST_BACKUP_PERIOD_HOURS,
@@ -190,16 +196,15 @@ export const GET = apiHandler(async () => {
readBackupPassActivity(prisma, DATA_BACKUP_QUEUE, weeklyRunningSince),
readBackupPassActivity(prisma, OFFHOST_BACKUP_QUEUE, offhostRunningSince),
]);
const [purgeCount, oldestPurge, lifecycle] = await Promise.all([
const [purgeCount, oldestPurge, lifecycle, database] = await Promise.all([
prisma.offhostPurgeRequest.count({ where: stillInBucket }),
prisma.offhostPurgeRequest.findFirst({
where: stillInBucket,
orderBy: { requestedAt: "asc" },
select: { requestedAt: true, lastFailure: true },
}),
configured
? probeOffhostLifecycle()
: Promise.resolve({ state: "unknown" as const, expirationDays: null }),
configured ? probeOffhostLifecycle() : Promise.resolve(LIFECYCLE_UNKNOWN),
getDatabaseBackupHealth(now),
]);
const payload: BackupsList = {
@@ -229,6 +234,7 @@ export const GET = apiHandler(async () => {
},
lifecycle,
},
database,
};
return apiSuccess(payload);
@@ -26,7 +26,11 @@ import { OffhostBackupsSection } from "../offhost-backups-section";
const QUIET = {
activity: { runningSince: null, interrupted: [] },
pendingDeletions: { count: 0, oldestRequestedAt: null, lastFailure: null },
lifecycle: { state: "configured" as const, expirationDays: 30 },
lifecycle: {
state: "configured" as const,
expirationDays: 30,
database: { state: "configured" as const, expirationDays: 30 },
},
};
function render(data: BackupsList["offhost"] | null): string {
@@ -133,7 +137,11 @@ describe("<OffhostBackupsSection>", () => {
oldestRequestedAt: "2026-09-20T10:00:00.000Z",
lastFailure: "AccessDenied",
},
lifecycle: { state: "missing", expirationDays: null },
lifecycle: {
state: "missing",
expirationDays: null,
database: { state: "missing", expirationDays: null },
},
});
expect(html).toContain('data-lifecycle-state="missing"');
expect(html).toContain("no lifecycle rule");
+9
View File
@@ -28,6 +28,7 @@ import type { BackupRow, BackupsList } from "@/types/backups";
import { RestoreJobStatus, visibleRestoreJobs } from "./backup-restore-status";
import { BackupRowsCards, BackupRowsTable } from "./backups/backup-rows";
import { DatabaseDumpRow } from "./backups/database-dump-row";
import { BackupUploadCard } from "./backups/backup-upload-card";
import {
RestoreRefusalNotice,
@@ -110,6 +111,14 @@ export function BackupsSection() {
}
/>
{data?.database ? (
<DatabaseDumpRow
health={data.database}
bucketLifecycle={
data.offhost.configured ? data.offhost.lifecycle.database : null
}
/>
) : null}
{data?.schedule ? (
<ScheduleHealthNotice schedule={data.schedule} />
) : null}
@@ -0,0 +1,93 @@
/**
* The database row on the backups card says where the newest good dump is,
* how long a deleted account stays in the dumps, and each thing that keeps a
* dump from being taken or from ever ageing out of the bucket.
*/
import { describe, expect, it } from "vitest";
import { renderToStaticMarkup } from "react-dom/server";
import { I18nProvider } from "@/lib/i18n/context";
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
import { DatabaseDumpRow } from "../database-dump-row";
const HEALTHY: DatabaseBackupHealth = {
enabled: true,
directory: "/var/lib/healthlog/backups",
keepNightly: 7,
localDirWritable: true,
offhostConfigured: true,
lastGood: {
at: "2026-10-24T23:31:00.000Z",
fileName: "healthlog-20261024T233000Z-daily.dump",
sizeBytes: 52_428_800,
local: true,
offhost: true,
},
lastFailure: null,
};
function render(
health: DatabaseBackupHealth,
bucketLifecycle: LifecycleVerdict | null = {
state: "configured",
expirationDays: 30,
},
): string {
return renderToStaticMarkup(
<I18nProvider initialLocale="en">
<DatabaseDumpRow health={health} bucketLifecycle={bucketLifecycle} />
</I18nProvider>,
);
}
describe("<DatabaseDumpRow>", () => {
it("names the newest good dump, where it is, and how long dumps keep deleted accounts", () => {
const html = render(HEALTHY);
expect(html).toContain("kept on this server and off-host");
expect(html).toContain("the newest 7 are kept on this server");
expect(html).toContain("removed after 30 days");
expect(html).not.toContain('data-slot="database-dump-failed"');
expect(html).not.toContain('data-slot="database-dump-setup"');
expect(html).not.toContain('data-slot="database-dump-no-lifecycle"');
});
it("warns when no bucket rule ever removes the dumps", () => {
const html = render(HEALTHY, { state: "missing", expirationDays: null });
expect(html).toContain('data-slot="database-dump-no-lifecycle"');
expect(html).toContain("never removed");
});
it("shows a failure newer than the last good dump, and not an older one", () => {
const failed = render({
...HEALTHY,
lastFailure: {
at: "2026-10-25T23:31:00.000Z",
error: "pg_dump exited 1: No space left on device",
},
});
expect(failed).toContain('data-slot="database-dump-failed"');
expect(failed).toContain("No space left on device");
const recovered = render({
...HEALTHY,
lastFailure: { at: "2026-10-20T23:31:00.000Z", error: "old" },
});
expect(recovered).not.toContain('data-slot="database-dump-failed"');
});
it("says when dumps go off-host only, or cannot be taken at all", () => {
expect(render({ ...HEALTHY, localDirWritable: false }, null)).toContain(
"dumps go off-host only",
);
expect(
render(
{ ...HEALTHY, localDirWritable: false, offhostConfigured: false },
null,
),
).toContain("no database dump can be taken");
expect(render({ ...HEALTHY, lastGood: null }, null)).toContain(
"No database dump yet",
);
});
});
@@ -0,0 +1,148 @@
"use client";
/**
* The nightly whole-database dump, as one row on the backups card.
*
* The account copies below it restore one account on this server; the dump is
* the copy that rebuilds the server. So the row leads the card and says three
* things: when the newest good dump was taken and where it is, whether the
* newest run failed, and whether this server can keep a dump at all.
*/
import { AlertTriangle, HardDrive } from "lucide-react";
import { SettingsInfoTile } from "@/components/settings/_info-tile";
import { formatBytes } from "@/lib/format/bytes";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
function whereKept(
good: NonNullable<DatabaseBackupHealth["lastGood"]>,
t: ReturnType<typeof useTranslations>["t"],
) {
if (good.local && good.offhost) {
return t("admin.section.backups.database.whereBoth");
}
if (good.offhost) return t("admin.section.backups.database.whereOffhost");
return t("admin.section.backups.database.whereLocal");
}
export function DatabaseDumpRow({
health,
bucketLifecycle,
}: {
health: DatabaseBackupHealth;
/** The bucket's rule for `database/`; null when nothing goes off-host. */
bucketLifecycle: LifecycleVerdict | null;
}) {
const { t, locale } = useTranslations();
const fmt = useFormatters();
const good = health.lastGood;
const failure = health.lastFailure;
const failedSinceGood =
failure !== null && (good === null || failure.at > good.at);
let setupProblem: string | null = null;
if (!health.enabled) {
setupProblem = t("admin.section.backups.database.disabled");
} else if (!health.localDirWritable) {
setupProblem = health.offhostConfigured
? t("admin.section.backups.database.noDirectoryOffhost", {
dir: health.directory,
})
: t("admin.section.backups.database.noDirectory", {
dir: health.directory,
});
}
return (
<div className="space-y-2" data-slot="database-dump">
<div className="flex items-start gap-2">
<HardDrive
className="text-muted-foreground mt-0.5 h-4 w-4 shrink-0"
aria-hidden="true"
/>
<div>
<p className="text-sm font-medium">
{t("admin.section.backups.database.title")}
</p>
<p
className="text-muted-foreground text-xs"
data-slot="database-dump-last-good"
>
{good
? t("admin.section.backups.database.lastGood", {
when: fmt.dateTime(good.at),
size: formatBytes(good.sizeBytes ?? 0, locale),
where: whereKept(good, t),
})
: t("admin.section.backups.database.none")}
</p>
{/* How long a deleted account stays in the dumps: the operator's
side of the deletion promise (GDPR retention of backups). */}
<p
className="text-muted-foreground text-xs"
data-slot="database-dump-retention"
>
{t("admin.section.backups.database.retentionLocal", {
nights: health.keepNightly,
})}
{bucketLifecycle?.state === "configured"
? ` ${t("admin.section.backups.database.retentionOffhost", {
days: bucketLifecycle.expirationDays ?? 0,
})}`
: null}
</p>
</div>
</div>
{failedSinceGood && failure ? (
<div
role="alert"
data-slot="database-dump-failed"
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start gap-2">
<AlertTriangle
className="mt-0.5 h-4 w-4 shrink-0"
aria-hidden="true"
/>
<div>
<p className="font-medium">
{t("admin.section.backups.database.failedTitle")}
</p>
<p className="text-xs">
{t("admin.section.backups.database.failedDescription", {
when: fmt.dateTime(failure.at),
reason: failure.error,
})}
</p>
</div>
</div>
</div>
) : null}
{bucketLifecycle?.state === "missing" ? (
<SettingsInfoTile
icon={AlertTriangle}
tone="warning"
role="alert"
data-slot="database-dump-no-lifecycle"
>
{t("admin.section.backups.database.retentionOffhostMissing")}
</SettingsInfoTile>
) : null}
{setupProblem ? (
<SettingsInfoTile
icon={AlertTriangle}
tone="warning"
role="alert"
data-slot="database-dump-setup"
>
{setupProblem}
</SettingsInfoTile>
) : null}
</div>
);
}
@@ -149,3 +149,56 @@ describe("collectReadiness — retired AI grant", () => {
expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined();
});
});
describe("collectReadiness — the database dump", () => {
const offhost = {
BACKUP_S3_ENDPOINT: "https://s3.example.test",
BACKUP_S3_BUCKET: "b",
BACKUP_S3_ACCESS_KEY: "a",
BACKUP_S3_SECRET_KEY: "s",
BACKUP_ENCRYPTION_KEY: HEX32,
};
const dumpLine = (
env: Record<string, string | undefined>,
writable: boolean,
) =>
collectReadiness(env, () => writable).lines.find(
(l) => l.label === "Database dump",
);
it("names the schedule, the directory and the copies kept", () => {
const line = dumpLine(baseEnv(), true);
expect(line?.status).toBe("ok");
expect(line?.detail).toBe(
"nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7",
);
expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain(
"copied off-host",
);
});
it("warns when nothing can be kept on the server", () => {
expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({
status: "warn",
detail: expect.stringContaining("goes off-host only"),
});
expect(dumpLine(baseEnv(), false)).toMatchObject({
status: "warn",
detail: expect.stringContaining("no database dump can be taken"),
});
});
it("warns when it is switched off, and names a schedule it ignored", () => {
expect(
dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true),
).toMatchObject({ status: "warn" });
const report = collectReadiness(
{ ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" },
() => true,
);
expect(
report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail,
).toBe("not a five-field cron; using 30 1 * * *");
expect(report.hasBlocker).toBe(false);
});
});
+76 -2
View File
@@ -19,6 +19,13 @@ import {
LEGACY_ANY_HOST_WARNING,
legacyAnyHostConfigured,
} from "@/lib/ai/local-host-allowlist";
import {
databaseBackupEnabled,
isWritableDirectory,
resolveBackupDir,
resolveBackupSchedule,
resolveDailyKeep,
} from "@/lib/database-backup/dump-core.mjs";
export interface ReadinessLine {
/** Short label, e.g. "ENCRYPTION_KEY". */
@@ -60,8 +67,73 @@ function anySet(env: Env, keys: string[]): boolean {
return keys.some((k) => (env[k] ?? "").trim().length > 0);
}
/** Inspect the environment and build the readiness report. Pure + side-effect free. */
export function collectReadiness(env: Env = process.env): ReadinessReport {
/**
* The nightly database dump's line. The dump is what rebuilds a lost server,
* so a host where it cannot be kept anywhere is said out loud at every boot,
* and so is one where it only goes off-host.
*/
export function databaseBackupReadiness(
env: Env,
writable: (dir: string) => boolean,
): ReadinessLine[] {
const label = "Database dump";
if (!databaseBackupEnabled(env)) {
return [
{
label,
status: "warn",
detail:
"DB_BACKUP_ENABLED=false: no nightly database dump is taken; back the database up another way",
},
];
}
const lines: ReadinessLine[] = [];
const schedule = resolveBackupSchedule(env);
if (!schedule.valid) {
lines.push({
label: "DB_BACKUP_SCHEDULE",
status: "warn",
detail: `not a five-field cron; using ${schedule.cron}`,
});
}
const dir = resolveBackupDir(env);
const offhost = allSet(env, [
"BACKUP_S3_ENDPOINT",
"BACKUP_S3_BUCKET",
"BACKUP_S3_ACCESS_KEY",
"BACKUP_S3_SECRET_KEY",
"BACKUP_ENCRYPTION_KEY",
]);
if (writable(dir)) {
lines.push({
label,
status: "ok",
detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`,
});
} else if (offhost) {
lines.push({
label,
status: "warn",
detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
});
} else {
lines.push({
label,
status: "warn",
detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
});
}
return lines;
}
/**
* Inspect the environment and build the readiness report. Pure apart from
* `writable`, the probe of the dump directory, which tests replace.
*/
export function collectReadiness(
env: Env = process.env,
writable: (dir: string) => boolean = isWritableDirectory,
): ReadinessReport {
const lines: ReadinessLine[] = [];
// --- Core secrets (boot blockers) ---------------------------------------
@@ -139,6 +211,8 @@ export function collectReadiness(env: Env = process.env): ReadinessReport {
});
}
lines.push(...databaseBackupReadiness(env, writable));
// --- Optional subsystems (informational) --------------------------------
const optional: Array<[string, boolean]> = [
["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])],
+2
View File
@@ -353,6 +353,8 @@ export const INSTANCE_SCOPED: Readonly<Record<string, string>> = {
HostMetric: "instance-wide host telemetry, not attributable to an account",
OffhostBackupKeyUse:
"per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account",
DatabaseBackupRun:
"the instance's ledger of nightly whole-database dumps; describes the server, not any account",
OffhostPurgeRequest:
"the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account",
CycleSymptomCategory:
@@ -0,0 +1,689 @@
/**
* The rules of the whole-database dump that need no database: names,
* retention, the manifest, key fingerprints, row-count comparison, the
* connection the client tools get, and the off-host envelope read back by the
* restore's own reader.
*
* The fingerprints and the envelope are checked against the app's own code
* (`src/lib/crypto.ts`, `src/lib/jobs/offhost-backup.ts`), not against a copy
* of the rule: the restore script cannot import TypeScript, so these are the
* tests that keep its plain-JS reading of both from drifting.
*/
import {
mkdtempSync,
createReadStream,
rmSync,
utimesSync,
writeFileSync,
} from "node:fs";
import { readdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { Readable } from "node:stream";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
applyRetention,
bucketConfigFromEnv,
pickBucketDump,
buildManifest,
compareKeyFingerprints,
compareRowCounts,
connectionFromUrl,
decodeKeyMaterial,
decryptOffhostFile,
dumpBaseName,
formatDumpTimestamp,
keyFingerprints,
listDumpSets,
offhostObjectKey,
parseDumpFileName,
parseManifest,
resolveBackupSchedule,
resolveDailyKeep,
selectForDeletion,
type DumpSet,
} from "../dump-core.mjs";
import type { S3Like } from "@/lib/jobs/offhost-backup";
/** Every byte of a stream, in order. */
async function collect(stream: AsyncIterable<unknown>): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const c of stream) chunks.push(Buffer.from(c as Uint8Array));
return Buffer.concat(chunks);
}
const HEX_A = "a".repeat(64);
const HEX_B = "b".repeat(64);
const HEX_C = "c".repeat(64);
const HEX_D = "d".repeat(64);
const dirs: string[] = [];
function tempDir(): string {
const dir = mkdtempSync(join(tmpdir(), "hl-dump-core-"));
dirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of dirs.splice(0))
rmSync(dir, { recursive: true, force: true });
vi.unstubAllEnvs();
});
describe("dump names", () => {
it("names a dump by its UTC instant and kind", () => {
const at = new Date("2026-10-25T01:30:07.123Z");
expect(formatDumpTimestamp(at)).toBe("20261025T013007Z");
expect(dumpBaseName("daily", at)).toBe("healthlog-20261025T013007Z-daily");
expect(dumpBaseName("pre-migration", at)).toBe(
"healthlog-20261025T013007Z-pre-migration",
);
});
it("parses every file it writes, and back to the same instant", () => {
const base = "healthlog-20261025T013007Z-daily";
const cases: Array<[string, string, string]> = [
[`${base}.dump`, "dump", "final"],
[`${base}.manifest.json`, "manifest", "final"],
[`${base}.dump.partial`, "dump", "partial"],
[`${base}.manifest.json.partial`, "manifest", "partial"],
[`${base}.dump.enc`, "dump", "encrypted"],
[`${base}.manifest.json.enc`, "manifest", "encrypted"],
];
for (const [name, role, variant] of cases) {
const parsed = parseDumpFileName(name);
expect(parsed, name).not.toBeNull();
expect(parsed?.base).toBe(base);
expect(parsed?.kind).toBe("daily");
expect(parsed?.role).toBe(role);
expect(parsed?.variant).toBe(variant);
expect(parsed?.timestamp.toISOString()).toBe("2026-10-25T01:30:07.000Z");
}
expect(
parseDumpFileName("healthlog-20261025T013007Z-pre-migration.dump")?.kind,
).toBe("pre-migration");
});
it("refuses every name it did not write", () => {
for (const name of [
"healthlog.dump",
"backup-20261025T013007Z-daily.dump",
"healthlog-20261025T013007Z-weekly.dump",
"healthlog-20261025-daily.dump",
"healthlog-20261025T013007Z-daily.sql",
"healthlog-20261025T013007Z-daily.dump.bak",
"healthlog-20261399T013007Z-daily.dump",
"notes.txt",
]) {
expect(parseDumpFileName(name), name).toBeNull();
}
});
});
describe("configuration", () => {
it("keeps at least one daily dump and ignores what is not a number", () => {
expect(resolveDailyKeep({})).toBe(7);
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "14" })).toBe(14);
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "0" })).toBe(1);
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "-3" })).toBe(7);
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "seven" })).toBe(7);
});
it("takes a five-field cron and says when it ignored one", () => {
expect(resolveBackupSchedule({})).toEqual({
cron: "30 1 * * *",
valid: true,
configured: false,
});
expect(
resolveBackupSchedule({ DB_BACKUP_SCHEDULE: " 0 2 * * 1-5 " }),
).toEqual({
cron: "0 2 * * 1-5",
valid: true,
configured: true,
});
for (const bad of ["daily", "0 2 * *", "0 2 * * * *", "0 2 * * mon; rm"]) {
expect(resolveBackupSchedule({ DB_BACKUP_SCHEDULE: bad })).toEqual({
cron: "30 1 * * *",
valid: false,
configured: true,
});
}
});
});
/** A dump set as `listDumpSets` reports it, for the pure retention rule. */
function set(
kind: "daily" | "pre-migration",
iso: string,
parts: { dump?: boolean; manifest?: boolean; partialAgeMs?: number } = {},
now = Date.parse("2026-10-25T12:00:00Z"),
): DumpSet {
const base = dumpBaseName(kind, new Date(iso));
return {
base,
kind,
timestamp: new Date(iso),
dump: parts.dump === false ? null : { path: `/b/${base}.dump`, bytes: 1 },
manifest: parts.manifest === false ? null : `/b/${base}.manifest.json`,
encryptedDump: null,
encryptedManifest: null,
leftovers:
parts.partialAgeMs === undefined
? []
: [
{
path: `/b/${base}.dump.partial`,
mtimeMs: now - parts.partialAgeMs,
},
],
};
}
describe("retention", () => {
const now = Date.parse("2026-10-25T12:00:00Z");
it("keeps the newest N daily dumps and removes the rest, both files each", () => {
const sets = [1, 2, 3, 4, 5].map((d) =>
set("daily", `2026-10-2${d}T01:30:00Z`),
);
const doomed = selectForDeletion(sets, {
keep: { daily: 3, "pre-migration": 2 },
now,
});
expect(doomed.sort()).toEqual(
[
"/b/healthlog-20261021T013000Z-daily.dump",
"/b/healthlog-20261021T013000Z-daily.manifest.json",
"/b/healthlog-20261022T013000Z-daily.dump",
"/b/healthlog-20261022T013000Z-daily.manifest.json",
].sort(),
);
});
it("never removes the newest good dump, whatever keep says", () => {
const sets = [
set("daily", "2026-10-24T01:30:00Z"),
set("daily", "2026-10-23T01:30:00Z"),
];
const doomed = selectForDeletion(sets, {
keep: { daily: 0, "pre-migration": 0 },
now,
});
expect(doomed).not.toContain("/b/healthlog-20261024T013000Z-daily.dump");
expect(doomed).toContain("/b/healthlog-20261023T013000Z-daily.dump");
});
it("counts only good dumps: an unfinished newer one does not push out the newest good one", () => {
const sets = [
set("daily", "2026-10-25T01:30:00Z", {
dump: false,
partialAgeMs: 60_000,
}),
set("daily", "2026-10-24T01:30:00Z"),
];
const doomed = selectForDeletion(sets, {
keep: { daily: 1, "pre-migration": 2 },
now,
});
expect(doomed).toEqual([]);
});
it("keeps two pre-migration dumps apart from the daily count", () => {
const sets = [
set("pre-migration", "2026-10-20T08:00:00Z"),
set("pre-migration", "2026-10-10T08:00:00Z"),
set("pre-migration", "2026-10-01T08:00:00Z"),
set("daily", "2026-10-24T01:30:00Z"),
];
const doomed = selectForDeletion(sets, {
keep: { daily: 1, "pre-migration": 2 },
now,
});
expect(doomed).toEqual([
"/b/healthlog-20261001T080000Z-pre-migration.dump",
"/b/healthlog-20261001T080000Z-pre-migration.manifest.json",
]);
});
it("removes a .partial only once it is older than twelve hours", () => {
const fresh = set("daily", "2026-10-25T01:30:00Z", {
dump: false,
manifest: false,
partialAgeMs: 60 * 60_000,
});
const stale = set("daily", "2026-10-23T01:30:00Z", {
dump: false,
manifest: false,
partialAgeMs: 13 * 60 * 60_000,
});
const doomed = selectForDeletion([fresh, stale], {
keep: { daily: 7, "pre-migration": 2 },
now,
});
expect(doomed).toEqual([
"/b/healthlog-20261023T013000Z-daily.dump.partial",
]);
});
it("works on a real directory and leaves files it did not write alone", async () => {
const dir = tempDir();
const names: string[] = [];
for (let day = 15; day <= 24; day++) {
const base = `healthlog-202610${day}T013000Z-daily`;
names.push(`${base}.dump`, `${base}.manifest.json`);
}
// Two that are not ours, and an off-host copy an operator put here.
names.push(
"my-own-dump.dump",
"healthlog-20261001T013000Z-daily.dump.bak",
"healthlog-20261002T013000Z-daily.dump.enc",
);
for (const name of names) writeFileSync(join(dir, name), "x");
// A crash's leftovers: one old, one from a run that may still be going.
const oldPartial = join(
dir,
"healthlog-20261003T013000Z-daily.dump.partial",
);
writeFileSync(oldPartial, "x");
const day = 24 * 60 * 60;
utimesSync(oldPartial, Date.now() / 1000 - day, Date.now() / 1000 - day);
writeFileSync(
join(dir, "healthlog-20261025T013000Z-daily.dump.partial"),
"x",
);
const removed = await applyRetention(dir, { dailyKeep: 7 });
expect(removed).toHaveLength(7);
const left = (await readdir(dir)).sort();
expect(left).toContain("my-own-dump.dump");
expect(left).toContain("healthlog-20261001T013000Z-daily.dump.bak");
expect(left).toContain("healthlog-20261002T013000Z-daily.dump.enc");
expect(left).toContain("healthlog-20261025T013000Z-daily.dump.partial");
expect(left).not.toContain("healthlog-20261003T013000Z-daily.dump.partial");
expect(left.filter((n) => n.endsWith("-daily.dump"))).toEqual(
[18, 19, 20, 21, 22, 23, 24].map(
(d) => `healthlog-202610${d}T013000Z-daily.dump`,
),
);
expect(listDumpSets(dir)[1].base).toBe("healthlog-20261024T013000Z-daily");
});
});
describe("key fingerprints", () => {
it("are the fingerprints the app's own key code gives, for the map and the single key", async () => {
vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ v1: HEX_A, v2: HEX_B }));
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "v2");
vi.stubEnv("ENCRYPTION_KEY", "");
const crypto = await import("@/lib/crypto");
crypto._resetCryptoCacheForTests();
const prints = keyFingerprints(process.env);
expect(prints.encryption.activeKeyId).toBe("v2");
expect(prints.encryption.keys).toEqual([
{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") },
{ id: "v2", fingerprint: crypto.getKeyFingerprint("v2") },
]);
vi.stubEnv("ENCRYPTION_KEYS", "");
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "");
const base64 = Buffer.from(HEX_C, "hex").toString("base64");
vi.stubEnv("ENCRYPTION_KEY", base64);
crypto._resetCryptoCacheForTests();
const single = keyFingerprints(process.env);
expect(single.encryption).toEqual({
activeKeyId: "v1",
keys: [{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }],
});
crypto._resetCryptoCacheForTests();
});
it("names the off-host key the way an off-host object records it", async () => {
const { offhostKeyId } = await import("@/lib/jobs/offhost-backup");
const prints = keyFingerprints({
ENCRYPTION_KEY: HEX_A,
BACKUP_ENCRYPTION_KEY: HEX_D,
BACKUP_ENCRYPTION_PREVIOUS_KEYS: ` ${HEX_C} ,`,
API_TOKEN_HMAC_KEY: "hmac-secret-of-some-length-0123456789",
});
expect(prints.backupEncryptionKey).toBe(
offhostKeyId(Buffer.from(HEX_D, "hex")),
);
expect(prints.backupEncryptionPreviousKeys).toEqual([
offhostKeyId(Buffer.from(HEX_C, "hex")),
]);
expect(prints.apiTokenHmacKey).toMatch(/^[0-9a-f]{12}$/);
});
it("never carries key material", () => {
const env = {
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A }),
BACKUP_ENCRYPTION_KEY: HEX_B,
API_TOKEN_HMAC_KEY: HEX_C,
};
const text = JSON.stringify(keyFingerprints(env));
for (const secret of [HEX_A, HEX_B, HEX_C]) {
expect(text).not.toContain(secret.slice(0, 16));
}
});
it("decodes keys the way the app does, and nothing that is not a key", () => {
expect(decodeKeyMaterial(HEX_A)?.length).toBe(32);
expect(
decodeKeyMaterial(Buffer.alloc(32, 7).toString("base64"))?.length,
).toBe(32);
expect(decodeKeyMaterial("")).toBeNull();
expect(decodeKeyMaterial("not a key!")).toBeNull();
});
});
describe("comparing a dump's keys with the environment's", () => {
const written = keyFingerprints({
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_B }),
ENCRYPTION_ACTIVE_KEY_ID: "v2",
BACKUP_ENCRYPTION_KEY: HEX_C,
API_TOKEN_HMAC_KEY: "hmac-one",
});
it("finds nothing when every key is there", () => {
const same = keyFingerprints({
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B, v1: HEX_A }),
ENCRYPTION_ACTIVE_KEY_ID: "v2",
BACKUP_ENCRYPTION_KEY: HEX_C,
API_TOKEN_HMAC_KEY: "hmac-one",
});
expect(compareKeyFingerprints(written, same)).toEqual({
findings: [],
refuse: false,
});
});
it("refuses when the active key is different, and says which", () => {
const wrong = keyFingerprints({
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_D }),
ENCRYPTION_ACTIVE_KEY_ID: "v2",
BACKUP_ENCRYPTION_KEY: HEX_C,
API_TOKEN_HMAC_KEY: "hmac-one",
});
const result = compareKeyFingerprints(written, wrong);
expect(result.refuse).toBe(true);
expect(result.findings).toHaveLength(1);
expect(result.findings[0]).toMatchObject({
key: "ENCRYPTION_KEYS[v2]",
state: "different",
severity: "refuse",
});
});
it("refuses when the active key is missing, warns for a retired one", () => {
const onlyOld = keyFingerprints({ ENCRYPTION_KEY: HEX_A });
const result = compareKeyFingerprints(written, onlyOld);
expect(result.refuse).toBe(true);
const byKey = Object.fromEntries(result.findings.map((f) => [f.key, f]));
expect(byKey["ENCRYPTION_KEYS[v2]"]).toMatchObject({
state: "missing",
severity: "refuse",
});
expect(byKey["ENCRYPTION_KEYS[v1]"]).toBeUndefined();
expect(byKey.BACKUP_ENCRYPTION_KEY).toMatchObject({
state: "missing",
severity: "warn",
});
expect(byKey.API_TOKEN_HMAC_KEY).toMatchObject({
state: "missing",
severity: "warn",
});
const noRetired = keyFingerprints({
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B }),
BACKUP_ENCRYPTION_KEY: HEX_C,
API_TOKEN_HMAC_KEY: "hmac-two",
});
const second = compareKeyFingerprints(written, noRetired);
expect(second.refuse).toBe(false);
expect(second.findings.map((f) => [f.key, f.severity])).toEqual([
["ENCRYPTION_KEYS[v1]", "warn"],
["API_TOKEN_HMAC_KEY", "warn"],
]);
});
});
describe("the manifest", () => {
const manifest = buildManifest({
kind: "daily",
file: "healthlog-20261025T013000Z-daily.dump",
createdAt: new Date("2026-10-25T01:30:00Z"),
appVersion: "1.46.0",
buildSha: "abc1234",
latestMigration: "0486_database_backup_runs",
server: { version: "18.6", versionNum: 180006 },
pgDump: "pg_dump (PostgreSQL) 18.6",
archive: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
rowCounts: { "public.users": 3, "pgboss.job": 12, "public.a": 0 },
env: { ENCRYPTION_KEY: HEX_A, API_TOKEN_HMAC_KEY: "h" },
});
it("carries what a restore and an operator need", () => {
expect(manifest).toMatchObject({
format: "healthlog-database-dump",
formatVersion: 1,
kind: "daily",
file: "healthlog-20261025T013000Z-daily.dump",
createdAt: "2026-10-25T01:30:00.000Z",
appVersion: "1.46.0",
buildSha: "abc1234",
latestMigration: "0486_database_backup_runs",
postgres: { serverVersion: "18.6", serverVersionNum: 180006 },
dump: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
});
expect(Object.keys(manifest.rowCounts)).toEqual([
"pgboss.job",
"public.a",
"public.users",
]);
expect(manifest.keys.encryption.keys[0].id).toBe("v1");
});
it("reads back, and refuses anything else", () => {
expect(parseManifest(JSON.parse(JSON.stringify(manifest)))).toEqual(
manifest,
);
expect(() => parseManifest({ format: "something-else" })).toThrow(
/not a HealthLog/,
);
expect(() => parseManifest({ ...manifest, formatVersion: 2 })).toThrow();
expect(() => parseManifest(null)).toThrow();
});
});
describe("row counts after a restore", () => {
it("names every table that is missing, extra or different", () => {
expect(
compareRowCounts(
{ "public.a": 1, "public.b": 2 },
{ "public.a": 1, "public.b": 2 },
),
).toEqual([]);
expect(
compareRowCounts(
{ "public.a": 1, "public.b": 2, "public.c": 5 },
{ "public.a": 1, "public.b": 0, "public.d": 1 },
),
).toEqual([
"public.b: 2 rows in the dump, 0 restored",
"public.c: in the dump with 5 rows, missing now",
"public.d: restored but not in the dump's manifest",
]);
});
});
describe("the connection the client tools get", () => {
it("drops Prisma's parameters and keeps the password out of the arguments", () => {
const c = connectionFromUrl(
"postgresql://health%40log:p%40ss%2Fw@db:6543/hl?schema=app&connection_limit=20&pool_timeout=20&sslmode=require",
);
expect(c.pgEnv).toEqual({
PGHOST: "db",
PGPORT: "6543",
PGUSER: "health@log",
PGPASSWORD: "p@ss/w",
PGDATABASE: "hl",
PGSSLMODE: "require",
});
expect(c.schema).toBe("app");
const cs = String(c.clientConfig.connectionString);
expect(cs).not.toContain("connection_limit");
expect(cs).not.toContain("schema=");
expect(cs).toContain("sslmode=require");
});
});
describe("an off-host copy read back by the restore", () => {
/** The uploader the job uses, into memory. */
async function upload(objectKey: string, key: Buffer, body: Buffer) {
const { uploadEncryptedBackup } = await import("@/lib/jobs/offhost-backup");
const stored = new Map<string, Buffer>();
const s3: S3Like = {
putStream: async (k: string, stream: Readable) => {
stored.set(k, await collect(stream));
},
putObject: async () => {},
getObject: async () => Buffer.alloc(0),
headObject: async () => false,
listObjects: async () => [],
deleteObject: async () => {},
};
await uploadEncryptedBackup(s3, objectKey, key, async (write) => {
// Several pieces, as the job streams the file.
for (let i = 0; i < body.length; i += 1000) {
await write(body.subarray(i, i + 1000));
}
});
return stored.get(objectKey) as Buffer;
}
it("opens what the job's uploader sealed, byte for byte", async () => {
const dir = tempDir();
const key = Buffer.from(HEX_D, "hex");
const body = Buffer.from(Array.from({ length: 50_000 }, (_, i) => i % 251));
const name = "healthlog-20261025T013000Z-daily.dump.enc";
const sealed = await upload(offhostObjectKey(name), key, body);
writeFileSync(join(dir, name), sealed);
await decryptOffhostFile({
inPath: join(dir, name),
outPath: join(dir, "opened.dump"),
objectKey: offhostObjectKey(name),
keys: [Buffer.from(HEX_A, "hex"), key],
});
expect(
(await collect(createReadStream(join(dir, "opened.dump")))).equals(body),
).toBe(true);
});
it("refuses a copy renamed, altered or under a key it does not hold, and leaves nothing", async () => {
const dir = tempDir();
const key = Buffer.from(HEX_D, "hex");
const body = Buffer.from("PGDMP fake archive bytes".repeat(100));
const name = "healthlog-20261025T013000Z-daily.dump.enc";
const sealed = await upload(offhostObjectKey(name), key, body);
writeFileSync(join(dir, name), sealed);
const out = join(dir, "opened.dump");
// Renamed: the object key is bound as associated data.
await expect(
decryptOffhostFile({
inPath: join(dir, name),
outPath: out,
objectKey: offhostObjectKey(
"healthlog-20261024T013000Z-daily.dump.enc",
),
keys: [key],
}),
).rejects.toThrow(/does not open/);
expect(await readdir(dir)).toEqual([name]);
// Altered.
const altered = Buffer.from(sealed);
altered[altered.length - 40] ^= 1;
writeFileSync(join(dir, name), altered);
await expect(
decryptOffhostFile({
inPath: join(dir, name),
outPath: out,
objectKey: offhostObjectKey(name),
keys: [key],
}),
).rejects.toThrow();
expect(await readdir(dir)).toEqual([name]);
// A key it was not written under.
writeFileSync(join(dir, name), sealed);
await expect(
decryptOffhostFile({
inPath: join(dir, name),
outPath: out,
objectKey: offhostObjectKey(name),
keys: [Buffer.from(HEX_A, "hex")],
}),
).rejects.toThrow(/neither BACKUP_ENCRYPTION_KEY nor/);
expect(await readdir(dir)).toEqual([name]);
});
});
describe("finding a dump in the bucket", () => {
const keys = [
"database/healthlog-20261023T013000Z-daily.dump.enc",
"database/healthlog-20261023T013000Z-daily.manifest.json.enc",
"database/healthlog-20261024T013000Z-daily.dump.enc",
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
// The newest dump whose manifest never arrived, and things that are
// not dumps at all.
"database/healthlog-20261025T013000Z-daily.dump.enc",
"2026-10-25/user-abc.json.enc",
"database/notes.txt",
];
it("takes the newest dump whose manifest is there too", () => {
expect(pickBucketDump(keys)).toEqual({
dumpKey: "database/healthlog-20261024T013000Z-daily.dump.enc",
manifestKey:
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
});
});
it("finds a named one with or without its suffix, and nothing else", () => {
for (const name of [
"healthlog-20261023T013000Z-daily.dump",
"healthlog-20261023T013000Z-daily.dump.enc",
"database/healthlog-20261023T013000Z-daily.dump.enc",
]) {
expect(pickBucketDump(keys, name)?.dumpKey).toBe(
"database/healthlog-20261023T013000Z-daily.dump.enc",
);
}
expect(
pickBucketDump(keys, "healthlog-20261025T013000Z-daily.dump"),
).toBeNull();
expect(pickBucketDump([])).toBeNull();
});
it("reads the bucket from the four required variables", () => {
const env = {
BACKUP_S3_ENDPOINT: "https://s3.example.test",
BACKUP_S3_BUCKET: "b",
BACKUP_S3_ACCESS_KEY: "a",
BACKUP_S3_SECRET_KEY: "s",
};
expect(bucketConfigFromEnv(env)).toEqual({
endpoint: "https://s3.example.test",
bucket: "b",
accessKey: "a",
secretKey: "s",
region: "auto",
});
expect(
bucketConfigFromEnv({ ...env, BACKUP_S3_SECRET_KEY: "" }),
).toBeNull();
});
});
File diff suppressed because it is too large. Load diff
+125
View File
@@ -0,0 +1,125 @@
/**
* Whether the nightly whole-database dump is happening, as facts a page or an
* alert can act on.
*
* Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of
* this process: whether `DB_BACKUP_DIR` is writable here, and whether the
* off-host bucket is configured. A ledger row that says a copy went off-host
* outranks this process's environment, for the reason the off-host card gives
* (a split install can hand the bucket variables to the worker only).
*/
import { prisma } from "@/lib/db";
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
import {
databaseBackupEnabled,
isWritableDirectory,
resolveBackupDir,
resolveDailyKeep,
} from "./dump-core.mjs";
export interface DatabaseBackupHealth {
/** `DB_BACKUP_ENABLED` is not off. */
enabled: boolean;
/** The directory dumps are kept in (`DB_BACKUP_DIR`). */
directory: string;
/**
* Nightly dumps kept in that directory (`DB_BACKUP_KEEP`). Also how long a
* deleted account stays in the dumps on this server, which the backups
* page says.
*/
keepNightly: number;
/** This process can write there. False means dumps only go off-host. */
localDirWritable: boolean;
/** The off-host bucket is configured, or a dump has been uploaded to it. */
offhostConfigured: boolean;
/** The newest dump that was kept on the server or reached the bucket. */
lastGood: {
at: string;
fileName: string;
sizeBytes: number | null;
local: boolean;
offhost: boolean;
} | null;
/** The newest run that failed, or that a dying process left unfinished. */
lastFailure: { at: string; error: string } | null;
}
/**
* A run still `running` this long after it started was cut off by its
* process: the job's expiry, past which pg-boss has failed it anyway.
*/
const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000;
export async function getDatabaseBackupHealth(
now: Date = new Date(),
env: NodeJS.ProcessEnv = process.env,
): Promise<DatabaseBackupHealth> {
const directory = resolveBackupDir(env);
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
[
prisma.databaseBackupRun.findFirst({
where: {
fileName: { not: null },
OR: [{ localResult: "written" }, { offhostResult: "uploaded" }],
},
orderBy: { startedAt: "desc" },
select: {
startedAt: true,
finishedAt: true,
fileName: true,
dumpBytes: true,
localResult: true,
offhostResult: true,
},
}),
prisma.databaseBackupRun.findFirst({
where: { status: "failed" },
orderBy: { startedAt: "desc" },
select: { startedAt: true, finishedAt: true, error: true },
}),
prisma.databaseBackupRun.findFirst({
where: {
status: "running",
startedAt: { lt: new Date(now.getTime() - ABANDONED_AFTER_MS) },
},
orderBy: { startedAt: "desc" },
select: { startedAt: true },
}),
prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }),
],
);
const failures = [
lastFailed && {
at: (lastFailed.finishedAt ?? lastFailed.startedAt).toISOString(),
error: lastFailed.error ?? "failed without a reason",
},
lastAbandoned && {
at: lastAbandoned.startedAt.toISOString(),
error: "the run did not finish (the process stopped under it)",
},
]
.filter((f): f is { at: string; error: string } => Boolean(f))
.sort((a, b) => b.at.localeCompare(a.at));
return {
enabled: databaseBackupEnabled(env),
directory,
keepNightly: resolveDailyKeep(env),
localDirWritable: isWritableDirectory(directory),
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
lastGood: lastGood
? {
at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(),
fileName: lastGood.fileName ?? "",
sizeBytes:
lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes),
local: lastGood.localResult === "written",
offhost: lastGood.offhostResult === "uploaded",
}
: null,
lastFailure: failures[0] ?? null,
};
}
+243
View File
@@ -0,0 +1,243 @@
/**
* One run of the nightly whole-database dump.
*
* The dump is the artifact that rebuilds a lost server: every account, every
* login, every grant and managed profile, the job queue and the instance
* settings, in one `pg_dump --format=custom` file with a manifest beside it
* (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the
* bundled compose file) and, when the off-host bucket is configured, uploaded
* under `database/` in the same envelope the per-account copies use, sealed
* with `BACKUP_ENCRYPTION_KEY`.
*
* Without a writable directory (an install that runs the image without the
* volume) the dump is written to a temporary directory, uploaded and removed:
* the off-host copy still happens, nothing is kept on the server, and the
* ledger and the boot summary say so. With neither, the run is skipped and
* recorded as such.
*
* Every run leaves one `DatabaseBackupRun` row: started, then finished with
* what happened on each leg. `getDatabaseBackupHealth` reads them.
*/
import { createReadStream } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import type { PrismaClient } from "@/generated/prisma/client";
import {
getS3Client,
loadOffhostConfig,
uploadEncryptedBackup,
type S3Like,
} from "@/lib/jobs/offhost-backup";
import { annotate } from "@/lib/logging/context";
import packageJson from "../../../package.json";
import {
applyRetention,
createDatabaseDump,
databaseBackupEnabled,
isWritableDirectory,
offhostObjectKey,
resolveBackupDir,
resolveDailyKeep,
} from "./dump-core.mjs";
/** How long ledger rows are kept. A run a day; a year and a bit of history. */
const LEDGER_RETENTION_DAYS = 400;
export type DatabaseBackupLocalResult =
"written" | "no_directory" | "failed" | "skipped";
export type DatabaseBackupOffhostResult =
"uploaded" | "not_configured" | "failed" | "skipped";
export interface DatabaseBackupReport {
status: "succeeded" | "failed" | "skipped";
fileName: string | null;
dumpBytes: number | null;
local: DatabaseBackupLocalResult;
offhost: DatabaseBackupOffhostResult;
/** Dumps and manifests retention removed. */
removed: number;
error: string | null;
}
export interface RunDatabaseBackupOptions {
now?: Date;
/** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */
deadline?: number;
env?: NodeJS.ProcessEnv;
/** A bucket double for tests; the configured bucket otherwise. */
s3?: S3Like;
}
/** The running build's version, as `/api/version` answers it. */
function appVersion(env: NodeJS.ProcessEnv): string {
return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version;
}
/** An error as one line for the ledger and the admin page. */
function reason(err: unknown): string {
const text = err instanceof Error ? err.message : String(err);
return text.replace(/\s+/g, " ").trim().slice(0, 500);
}
/** Upload one local file as an off-host object, streamed. */
async function uploadFile(
s3: S3Like,
key: Buffer,
path: string,
): Promise<number> {
return uploadEncryptedBackup(
s3,
offhostObjectKey(`${basename(path)}.enc`),
key,
async (write) => {
for await (const chunk of createReadStream(path)) {
await write(chunk as Buffer);
}
},
);
}
/**
* Take tonight's dump, keep it, copy it off-host, prune the old ones, and
* record what happened. Never throws: a failure is the report's `status`.
*/
export async function runDatabaseBackup(
prisma: PrismaClient,
options: RunDatabaseBackupOptions = {},
): Promise<DatabaseBackupReport> {
const env = options.env ?? process.env;
const now = options.now ?? new Date();
const dir = resolveBackupDir(env);
const localWritable = isWritableDirectory(dir);
const offhostConfig = (() => {
try {
return loadOffhostConfig();
} catch {
// A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the
// same value with their own message; here it reads as an upload error.
return "invalid" as const;
}
})();
// Each leg reads "failed" until it has succeeded, so a run the process died
// under says so once its expiry has passed.
const report: DatabaseBackupReport = {
status: "failed",
fileName: null,
dumpBytes: null,
local: localWritable ? "failed" : "no_directory",
offhost: offhostConfig ? "failed" : "not_configured",
removed: 0,
error: null,
};
const run = await prisma.databaseBackupRun.create({
data: {
startedAt: now,
status: "running",
localResult: report.local,
offhostResult: report.offhost,
},
select: { id: true },
});
let scratch: string | null = null;
try {
if (!databaseBackupEnabled(env)) {
report.status = "skipped";
report.local = "skipped";
report.offhost = "skipped";
report.error = "DB_BACKUP_ENABLED is false";
} else if (!localWritable && !offhostConfig) {
report.status = "skipped";
report.error = `no writable backup directory (${dir}) and no off-host bucket configured`;
} else {
const target = localWritable
? dir
: (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-")));
const dump = await createDatabaseDump({
databaseUrl: env.DATABASE_URL ?? "",
dir: target,
kind: "daily",
now,
env,
appVersion: appVersion(env),
buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null,
deadline: options.deadline,
});
if (localWritable) report.local = "written";
report.fileName = basename(dump.dumpPath);
report.dumpBytes = dump.manifest.dump.bytes;
let uploadError: string | null = null;
if (offhostConfig === "invalid") {
uploadError =
"BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64";
} else if (offhostConfig) {
try {
const s3 = options.s3 ?? (await getS3Client(offhostConfig));
// The manifest goes up first: a dump object in the bucket always
// has the manifest a restore checks it against.
await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath);
await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath);
report.offhost = "uploaded";
} catch (err) {
uploadError = `off-host upload failed: ${reason(err)}`;
}
}
if (localWritable) {
const removed = await applyRetention(dir, {
dailyKeep: resolveDailyKeep(env),
now: now.getTime(),
});
report.removed = removed.length;
}
// Good when the dump exists somewhere it survives: on the volume, or in
// the bucket. An upload that failed beside a kept dump still fails the
// run, because the operator asked for both.
report.status = uploadError ? "failed" : "succeeded";
report.error = uploadError;
}
} catch (err) {
report.status = "failed";
report.error = reason(err);
} finally {
if (scratch) await rm(scratch, { recursive: true, force: true });
}
await prisma.databaseBackupRun.update({
where: { id: run.id },
data: {
finishedAt: new Date(),
status: report.status,
fileName: report.fileName,
dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes),
localResult: report.local,
offhostResult: report.offhost,
error: report.error,
},
});
await prisma.databaseBackupRun.deleteMany({
where: {
startedAt: {
lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000),
},
},
});
annotate({
action: { name: "database.backup.run" },
meta: {
status: report.status,
local: report.local,
offhost: report.offhost,
dump_bytes: report.dumpBytes ?? 0,
removed: report.removed,
},
});
return report;
}
@@ -0,0 +1,61 @@
/**
* Which lifecycle rules retire the database dumps under `database/`, apart
* from the account copies: a rule scoped to another prefix, or to a tag, does
* not touch them.
*/
import { describe, expect, it } from "vitest";
import type { LifecycleRule } from "@aws-sdk/client-s3";
import { lifecycleFromRules, ruleAppliesTo } from "../offhost-lifecycle";
const DUMP = "database/healthlog-20261025T013000Z-daily.dump.enc";
function rule(days: number, extra: Partial<LifecycleRule> = {}): LifecycleRule {
return { Status: "Enabled", Expiration: { Days: days }, ...extra };
}
describe("lifecycle rules and the database dumps", () => {
it("counts a rule for the whole bucket, or for database/, for both", () => {
expect(lifecycleFromRules([rule(30)])).toEqual({
state: "configured",
expirationDays: 30,
database: { state: "configured", expirationDays: 30 },
});
expect(
lifecycleFromRules([
rule(30, { Filter: { Prefix: "2026-" } }),
rule(14, { Filter: { Prefix: "database/" } }),
]).database,
).toEqual({ state: "configured", expirationDays: 14 });
});
it("says database/ is not covered when the only rule is scoped elsewhere", () => {
const result = lifecycleFromRules([
rule(30, { Filter: { Prefix: "2026-" } }),
]);
expect(result.state).toBe("configured");
expect(result.database).toEqual({ state: "missing", expirationDays: null });
});
it("reads the legacy prefix, the And filter, tags and disabled rules", () => {
expect(ruleAppliesTo(rule(5, { Prefix: "database/" }), DUMP)).toBe(true);
expect(ruleAppliesTo(rule(5, { Prefix: "other/" }), DUMP)).toBe(false);
expect(
ruleAppliesTo(
rule(5, {
Filter: { And: { Prefix: "database/", ObjectSizeGreaterThan: 1 } },
}),
DUMP,
),
).toBe(true);
expect(
ruleAppliesTo(
rule(5, { Filter: { Tag: { Key: "k", Value: "v" } } }),
DUMP,
),
).toBe(false);
expect(
lifecycleFromRules([rule(7, { Status: "Disabled" })]).database.state,
).toBe("missing");
});
});
+40
View File
@@ -0,0 +1,40 @@
/**
* The queue facts of the nightly whole-database dump: its name, its schedule
* and its send options. Apart from the job so the worker registry can read
* them without loading it (`job-registry-graph.test.ts`).
*/
import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy";
import { resolveBackupSchedule } from "@/lib/database-backup/dump-core.mjs";
export const DATABASE_BACKUP_QUEUE = "database-backup";
/**
* `DB_BACKUP_SCHEDULE`, read once when the worker loads its registry, in the
* zone every cron runs in (`createAndSchedule`, Europe/Berlin). Default 01:30,
* an hour before the off-host account copies so the two uploads do not meet.
* A value that is not a five-field cron keeps the default, and the boot
* readiness summary says so.
*/
export const DATABASE_BACKUP_CRON = resolveBackupSchedule(process.env).cron;
/**
* Four hours. `pg_dump` of a large instance takes minutes, the upload of the
* dump about as long as the dump; the window is there to be impossible to hit
* for a working run. The handler kills `pg_dump` at three quarters of it
* (`jobDeadline`), so a hung dump ends as a failed run with a reason rather
* than an expired job.
*/
export const DATABASE_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60;
/**
* What the nightly schedule sends with: the expiry, the backups' heartbeat,
* and one retry. A dump that failed for a passing reason (a lock it waited
* out, a bucket that answered 503) gets a second attempt; one that fails
* again waits for the next night, and the failure stays on the admin page.
*/
export const DATABASE_BACKUP_SEND_OPTIONS = {
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS,
retryLimit: 1,
retryDelay: 15 * 60,
} as const;
+6
View File
@@ -78,6 +78,12 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"continued",
"conversation_titles_migrated",
"created",
// The nightly whole-database dump: where it went and how big.
"database_backup_bytes",
"database_backup_local",
"database_backup_offhost",
"database_backup_removed",
"database_backup_status",
"daily_rows_retired",
"days_consolidated",
"days_rebuilt",
+17 -30
View File
@@ -44,6 +44,12 @@ import {
import { annotate, getEvent } from "@/lib/logging/context";
import { envValue } from "@/lib/env";
import { OFFHOST_BACKUP_QUEUE } from "./offhost-backup-queue";
import {
LIFECYCLE_MISSING,
LIFECYCLE_UNKNOWN,
lifecycleFromRules,
type OffhostLifecycle,
} from "./offhost-lifecycle";
const ALGORITHM = "aes-256-gcm";
const IV_LENGTH = 12;
@@ -494,9 +500,9 @@ function createEnvelopeStream(
return { stream, bytes: () => written };
}
/** Produces the backup JSON in pieces. Every piece is written in order. */
/** Produces the plaintext (JSON text, or a dump's bytes) in ordered pieces. */
export type BackupJsonProducer = (
write: (chunk: string) => Promise<void>,
write: (chunk: string | Buffer) => Promise<void>,
) => Promise<unknown>;
export interface UploadBackupOptions {
@@ -572,9 +578,9 @@ export async function uploadEncryptedBackup(
},
);
const write = async (chunk: string): Promise<void> => {
const write = async (chunk: string | Buffer): Promise<void> => {
if (failure) throw failure;
if (gzip.write(chunk, "utf8")) return;
if (gzip.write(chunk, "utf8")) return; // a Buffer ignores the encoding
await new Promise<void>((resolve, reject) => {
const onDrain = (): void => {
gzip.off("error", onError);
@@ -634,18 +640,7 @@ export interface S3Like {
getLifecycle?(): Promise<OffhostLifecycle>;
}
/**
* Whether the bucket retires old copies by itself.
*
* `configured` names the shortest expiry of an enabled rule; `missing` means
* the bucket answered that it has no rule; `unknown` means it would not say
* (a credential without `s3:GetLifecycleConfiguration`, a target that does not
* implement the call, a network failure) and says nothing either way.
*/
export interface OffhostLifecycle {
state: "configured" | "missing" | "unknown";
expirationDays: number | null;
}
export type { OffhostLifecycle } from "./offhost-lifecycle";
export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
// Dynamic import so unit tests + dev environments without the SDK don't fail.
@@ -765,13 +760,7 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
}),
{ abortSignal: AbortSignal.timeout(5_000) },
);
const days = (out.Rules ?? [])
.filter((rule) => rule.Status === "Enabled")
.map((rule) => rule.Expiration?.Days)
.filter((d): d is number => typeof d === "number" && d > 0);
return days.length > 0
? { state: "configured", expirationDays: Math.min(...days) }
: { state: "missing", expirationDays: null };
return lifecycleFromRules(out.Rules ?? []);
} catch (err) {
const name = (err as { name?: string; Code?: string }).name;
const code = (err as { Code?: string }).Code;
@@ -779,9 +768,9 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
name === "NoSuchLifecycleConfiguration" ||
code === "NoSuchLifecycleConfiguration"
) {
return { state: "missing", expirationDays: null };
return LIFECYCLE_MISSING;
}
return { state: "unknown", expirationDays: null };
return LIFECYCLE_UNKNOWN;
}
},
};
@@ -806,15 +795,13 @@ export async function probeOffhostLifecycle(
return lifecycleCache.value;
}
const cfg = loadOffhostConfigSafe();
if (!cfg) return { state: "unknown", expirationDays: null };
if (!cfg) return LIFECYCLE_UNKNOWN;
let value: OffhostLifecycle;
try {
const s3 = s3Override ?? (await getS3Client(cfg));
value = s3.getLifecycle
? await s3.getLifecycle()
: { state: "unknown", expirationDays: null };
value = s3.getLifecycle ? await s3.getLifecycle() : LIFECYCLE_UNKNOWN;
} catch {
value = { state: "unknown", expirationDays: null };
value = LIFECYCLE_UNKNOWN;
}
if (!s3Override) lifecycleCache = { at: now, value };
return value;
+79
View File
@@ -0,0 +1,79 @@
/**
* Whether the operator's bucket retires old copies by itself, read from its
* lifecycle rules (`GetBucketLifecycleConfiguration`).
*
* Two answers, because the bucket holds two kinds of object with different
* keys: the per-account copies (`<date>/user-<id>.json.enc`) and the database
* dumps (`database/…`). A rule scoped to a prefix covers only the keys under
* it, so a bucket whose one rule expires `2026-` keys ages the account copies
* out and keeps every database dump forever, and a database dump keeps every
* deleted account's data for as long as it exists. The account answer keeps
* its earlier reading (the shortest expiry of any enabled rule); the database
* answer counts only the rules that apply to a key under `database/`.
*
* `configured` names the shortest expiry; `missing` means the bucket answered
* that no rule applies; `unknown` means it would not say (a credential
* without `s3:GetLifecycleConfiguration`, a target that does not implement the
* call, a network failure) and says nothing either way.
*/
import type { LifecycleRule } from "@aws-sdk/client-s3";
export interface LifecycleVerdict {
state: "configured" | "missing" | "unknown";
expirationDays: number | null;
}
export interface OffhostLifecycle extends LifecycleVerdict {
/** The same question for the database dumps under `database/`. */
database: LifecycleVerdict;
}
export const LIFECYCLE_UNKNOWN: OffhostLifecycle = {
state: "unknown",
expirationDays: null,
database: { state: "unknown", expirationDays: null },
};
export const LIFECYCLE_MISSING: OffhostLifecycle = {
state: "missing",
expirationDays: null,
database: { state: "missing", expirationDays: null },
};
/** A key a database dump is stored under, to test the rules against. */
const DATABASE_SAMPLE_KEY =
"database/healthlog-20260101T013000Z-daily.dump.enc";
/**
* Whether a rule's filter takes in `key`. Our objects carry no tags, so a
* rule that filters on a tag never applies to them; a size bound is not a
* reason to call a rule inapplicable, so it is read as applying.
*/
export function ruleAppliesTo(rule: LifecycleRule, key: string): boolean {
const filter = rule.Filter;
if (filter?.Tag || filter?.And?.Tags?.length) return false;
const prefix = filter?.Prefix ?? filter?.And?.Prefix ?? rule.Prefix ?? "";
return key.startsWith(prefix);
}
function verdict(rules: readonly LifecycleRule[]): LifecycleVerdict {
const days = rules
.filter((rule) => rule.Status === "Enabled")
.map((rule) => rule.Expiration?.Days)
.filter((d): d is number => typeof d === "number" && d > 0);
return days.length > 0
? { state: "configured", expirationDays: Math.min(...days) }
: { state: "missing", expirationDays: null };
}
/** Both answers from the rules a bucket returned. */
export function lifecycleFromRules(
rules: readonly LifecycleRule[],
): OffhostLifecycle {
return {
...verdict(rules),
database: verdict(
rules.filter((rule) => ruleAppliesTo(rule, DATABASE_SAMPLE_KEY)),
),
};
}
+12
View File
@@ -42,6 +42,7 @@ import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key-
import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill";
import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run";
import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy";
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue";
import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore";
import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue";
@@ -332,6 +333,17 @@ export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
exclusive: "lockedPass",
why: "Serialises every account's whole record.",
},
"database-backup": {
runtime: "long",
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
expiryVia: ["DATABASE_BACKUP_SEND_OPTIONS"],
stop: {
file: "lib/jobs/reminder/backup-handlers.ts",
fn: "handleDatabaseBackup",
},
exclusive: "lockedPass",
why: "Runs pg_dump over the whole database and uploads the dump off-host.",
},
"data-backup-offhost": {
runtime: "long",
expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS,
+17 -1
View File
@@ -1,5 +1,6 @@
/**
* The housekeeping queues: backups (weekly in-database, nightly off-host),
* The housekeeping queues: backups (the nightly whole-database dump, weekly
* in-database account copies, nightly off-host),
* the restore drill and restores, every ledger and retention cleanup, the
* medication maintenance passes, the imports (Apple Health, Health Connect,
* medication intake), the converging encryption backfills, the document vault
@@ -47,6 +48,11 @@ import {
DATA_BACKUP_QUEUE,
DATA_BACKUP_SEND_OPTIONS,
} from "@/lib/jobs/data-backup-policy";
import {
DATABASE_BACKUP_CRON,
DATABASE_BACKUP_QUEUE,
DATABASE_BACKUP_SEND_OPTIONS,
} from "@/lib/jobs/database-backup-queue";
import {
DAWARICH_SYNC_CRON,
DAWARICH_SYNC_QUEUE,
@@ -300,6 +306,16 @@ const jobs: readonly JobDefinition[] = [
lane: true,
load: async () => (await backups()).handleDataBackup,
}),
// The whole-database dump: the copy that rebuilds a lost server. Locked, so
// a retry that starts while a run is still dumping waits its turn.
defineJob({
queue: DATABASE_BACKUP_QUEUE,
cron: DATABASE_BACKUP_CRON,
scheduleOptions: DATABASE_BACKUP_SEND_OPTIONS,
work: { localConcurrency: 1 },
lock: () => WHOLE_PASS,
load: async () => (await backups()).handleDatabaseBackup,
}),
prune(
RATE_LIMIT_CLEANUP_QUEUE,
RATE_LIMIT_CLEANUP_CRON,
+32 -1
View File
@@ -5,7 +5,7 @@
* schedules, and boss.work registrations.
*/
import { type Job, type JobWithMetadata } from "pg-boss";
import { jobBudget } from "@/lib/jobs/job-budget";
import { jobBudget, jobDeadline } from "@/lib/jobs/job-budget";
import { recordError } from "@/lib/jobs/worker-status";
import {
BackupBlobTooLargeError,
@@ -31,6 +31,37 @@ import {
} from "@/lib/jobs/backup-pass-attempts";
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
/**
* The nightly whole-database dump (`database-backup`).
*
* One `pg_dump` in a child process, so it is not in the heavy lane: the heap
* holds a part of the upload at a time, not the database. `pg_dump` is stopped
* at three quarters of the job's expiry (`jobDeadline`), so a dump that hangs
* on a lock ends as a failed run with its reason rather than an expired job.
* A skipped run (switched off, nowhere to put it) completes; the admin page
* and the boot summary carry why.
*/
export async function handleDatabaseBackup(
jobs: Job<Record<string, never>>[],
): Promise<JobOutcome> {
return withBackgroundEvent("job.database_backup", async () => {
const { runDatabaseBackup } = await import("@/lib/database-backup/run");
const report = await runDatabaseBackup(getWorkerPrisma(), {
deadline: jobs[0] ? jobDeadline(jobs[0]) : undefined,
});
const did = {
database_backup_status: report.status,
database_backup_local: report.local,
database_backup_offhost: report.offhost,
database_backup_bytes: report.dumpBytes ?? 0,
database_backup_removed: report.removed,
};
return report.status === "failed"
? jobFailed("database dump failed", report.error ?? undefined, did)
: jobDone(did);
});
}
export interface DataBackupPayload {
triggeredAt: string;
}
+1
View File
@@ -99,6 +99,7 @@ const subsystemSurface = {
"audit-log-cleanup": { audience: "system" },
"step-up-elevation-cleanup": { audience: "system" },
"data-backup-offhost": { audience: "system" },
"database-backup": { audience: "system" },
"data-restore-drill": { audience: "system" },
"offhost-backup-purge": { audience: "system" },
"backup-restore": { audience: "system" },
+9 -4
View File
@@ -9,6 +9,8 @@
* both sides a route-handler-independent shared type home; the route
* keeps owning the HTTP contract, the component keeps owning the UI.
*/
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
import type { OffhostLifecycle } from "@/lib/jobs/offhost-lifecycle";
import type { BackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness";
@@ -76,10 +78,7 @@ export interface OffhostBackupOverview {
* (`GetBucketLifecycleConfiguration`), cached for ten minutes; `unknown`
* when the credential may not read it.
*/
lifecycle: {
state: "configured" | "missing" | "unknown";
expirationDays: number | null;
};
lifecycle: OffhostLifecycle;
}
export interface BackupsList {
@@ -101,6 +100,12 @@ export interface BackupsList {
* the operator's bucket, and those two fail independently.
*/
offhost: OffhostBackupOverview;
/**
* The nightly whole-database dump, the copy that rebuilds a lost server:
* the newest good dump (kept here, off-host, or both), the newest failed
* run, and whether this server can keep dumps at all.
*/
database: DatabaseBackupHealth;
/**
* Soft retention hint — the worker is configured for weekly backups
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and