mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-13 22:32:04 +00:00
Merge remote-tracking branch 'origin/release/v1.46.0' into fix/v146-security
This commit is contained in:
commit
e79ea08b13
55 files changed
+5523
-91
No files matched your search
@@ -58,7 +58,9 @@ describe("the Postgres 16 -> 18 move in docker-compose.yml", () => {
|
||||
it("keeps the new data where the Postgres 18 image expects it", () => {
|
||||
// Mounted at .../data, as 16 had it, the 18 image refuses to start.
|
||||
expect(db.volumes).toEqual(["postgres-data:/var/lib/postgresql"]);
|
||||
// `backups` holds the app's database dumps, not database files.
|
||||
expect(Object.keys(compose.volumes).sort()).toEqual([
|
||||
"backups",
|
||||
"pgdata",
|
||||
"postgres-data",
|
||||
]);
|
||||
|
||||
@@ -54,9 +54,29 @@ vi.mock("@/lib/jobs/offhost-backup", () => ({
|
||||
probeOffhostLifecycle: async () => ({
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
database: { state: "missing", expirationDays: null },
|
||||
}),
|
||||
}));
|
||||
|
||||
const databaseHealth = {
|
||||
enabled: true,
|
||||
directory: "/var/lib/healthlog/backups",
|
||||
keepNightly: 7,
|
||||
localDirWritable: true,
|
||||
offhostConfigured: true,
|
||||
lastGood: {
|
||||
at: "2026-09-21T23:31:40.000Z",
|
||||
fileName: "healthlog-20260921T233000Z-daily.dump",
|
||||
sizeBytes: 52_428_800,
|
||||
local: true,
|
||||
offhost: true,
|
||||
},
|
||||
lastFailure: null,
|
||||
};
|
||||
vi.mock("@/lib/database-backup/health", () => ({
|
||||
getDatabaseBackupHealth: async () => databaseHealth,
|
||||
}));
|
||||
|
||||
import { GET } from "../route";
|
||||
import type { BackupsList } from "@/types/backups";
|
||||
|
||||
@@ -166,7 +186,11 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
// No bucket to ask on a host that is not configured.
|
||||
lifecycle: { state: "unknown", expirationDays: null },
|
||||
lifecycle: {
|
||||
state: "unknown",
|
||||
expirationDays: null,
|
||||
database: { state: "unknown", expirationDays: null },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -229,6 +253,7 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
expect(body.offhost.lifecycle).toEqual({
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
database: { state: "missing", expirationDays: null },
|
||||
});
|
||||
expect(body.offhost.pendingDeletions).toEqual({
|
||||
count: 2,
|
||||
@@ -285,3 +310,10 @@ describe("GET /api/admin/backups — a run in progress and one a process died un
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("GET /api/admin/backups — the database dump", () => {
|
||||
it("carries the dump's health beside the account copies", async () => {
|
||||
const body = await read();
|
||||
expect(body.database).toEqual(databaseHealth);
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,10 @@
|
||||
* stopped six weeks ago — every row it lists has a perfectly ordinary
|
||||
* timestamp either way.
|
||||
*
|
||||
* The database row comes first in what an operator needs and last in this
|
||||
* file: the nightly whole-database dump (`getDatabaseBackupHealth`), the
|
||||
* one copy that rebuilds a lost server, read from its own ledger.
|
||||
*
|
||||
* And alongside both, the off-host leg: one row per account saying when this
|
||||
* host last put that account's encrypted copy in the operator's bucket and how
|
||||
* stale that is against the nightly schedule. It comes from the ledger the
|
||||
@@ -31,11 +35,13 @@ import {
|
||||
} from "@/lib/jobs/job-failures";
|
||||
import { readBackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
|
||||
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
|
||||
import { getDatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import {
|
||||
offhostBackupConfigured,
|
||||
probeOffhostLifecycle,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import { OFFHOST_BACKUP_QUEUE } from "@/lib/jobs/offhost-backup-queue";
|
||||
import { LIFECYCLE_UNKNOWN } from "@/lib/jobs/offhost-lifecycle";
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
@@ -190,16 +196,15 @@ export const GET = apiHandler(async () => {
|
||||
readBackupPassActivity(prisma, DATA_BACKUP_QUEUE, weeklyRunningSince),
|
||||
readBackupPassActivity(prisma, OFFHOST_BACKUP_QUEUE, offhostRunningSince),
|
||||
]);
|
||||
const [purgeCount, oldestPurge, lifecycle] = await Promise.all([
|
||||
const [purgeCount, oldestPurge, lifecycle, database] = await Promise.all([
|
||||
prisma.offhostPurgeRequest.count({ where: stillInBucket }),
|
||||
prisma.offhostPurgeRequest.findFirst({
|
||||
where: stillInBucket,
|
||||
orderBy: { requestedAt: "asc" },
|
||||
select: { requestedAt: true, lastFailure: true },
|
||||
}),
|
||||
configured
|
||||
? probeOffhostLifecycle()
|
||||
: Promise.resolve({ state: "unknown" as const, expirationDays: null }),
|
||||
configured ? probeOffhostLifecycle() : Promise.resolve(LIFECYCLE_UNKNOWN),
|
||||
getDatabaseBackupHealth(now),
|
||||
]);
|
||||
|
||||
const payload: BackupsList = {
|
||||
@@ -229,6 +234,7 @@ export const GET = apiHandler(async () => {
|
||||
},
|
||||
lifecycle,
|
||||
},
|
||||
database,
|
||||
};
|
||||
|
||||
return apiSuccess(payload);
|
||||
|
||||
@@ -26,7 +26,11 @@ import { OffhostBackupsSection } from "../offhost-backups-section";
|
||||
const QUIET = {
|
||||
activity: { runningSince: null, interrupted: [] },
|
||||
pendingDeletions: { count: 0, oldestRequestedAt: null, lastFailure: null },
|
||||
lifecycle: { state: "configured" as const, expirationDays: 30 },
|
||||
lifecycle: {
|
||||
state: "configured" as const,
|
||||
expirationDays: 30,
|
||||
database: { state: "configured" as const, expirationDays: 30 },
|
||||
},
|
||||
};
|
||||
|
||||
function render(data: BackupsList["offhost"] | null): string {
|
||||
@@ -133,7 +137,11 @@ describe("<OffhostBackupsSection>", () => {
|
||||
oldestRequestedAt: "2026-09-20T10:00:00.000Z",
|
||||
lastFailure: "AccessDenied",
|
||||
},
|
||||
lifecycle: { state: "missing", expirationDays: null },
|
||||
lifecycle: {
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
database: { state: "missing", expirationDays: null },
|
||||
},
|
||||
});
|
||||
expect(html).toContain('data-lifecycle-state="missing"');
|
||||
expect(html).toContain("no lifecycle rule");
|
||||
|
||||
@@ -28,6 +28,7 @@ import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
|
||||
import { RestoreJobStatus, visibleRestoreJobs } from "./backup-restore-status";
|
||||
import { BackupRowsCards, BackupRowsTable } from "./backups/backup-rows";
|
||||
import { DatabaseDumpRow } from "./backups/database-dump-row";
|
||||
import { BackupUploadCard } from "./backups/backup-upload-card";
|
||||
import {
|
||||
RestoreRefusalNotice,
|
||||
@@ -110,6 +111,14 @@ export function BackupsSection() {
|
||||
}
|
||||
/>
|
||||
|
||||
{data?.database ? (
|
||||
<DatabaseDumpRow
|
||||
health={data.database}
|
||||
bucketLifecycle={
|
||||
data.offhost.configured ? data.offhost.lifecycle.database : null
|
||||
}
|
||||
/>
|
||||
) : null}
|
||||
{data?.schedule ? (
|
||||
<ScheduleHealthNotice schedule={data.schedule} />
|
||||
) : null}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* The database row on the backups card says where the newest good dump is,
|
||||
* how long a deleted account stays in the dumps, and each thing that keeps a
|
||||
* dump from being taken or from ever ageing out of the bucket.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
|
||||
|
||||
import { DatabaseDumpRow } from "../database-dump-row";
|
||||
|
||||
const HEALTHY: DatabaseBackupHealth = {
|
||||
enabled: true,
|
||||
directory: "/var/lib/healthlog/backups",
|
||||
keepNightly: 7,
|
||||
localDirWritable: true,
|
||||
offhostConfigured: true,
|
||||
lastGood: {
|
||||
at: "2026-10-24T23:31:00.000Z",
|
||||
fileName: "healthlog-20261024T233000Z-daily.dump",
|
||||
sizeBytes: 52_428_800,
|
||||
local: true,
|
||||
offhost: true,
|
||||
},
|
||||
lastFailure: null,
|
||||
};
|
||||
|
||||
function render(
|
||||
health: DatabaseBackupHealth,
|
||||
bucketLifecycle: LifecycleVerdict | null = {
|
||||
state: "configured",
|
||||
expirationDays: 30,
|
||||
},
|
||||
): string {
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<DatabaseDumpRow health={health} bucketLifecycle={bucketLifecycle} />
|
||||
</I18nProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
describe("<DatabaseDumpRow>", () => {
|
||||
it("names the newest good dump, where it is, and how long dumps keep deleted accounts", () => {
|
||||
const html = render(HEALTHY);
|
||||
expect(html).toContain("kept on this server and off-host");
|
||||
expect(html).toContain("the newest 7 are kept on this server");
|
||||
expect(html).toContain("removed after 30 days");
|
||||
expect(html).not.toContain('data-slot="database-dump-failed"');
|
||||
expect(html).not.toContain('data-slot="database-dump-setup"');
|
||||
expect(html).not.toContain('data-slot="database-dump-no-lifecycle"');
|
||||
});
|
||||
|
||||
it("warns when no bucket rule ever removes the dumps", () => {
|
||||
const html = render(HEALTHY, { state: "missing", expirationDays: null });
|
||||
expect(html).toContain('data-slot="database-dump-no-lifecycle"');
|
||||
expect(html).toContain("never removed");
|
||||
});
|
||||
|
||||
it("shows a failure newer than the last good dump, and not an older one", () => {
|
||||
const failed = render({
|
||||
...HEALTHY,
|
||||
lastFailure: {
|
||||
at: "2026-10-25T23:31:00.000Z",
|
||||
error: "pg_dump exited 1: No space left on device",
|
||||
},
|
||||
});
|
||||
expect(failed).toContain('data-slot="database-dump-failed"');
|
||||
expect(failed).toContain("No space left on device");
|
||||
const recovered = render({
|
||||
...HEALTHY,
|
||||
lastFailure: { at: "2026-10-20T23:31:00.000Z", error: "old" },
|
||||
});
|
||||
expect(recovered).not.toContain('data-slot="database-dump-failed"');
|
||||
});
|
||||
|
||||
it("says when dumps go off-host only, or cannot be taken at all", () => {
|
||||
expect(render({ ...HEALTHY, localDirWritable: false }, null)).toContain(
|
||||
"dumps go off-host only",
|
||||
);
|
||||
expect(
|
||||
render(
|
||||
{ ...HEALTHY, localDirWritable: false, offhostConfigured: false },
|
||||
null,
|
||||
),
|
||||
).toContain("no database dump can be taken");
|
||||
expect(render({ ...HEALTHY, lastGood: null }, null)).toContain(
|
||||
"No database dump yet",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,148 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* The nightly whole-database dump, as one row on the backups card.
|
||||
*
|
||||
* The account copies below it restore one account on this server; the dump is
|
||||
* the copy that rebuilds the server. So the row leads the card and says three
|
||||
* things: when the newest good dump was taken and where it is, whether the
|
||||
* newest run failed, and whether this server can keep a dump at all.
|
||||
*/
|
||||
import { AlertTriangle, HardDrive } from "lucide-react";
|
||||
|
||||
import { SettingsInfoTile } from "@/components/settings/_info-tile";
|
||||
import { formatBytes } from "@/lib/format/bytes";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
|
||||
|
||||
function whereKept(
|
||||
good: NonNullable<DatabaseBackupHealth["lastGood"]>,
|
||||
t: ReturnType<typeof useTranslations>["t"],
|
||||
) {
|
||||
if (good.local && good.offhost) {
|
||||
return t("admin.section.backups.database.whereBoth");
|
||||
}
|
||||
if (good.offhost) return t("admin.section.backups.database.whereOffhost");
|
||||
return t("admin.section.backups.database.whereLocal");
|
||||
}
|
||||
|
||||
export function DatabaseDumpRow({
|
||||
health,
|
||||
bucketLifecycle,
|
||||
}: {
|
||||
health: DatabaseBackupHealth;
|
||||
/** The bucket's rule for `database/`; null when nothing goes off-host. */
|
||||
bucketLifecycle: LifecycleVerdict | null;
|
||||
}) {
|
||||
const { t, locale } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
const good = health.lastGood;
|
||||
const failure = health.lastFailure;
|
||||
const failedSinceGood =
|
||||
failure !== null && (good === null || failure.at > good.at);
|
||||
|
||||
let setupProblem: string | null = null;
|
||||
if (!health.enabled) {
|
||||
setupProblem = t("admin.section.backups.database.disabled");
|
||||
} else if (!health.localDirWritable) {
|
||||
setupProblem = health.offhostConfigured
|
||||
? t("admin.section.backups.database.noDirectoryOffhost", {
|
||||
dir: health.directory,
|
||||
})
|
||||
: t("admin.section.backups.database.noDirectory", {
|
||||
dir: health.directory,
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-2" data-slot="database-dump">
|
||||
<div className="flex items-start gap-2">
|
||||
<HardDrive
|
||||
className="text-muted-foreground mt-0.5 h-4 w-4 shrink-0"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<div>
|
||||
<p className="text-sm font-medium">
|
||||
{t("admin.section.backups.database.title")}
|
||||
</p>
|
||||
<p
|
||||
className="text-muted-foreground text-xs"
|
||||
data-slot="database-dump-last-good"
|
||||
>
|
||||
{good
|
||||
? t("admin.section.backups.database.lastGood", {
|
||||
when: fmt.dateTime(good.at),
|
||||
size: formatBytes(good.sizeBytes ?? 0, locale),
|
||||
where: whereKept(good, t),
|
||||
})
|
||||
: t("admin.section.backups.database.none")}
|
||||
</p>
|
||||
{/* How long a deleted account stays in the dumps: the operator's
|
||||
side of the deletion promise (GDPR retention of backups). */}
|
||||
<p
|
||||
className="text-muted-foreground text-xs"
|
||||
data-slot="database-dump-retention"
|
||||
>
|
||||
{t("admin.section.backups.database.retentionLocal", {
|
||||
nights: health.keepNightly,
|
||||
})}
|
||||
{bucketLifecycle?.state === "configured"
|
||||
? ` ${t("admin.section.backups.database.retentionOffhost", {
|
||||
days: bucketLifecycle.expirationDays ?? 0,
|
||||
})}`
|
||||
: null}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{failedSinceGood && failure ? (
|
||||
<div
|
||||
role="alert"
|
||||
data-slot="database-dump-failed"
|
||||
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
|
||||
>
|
||||
<div className="flex items-start gap-2">
|
||||
<AlertTriangle
|
||||
className="mt-0.5 h-4 w-4 shrink-0"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<div>
|
||||
<p className="font-medium">
|
||||
{t("admin.section.backups.database.failedTitle")}
|
||||
</p>
|
||||
<p className="text-xs">
|
||||
{t("admin.section.backups.database.failedDescription", {
|
||||
when: fmt.dateTime(failure.at),
|
||||
reason: failure.error,
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{bucketLifecycle?.state === "missing" ? (
|
||||
<SettingsInfoTile
|
||||
icon={AlertTriangle}
|
||||
tone="warning"
|
||||
role="alert"
|
||||
data-slot="database-dump-no-lifecycle"
|
||||
>
|
||||
{t("admin.section.backups.database.retentionOffhostMissing")}
|
||||
</SettingsInfoTile>
|
||||
) : null}
|
||||
|
||||
{setupProblem ? (
|
||||
<SettingsInfoTile
|
||||
icon={AlertTriangle}
|
||||
tone="warning"
|
||||
role="alert"
|
||||
data-slot="database-dump-setup"
|
||||
>
|
||||
{setupProblem}
|
||||
</SettingsInfoTile>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -149,3 +149,56 @@ describe("collectReadiness — retired AI grant", () => {
|
||||
expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("collectReadiness — the database dump", () => {
|
||||
const offhost = {
|
||||
BACKUP_S3_ENDPOINT: "https://s3.example.test",
|
||||
BACKUP_S3_BUCKET: "b",
|
||||
BACKUP_S3_ACCESS_KEY: "a",
|
||||
BACKUP_S3_SECRET_KEY: "s",
|
||||
BACKUP_ENCRYPTION_KEY: HEX32,
|
||||
};
|
||||
const dumpLine = (
|
||||
env: Record<string, string | undefined>,
|
||||
writable: boolean,
|
||||
) =>
|
||||
collectReadiness(env, () => writable).lines.find(
|
||||
(l) => l.label === "Database dump",
|
||||
);
|
||||
|
||||
it("names the schedule, the directory and the copies kept", () => {
|
||||
const line = dumpLine(baseEnv(), true);
|
||||
expect(line?.status).toBe("ok");
|
||||
expect(line?.detail).toBe(
|
||||
"nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7",
|
||||
);
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain(
|
||||
"copied off-host",
|
||||
);
|
||||
});
|
||||
|
||||
it("warns when nothing can be kept on the server", () => {
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({
|
||||
status: "warn",
|
||||
detail: expect.stringContaining("goes off-host only"),
|
||||
});
|
||||
expect(dumpLine(baseEnv(), false)).toMatchObject({
|
||||
status: "warn",
|
||||
detail: expect.stringContaining("no database dump can be taken"),
|
||||
});
|
||||
});
|
||||
|
||||
it("warns when it is switched off, and names a schedule it ignored", () => {
|
||||
expect(
|
||||
dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true),
|
||||
).toMatchObject({ status: "warn" });
|
||||
const report = collectReadiness(
|
||||
{ ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" },
|
||||
() => true,
|
||||
);
|
||||
expect(
|
||||
report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail,
|
||||
).toBe("not a five-field cron; using 30 1 * * *");
|
||||
expect(report.hasBlocker).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -19,6 +19,13 @@ import {
|
||||
LEGACY_ANY_HOST_WARNING,
|
||||
legacyAnyHostConfigured,
|
||||
} from "@/lib/ai/local-host-allowlist";
|
||||
import {
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
resolveBackupDir,
|
||||
resolveBackupSchedule,
|
||||
resolveDailyKeep,
|
||||
} from "@/lib/database-backup/dump-core.mjs";
|
||||
|
||||
export interface ReadinessLine {
|
||||
/** Short label, e.g. "ENCRYPTION_KEY". */
|
||||
@@ -60,8 +67,73 @@ function anySet(env: Env, keys: string[]): boolean {
|
||||
return keys.some((k) => (env[k] ?? "").trim().length > 0);
|
||||
}
|
||||
|
||||
/** Inspect the environment and build the readiness report. Pure + side-effect free. */
|
||||
export function collectReadiness(env: Env = process.env): ReadinessReport {
|
||||
/**
|
||||
* The nightly database dump's line. The dump is what rebuilds a lost server,
|
||||
* so a host where it cannot be kept anywhere is said out loud at every boot,
|
||||
* and so is one where it only goes off-host.
|
||||
*/
|
||||
export function databaseBackupReadiness(
|
||||
env: Env,
|
||||
writable: (dir: string) => boolean,
|
||||
): ReadinessLine[] {
|
||||
const label = "Database dump";
|
||||
if (!databaseBackupEnabled(env)) {
|
||||
return [
|
||||
{
|
||||
label,
|
||||
status: "warn",
|
||||
detail:
|
||||
"DB_BACKUP_ENABLED=false: no nightly database dump is taken; back the database up another way",
|
||||
},
|
||||
];
|
||||
}
|
||||
const lines: ReadinessLine[] = [];
|
||||
const schedule = resolveBackupSchedule(env);
|
||||
if (!schedule.valid) {
|
||||
lines.push({
|
||||
label: "DB_BACKUP_SCHEDULE",
|
||||
status: "warn",
|
||||
detail: `not a five-field cron; using ${schedule.cron}`,
|
||||
});
|
||||
}
|
||||
const dir = resolveBackupDir(env);
|
||||
const offhost = allSet(env, [
|
||||
"BACKUP_S3_ENDPOINT",
|
||||
"BACKUP_S3_BUCKET",
|
||||
"BACKUP_S3_ACCESS_KEY",
|
||||
"BACKUP_S3_SECRET_KEY",
|
||||
"BACKUP_ENCRYPTION_KEY",
|
||||
]);
|
||||
if (writable(dir)) {
|
||||
lines.push({
|
||||
label,
|
||||
status: "ok",
|
||||
detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`,
|
||||
});
|
||||
} else if (offhost) {
|
||||
lines.push({
|
||||
label,
|
||||
status: "warn",
|
||||
detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
|
||||
});
|
||||
} else {
|
||||
lines.push({
|
||||
label,
|
||||
status: "warn",
|
||||
detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
|
||||
});
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect the environment and build the readiness report. Pure apart from
|
||||
* `writable`, the probe of the dump directory, which tests replace.
|
||||
*/
|
||||
export function collectReadiness(
|
||||
env: Env = process.env,
|
||||
writable: (dir: string) => boolean = isWritableDirectory,
|
||||
): ReadinessReport {
|
||||
const lines: ReadinessLine[] = [];
|
||||
|
||||
// --- Core secrets (boot blockers) ---------------------------------------
|
||||
@@ -139,6 +211,8 @@ export function collectReadiness(env: Env = process.env): ReadinessReport {
|
||||
});
|
||||
}
|
||||
|
||||
lines.push(...databaseBackupReadiness(env, writable));
|
||||
|
||||
// --- Optional subsystems (informational) --------------------------------
|
||||
const optional: Array<[string, boolean]> = [
|
||||
["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])],
|
||||
|
||||
@@ -353,6 +353,8 @@ export const INSTANCE_SCOPED: Readonly<Record<string, string>> = {
|
||||
HostMetric: "instance-wide host telemetry, not attributable to an account",
|
||||
OffhostBackupKeyUse:
|
||||
"per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account",
|
||||
DatabaseBackupRun:
|
||||
"the instance's ledger of nightly whole-database dumps; describes the server, not any account",
|
||||
OffhostPurgeRequest:
|
||||
"the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account",
|
||||
CycleSymptomCategory:
|
||||
|
||||
@@ -0,0 +1,689 @@
|
||||
/**
|
||||
* The rules of the whole-database dump that need no database: names,
|
||||
* retention, the manifest, key fingerprints, row-count comparison, the
|
||||
* connection the client tools get, and the off-host envelope read back by the
|
||||
* restore's own reader.
|
||||
*
|
||||
* The fingerprints and the envelope are checked against the app's own code
|
||||
* (`src/lib/crypto.ts`, `src/lib/jobs/offhost-backup.ts`), not against a copy
|
||||
* of the rule: the restore script cannot import TypeScript, so these are the
|
||||
* tests that keep its plain-JS reading of both from drifting.
|
||||
*/
|
||||
import {
|
||||
mkdtempSync,
|
||||
createReadStream,
|
||||
rmSync,
|
||||
utimesSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
bucketConfigFromEnv,
|
||||
pickBucketDump,
|
||||
buildManifest,
|
||||
compareKeyFingerprints,
|
||||
compareRowCounts,
|
||||
connectionFromUrl,
|
||||
decodeKeyMaterial,
|
||||
decryptOffhostFile,
|
||||
dumpBaseName,
|
||||
formatDumpTimestamp,
|
||||
keyFingerprints,
|
||||
listDumpSets,
|
||||
offhostObjectKey,
|
||||
parseDumpFileName,
|
||||
parseManifest,
|
||||
resolveBackupSchedule,
|
||||
resolveDailyKeep,
|
||||
selectForDeletion,
|
||||
type DumpSet,
|
||||
} from "../dump-core.mjs";
|
||||
import type { S3Like } from "@/lib/jobs/offhost-backup";
|
||||
|
||||
/** Every byte of a stream, in order. */
|
||||
async function collect(stream: AsyncIterable<unknown>): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const c of stream) chunks.push(Buffer.from(c as Uint8Array));
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
const HEX_A = "a".repeat(64);
|
||||
const HEX_B = "b".repeat(64);
|
||||
const HEX_C = "c".repeat(64);
|
||||
const HEX_D = "d".repeat(64);
|
||||
|
||||
const dirs: string[] = [];
|
||||
function tempDir(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hl-dump-core-"));
|
||||
dirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0))
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe("dump names", () => {
|
||||
it("names a dump by its UTC instant and kind", () => {
|
||||
const at = new Date("2026-10-25T01:30:07.123Z");
|
||||
expect(formatDumpTimestamp(at)).toBe("20261025T013007Z");
|
||||
expect(dumpBaseName("daily", at)).toBe("healthlog-20261025T013007Z-daily");
|
||||
expect(dumpBaseName("pre-migration", at)).toBe(
|
||||
"healthlog-20261025T013007Z-pre-migration",
|
||||
);
|
||||
});
|
||||
|
||||
it("parses every file it writes, and back to the same instant", () => {
|
||||
const base = "healthlog-20261025T013007Z-daily";
|
||||
const cases: Array<[string, string, string]> = [
|
||||
[`${base}.dump`, "dump", "final"],
|
||||
[`${base}.manifest.json`, "manifest", "final"],
|
||||
[`${base}.dump.partial`, "dump", "partial"],
|
||||
[`${base}.manifest.json.partial`, "manifest", "partial"],
|
||||
[`${base}.dump.enc`, "dump", "encrypted"],
|
||||
[`${base}.manifest.json.enc`, "manifest", "encrypted"],
|
||||
];
|
||||
for (const [name, role, variant] of cases) {
|
||||
const parsed = parseDumpFileName(name);
|
||||
expect(parsed, name).not.toBeNull();
|
||||
expect(parsed?.base).toBe(base);
|
||||
expect(parsed?.kind).toBe("daily");
|
||||
expect(parsed?.role).toBe(role);
|
||||
expect(parsed?.variant).toBe(variant);
|
||||
expect(parsed?.timestamp.toISOString()).toBe("2026-10-25T01:30:07.000Z");
|
||||
}
|
||||
expect(
|
||||
parseDumpFileName("healthlog-20261025T013007Z-pre-migration.dump")?.kind,
|
||||
).toBe("pre-migration");
|
||||
});
|
||||
|
||||
it("refuses every name it did not write", () => {
|
||||
for (const name of [
|
||||
"healthlog.dump",
|
||||
"backup-20261025T013007Z-daily.dump",
|
||||
"healthlog-20261025T013007Z-weekly.dump",
|
||||
"healthlog-20261025-daily.dump",
|
||||
"healthlog-20261025T013007Z-daily.sql",
|
||||
"healthlog-20261025T013007Z-daily.dump.bak",
|
||||
"healthlog-20261399T013007Z-daily.dump",
|
||||
"notes.txt",
|
||||
]) {
|
||||
expect(parseDumpFileName(name), name).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("configuration", () => {
|
||||
it("keeps at least one daily dump and ignores what is not a number", () => {
|
||||
expect(resolveDailyKeep({})).toBe(7);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "14" })).toBe(14);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "0" })).toBe(1);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "-3" })).toBe(7);
|
||||
expect(resolveDailyKeep({ DB_BACKUP_KEEP: "seven" })).toBe(7);
|
||||
});
|
||||
|
||||
it("takes a five-field cron and says when it ignored one", () => {
|
||||
expect(resolveBackupSchedule({})).toEqual({
|
||||
cron: "30 1 * * *",
|
||||
valid: true,
|
||||
configured: false,
|
||||
});
|
||||
expect(
|
||||
resolveBackupSchedule({ DB_BACKUP_SCHEDULE: " 0 2 * * 1-5 " }),
|
||||
).toEqual({
|
||||
cron: "0 2 * * 1-5",
|
||||
valid: true,
|
||||
configured: true,
|
||||
});
|
||||
for (const bad of ["daily", "0 2 * *", "0 2 * * * *", "0 2 * * mon; rm"]) {
|
||||
expect(resolveBackupSchedule({ DB_BACKUP_SCHEDULE: bad })).toEqual({
|
||||
cron: "30 1 * * *",
|
||||
valid: false,
|
||||
configured: true,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/** A dump set as `listDumpSets` reports it, for the pure retention rule. */
|
||||
function set(
|
||||
kind: "daily" | "pre-migration",
|
||||
iso: string,
|
||||
parts: { dump?: boolean; manifest?: boolean; partialAgeMs?: number } = {},
|
||||
now = Date.parse("2026-10-25T12:00:00Z"),
|
||||
): DumpSet {
|
||||
const base = dumpBaseName(kind, new Date(iso));
|
||||
return {
|
||||
base,
|
||||
kind,
|
||||
timestamp: new Date(iso),
|
||||
dump: parts.dump === false ? null : { path: `/b/${base}.dump`, bytes: 1 },
|
||||
manifest: parts.manifest === false ? null : `/b/${base}.manifest.json`,
|
||||
encryptedDump: null,
|
||||
encryptedManifest: null,
|
||||
leftovers:
|
||||
parts.partialAgeMs === undefined
|
||||
? []
|
||||
: [
|
||||
{
|
||||
path: `/b/${base}.dump.partial`,
|
||||
mtimeMs: now - parts.partialAgeMs,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("retention", () => {
|
||||
const now = Date.parse("2026-10-25T12:00:00Z");
|
||||
|
||||
it("keeps the newest N daily dumps and removes the rest, both files each", () => {
|
||||
const sets = [1, 2, 3, 4, 5].map((d) =>
|
||||
set("daily", `2026-10-2${d}T01:30:00Z`),
|
||||
);
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 3, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed.sort()).toEqual(
|
||||
[
|
||||
"/b/healthlog-20261021T013000Z-daily.dump",
|
||||
"/b/healthlog-20261021T013000Z-daily.manifest.json",
|
||||
"/b/healthlog-20261022T013000Z-daily.dump",
|
||||
"/b/healthlog-20261022T013000Z-daily.manifest.json",
|
||||
].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("never removes the newest good dump, whatever keep says", () => {
|
||||
const sets = [
|
||||
set("daily", "2026-10-24T01:30:00Z"),
|
||||
set("daily", "2026-10-23T01:30:00Z"),
|
||||
];
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 0, "pre-migration": 0 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).not.toContain("/b/healthlog-20261024T013000Z-daily.dump");
|
||||
expect(doomed).toContain("/b/healthlog-20261023T013000Z-daily.dump");
|
||||
});
|
||||
|
||||
it("counts only good dumps: an unfinished newer one does not push out the newest good one", () => {
|
||||
const sets = [
|
||||
set("daily", "2026-10-25T01:30:00Z", {
|
||||
dump: false,
|
||||
partialAgeMs: 60_000,
|
||||
}),
|
||||
set("daily", "2026-10-24T01:30:00Z"),
|
||||
];
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 1, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps two pre-migration dumps apart from the daily count", () => {
|
||||
const sets = [
|
||||
set("pre-migration", "2026-10-20T08:00:00Z"),
|
||||
set("pre-migration", "2026-10-10T08:00:00Z"),
|
||||
set("pre-migration", "2026-10-01T08:00:00Z"),
|
||||
set("daily", "2026-10-24T01:30:00Z"),
|
||||
];
|
||||
const doomed = selectForDeletion(sets, {
|
||||
keep: { daily: 1, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).toEqual([
|
||||
"/b/healthlog-20261001T080000Z-pre-migration.dump",
|
||||
"/b/healthlog-20261001T080000Z-pre-migration.manifest.json",
|
||||
]);
|
||||
});
|
||||
|
||||
it("removes a .partial only once it is older than twelve hours", () => {
|
||||
const fresh = set("daily", "2026-10-25T01:30:00Z", {
|
||||
dump: false,
|
||||
manifest: false,
|
||||
partialAgeMs: 60 * 60_000,
|
||||
});
|
||||
const stale = set("daily", "2026-10-23T01:30:00Z", {
|
||||
dump: false,
|
||||
manifest: false,
|
||||
partialAgeMs: 13 * 60 * 60_000,
|
||||
});
|
||||
const doomed = selectForDeletion([fresh, stale], {
|
||||
keep: { daily: 7, "pre-migration": 2 },
|
||||
now,
|
||||
});
|
||||
expect(doomed).toEqual([
|
||||
"/b/healthlog-20261023T013000Z-daily.dump.partial",
|
||||
]);
|
||||
});
|
||||
|
||||
it("works on a real directory and leaves files it did not write alone", async () => {
|
||||
const dir = tempDir();
|
||||
const names: string[] = [];
|
||||
for (let day = 15; day <= 24; day++) {
|
||||
const base = `healthlog-202610${day}T013000Z-daily`;
|
||||
names.push(`${base}.dump`, `${base}.manifest.json`);
|
||||
}
|
||||
// Two that are not ours, and an off-host copy an operator put here.
|
||||
names.push(
|
||||
"my-own-dump.dump",
|
||||
"healthlog-20261001T013000Z-daily.dump.bak",
|
||||
"healthlog-20261002T013000Z-daily.dump.enc",
|
||||
);
|
||||
for (const name of names) writeFileSync(join(dir, name), "x");
|
||||
// A crash's leftovers: one old, one from a run that may still be going.
|
||||
const oldPartial = join(
|
||||
dir,
|
||||
"healthlog-20261003T013000Z-daily.dump.partial",
|
||||
);
|
||||
writeFileSync(oldPartial, "x");
|
||||
const day = 24 * 60 * 60;
|
||||
utimesSync(oldPartial, Date.now() / 1000 - day, Date.now() / 1000 - day);
|
||||
writeFileSync(
|
||||
join(dir, "healthlog-20261025T013000Z-daily.dump.partial"),
|
||||
"x",
|
||||
);
|
||||
|
||||
const removed = await applyRetention(dir, { dailyKeep: 7 });
|
||||
expect(removed).toHaveLength(7);
|
||||
const left = (await readdir(dir)).sort();
|
||||
expect(left).toContain("my-own-dump.dump");
|
||||
expect(left).toContain("healthlog-20261001T013000Z-daily.dump.bak");
|
||||
expect(left).toContain("healthlog-20261002T013000Z-daily.dump.enc");
|
||||
expect(left).toContain("healthlog-20261025T013000Z-daily.dump.partial");
|
||||
expect(left).not.toContain("healthlog-20261003T013000Z-daily.dump.partial");
|
||||
expect(left.filter((n) => n.endsWith("-daily.dump"))).toEqual(
|
||||
[18, 19, 20, 21, 22, 23, 24].map(
|
||||
(d) => `healthlog-202610${d}T013000Z-daily.dump`,
|
||||
),
|
||||
);
|
||||
expect(listDumpSets(dir)[1].base).toBe("healthlog-20261024T013000Z-daily");
|
||||
});
|
||||
});
|
||||
|
||||
describe("key fingerprints", () => {
|
||||
it("are the fingerprints the app's own key code gives, for the map and the single key", async () => {
|
||||
vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ v1: HEX_A, v2: HEX_B }));
|
||||
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "v2");
|
||||
vi.stubEnv("ENCRYPTION_KEY", "");
|
||||
const crypto = await import("@/lib/crypto");
|
||||
crypto._resetCryptoCacheForTests();
|
||||
const prints = keyFingerprints(process.env);
|
||||
expect(prints.encryption.activeKeyId).toBe("v2");
|
||||
expect(prints.encryption.keys).toEqual([
|
||||
{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") },
|
||||
{ id: "v2", fingerprint: crypto.getKeyFingerprint("v2") },
|
||||
]);
|
||||
|
||||
vi.stubEnv("ENCRYPTION_KEYS", "");
|
||||
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "");
|
||||
const base64 = Buffer.from(HEX_C, "hex").toString("base64");
|
||||
vi.stubEnv("ENCRYPTION_KEY", base64);
|
||||
crypto._resetCryptoCacheForTests();
|
||||
const single = keyFingerprints(process.env);
|
||||
expect(single.encryption).toEqual({
|
||||
activeKeyId: "v1",
|
||||
keys: [{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }],
|
||||
});
|
||||
crypto._resetCryptoCacheForTests();
|
||||
});
|
||||
|
||||
it("names the off-host key the way an off-host object records it", async () => {
|
||||
const { offhostKeyId } = await import("@/lib/jobs/offhost-backup");
|
||||
const prints = keyFingerprints({
|
||||
ENCRYPTION_KEY: HEX_A,
|
||||
BACKUP_ENCRYPTION_KEY: HEX_D,
|
||||
BACKUP_ENCRYPTION_PREVIOUS_KEYS: ` ${HEX_C} ,`,
|
||||
API_TOKEN_HMAC_KEY: "hmac-secret-of-some-length-0123456789",
|
||||
});
|
||||
expect(prints.backupEncryptionKey).toBe(
|
||||
offhostKeyId(Buffer.from(HEX_D, "hex")),
|
||||
);
|
||||
expect(prints.backupEncryptionPreviousKeys).toEqual([
|
||||
offhostKeyId(Buffer.from(HEX_C, "hex")),
|
||||
]);
|
||||
expect(prints.apiTokenHmacKey).toMatch(/^[0-9a-f]{12}$/);
|
||||
});
|
||||
|
||||
it("never carries key material", () => {
|
||||
const env = {
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A }),
|
||||
BACKUP_ENCRYPTION_KEY: HEX_B,
|
||||
API_TOKEN_HMAC_KEY: HEX_C,
|
||||
};
|
||||
const text = JSON.stringify(keyFingerprints(env));
|
||||
for (const secret of [HEX_A, HEX_B, HEX_C]) {
|
||||
expect(text).not.toContain(secret.slice(0, 16));
|
||||
}
|
||||
});
|
||||
|
||||
it("decodes keys the way the app does, and nothing that is not a key", () => {
|
||||
expect(decodeKeyMaterial(HEX_A)?.length).toBe(32);
|
||||
expect(
|
||||
decodeKeyMaterial(Buffer.alloc(32, 7).toString("base64"))?.length,
|
||||
).toBe(32);
|
||||
expect(decodeKeyMaterial("")).toBeNull();
|
||||
expect(decodeKeyMaterial("not a key!")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("comparing a dump's keys with the environment's", () => {
|
||||
const written = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_B }),
|
||||
ENCRYPTION_ACTIVE_KEY_ID: "v2",
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-one",
|
||||
});
|
||||
|
||||
it("finds nothing when every key is there", () => {
|
||||
const same = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B, v1: HEX_A }),
|
||||
ENCRYPTION_ACTIVE_KEY_ID: "v2",
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-one",
|
||||
});
|
||||
expect(compareKeyFingerprints(written, same)).toEqual({
|
||||
findings: [],
|
||||
refuse: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses when the active key is different, and says which", () => {
|
||||
const wrong = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_D }),
|
||||
ENCRYPTION_ACTIVE_KEY_ID: "v2",
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-one",
|
||||
});
|
||||
const result = compareKeyFingerprints(written, wrong);
|
||||
expect(result.refuse).toBe(true);
|
||||
expect(result.findings).toHaveLength(1);
|
||||
expect(result.findings[0]).toMatchObject({
|
||||
key: "ENCRYPTION_KEYS[v2]",
|
||||
state: "different",
|
||||
severity: "refuse",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses when the active key is missing, warns for a retired one", () => {
|
||||
const onlyOld = keyFingerprints({ ENCRYPTION_KEY: HEX_A });
|
||||
const result = compareKeyFingerprints(written, onlyOld);
|
||||
expect(result.refuse).toBe(true);
|
||||
const byKey = Object.fromEntries(result.findings.map((f) => [f.key, f]));
|
||||
expect(byKey["ENCRYPTION_KEYS[v2]"]).toMatchObject({
|
||||
state: "missing",
|
||||
severity: "refuse",
|
||||
});
|
||||
expect(byKey["ENCRYPTION_KEYS[v1]"]).toBeUndefined();
|
||||
expect(byKey.BACKUP_ENCRYPTION_KEY).toMatchObject({
|
||||
state: "missing",
|
||||
severity: "warn",
|
||||
});
|
||||
expect(byKey.API_TOKEN_HMAC_KEY).toMatchObject({
|
||||
state: "missing",
|
||||
severity: "warn",
|
||||
});
|
||||
|
||||
const noRetired = keyFingerprints({
|
||||
ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B }),
|
||||
BACKUP_ENCRYPTION_KEY: HEX_C,
|
||||
API_TOKEN_HMAC_KEY: "hmac-two",
|
||||
});
|
||||
const second = compareKeyFingerprints(written, noRetired);
|
||||
expect(second.refuse).toBe(false);
|
||||
expect(second.findings.map((f) => [f.key, f.severity])).toEqual([
|
||||
["ENCRYPTION_KEYS[v1]", "warn"],
|
||||
["API_TOKEN_HMAC_KEY", "warn"],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the manifest", () => {
|
||||
const manifest = buildManifest({
|
||||
kind: "daily",
|
||||
file: "healthlog-20261025T013000Z-daily.dump",
|
||||
createdAt: new Date("2026-10-25T01:30:00Z"),
|
||||
appVersion: "1.46.0",
|
||||
buildSha: "abc1234",
|
||||
latestMigration: "0486_database_backup_runs",
|
||||
server: { version: "18.6", versionNum: 180006 },
|
||||
pgDump: "pg_dump (PostgreSQL) 18.6",
|
||||
archive: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
|
||||
rowCounts: { "public.users": 3, "pgboss.job": 12, "public.a": 0 },
|
||||
env: { ENCRYPTION_KEY: HEX_A, API_TOKEN_HMAC_KEY: "h" },
|
||||
});
|
||||
|
||||
it("carries what a restore and an operator need", () => {
|
||||
expect(manifest).toMatchObject({
|
||||
format: "healthlog-database-dump",
|
||||
formatVersion: 1,
|
||||
kind: "daily",
|
||||
file: "healthlog-20261025T013000Z-daily.dump",
|
||||
createdAt: "2026-10-25T01:30:00.000Z",
|
||||
appVersion: "1.46.0",
|
||||
buildSha: "abc1234",
|
||||
latestMigration: "0486_database_backup_runs",
|
||||
postgres: { serverVersion: "18.6", serverVersionNum: 180006 },
|
||||
dump: { bytes: 1234, sha256: "f".repeat(64), entries: 900 },
|
||||
});
|
||||
expect(Object.keys(manifest.rowCounts)).toEqual([
|
||||
"pgboss.job",
|
||||
"public.a",
|
||||
"public.users",
|
||||
]);
|
||||
expect(manifest.keys.encryption.keys[0].id).toBe("v1");
|
||||
});
|
||||
|
||||
it("reads back, and refuses anything else", () => {
|
||||
expect(parseManifest(JSON.parse(JSON.stringify(manifest)))).toEqual(
|
||||
manifest,
|
||||
);
|
||||
expect(() => parseManifest({ format: "something-else" })).toThrow(
|
||||
/not a HealthLog/,
|
||||
);
|
||||
expect(() => parseManifest({ ...manifest, formatVersion: 2 })).toThrow();
|
||||
expect(() => parseManifest(null)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("row counts after a restore", () => {
|
||||
it("names every table that is missing, extra or different", () => {
|
||||
expect(
|
||||
compareRowCounts(
|
||||
{ "public.a": 1, "public.b": 2 },
|
||||
{ "public.a": 1, "public.b": 2 },
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
compareRowCounts(
|
||||
{ "public.a": 1, "public.b": 2, "public.c": 5 },
|
||||
{ "public.a": 1, "public.b": 0, "public.d": 1 },
|
||||
),
|
||||
).toEqual([
|
||||
"public.b: 2 rows in the dump, 0 restored",
|
||||
"public.c: in the dump with 5 rows, missing now",
|
||||
"public.d: restored but not in the dump's manifest",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the connection the client tools get", () => {
|
||||
it("drops Prisma's parameters and keeps the password out of the arguments", () => {
|
||||
const c = connectionFromUrl(
|
||||
"postgresql://health%40log:p%40ss%2Fw@db:6543/hl?schema=app&connection_limit=20&pool_timeout=20&sslmode=require",
|
||||
);
|
||||
expect(c.pgEnv).toEqual({
|
||||
PGHOST: "db",
|
||||
PGPORT: "6543",
|
||||
PGUSER: "health@log",
|
||||
PGPASSWORD: "p@ss/w",
|
||||
PGDATABASE: "hl",
|
||||
PGSSLMODE: "require",
|
||||
});
|
||||
expect(c.schema).toBe("app");
|
||||
const cs = String(c.clientConfig.connectionString);
|
||||
expect(cs).not.toContain("connection_limit");
|
||||
expect(cs).not.toContain("schema=");
|
||||
expect(cs).toContain("sslmode=require");
|
||||
});
|
||||
});
|
||||
|
||||
describe("an off-host copy read back by the restore", () => {
|
||||
/** The uploader the job uses, into memory. */
|
||||
async function upload(objectKey: string, key: Buffer, body: Buffer) {
|
||||
const { uploadEncryptedBackup } = await import("@/lib/jobs/offhost-backup");
|
||||
const stored = new Map<string, Buffer>();
|
||||
const s3: S3Like = {
|
||||
putStream: async (k: string, stream: Readable) => {
|
||||
stored.set(k, await collect(stream));
|
||||
},
|
||||
putObject: async () => {},
|
||||
getObject: async () => Buffer.alloc(0),
|
||||
headObject: async () => false,
|
||||
listObjects: async () => [],
|
||||
deleteObject: async () => {},
|
||||
};
|
||||
await uploadEncryptedBackup(s3, objectKey, key, async (write) => {
|
||||
// Several pieces, as the job streams the file.
|
||||
for (let i = 0; i < body.length; i += 1000) {
|
||||
await write(body.subarray(i, i + 1000));
|
||||
}
|
||||
});
|
||||
return stored.get(objectKey) as Buffer;
|
||||
}
|
||||
|
||||
it("opens what the job's uploader sealed, byte for byte", async () => {
|
||||
const dir = tempDir();
|
||||
const key = Buffer.from(HEX_D, "hex");
|
||||
const body = Buffer.from(Array.from({ length: 50_000 }, (_, i) => i % 251));
|
||||
const name = "healthlog-20261025T013000Z-daily.dump.enc";
|
||||
const sealed = await upload(offhostObjectKey(name), key, body);
|
||||
writeFileSync(join(dir, name), sealed);
|
||||
|
||||
await decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: join(dir, "opened.dump"),
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys: [Buffer.from(HEX_A, "hex"), key],
|
||||
});
|
||||
expect(
|
||||
(await collect(createReadStream(join(dir, "opened.dump")))).equals(body),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("refuses a copy renamed, altered or under a key it does not hold, and leaves nothing", async () => {
|
||||
const dir = tempDir();
|
||||
const key = Buffer.from(HEX_D, "hex");
|
||||
const body = Buffer.from("PGDMP fake archive bytes".repeat(100));
|
||||
const name = "healthlog-20261025T013000Z-daily.dump.enc";
|
||||
const sealed = await upload(offhostObjectKey(name), key, body);
|
||||
writeFileSync(join(dir, name), sealed);
|
||||
const out = join(dir, "opened.dump");
|
||||
|
||||
// Renamed: the object key is bound as associated data.
|
||||
await expect(
|
||||
decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: out,
|
||||
objectKey: offhostObjectKey(
|
||||
"healthlog-20261024T013000Z-daily.dump.enc",
|
||||
),
|
||||
keys: [key],
|
||||
}),
|
||||
).rejects.toThrow(/does not open/);
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
|
||||
// Altered.
|
||||
const altered = Buffer.from(sealed);
|
||||
altered[altered.length - 40] ^= 1;
|
||||
writeFileSync(join(dir, name), altered);
|
||||
await expect(
|
||||
decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: out,
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys: [key],
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
|
||||
// A key it was not written under.
|
||||
writeFileSync(join(dir, name), sealed);
|
||||
await expect(
|
||||
decryptOffhostFile({
|
||||
inPath: join(dir, name),
|
||||
outPath: out,
|
||||
objectKey: offhostObjectKey(name),
|
||||
keys: [Buffer.from(HEX_A, "hex")],
|
||||
}),
|
||||
).rejects.toThrow(/neither BACKUP_ENCRYPTION_KEY nor/);
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("finding a dump in the bucket", () => {
|
||||
const keys = [
|
||||
"database/healthlog-20261023T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261023T013000Z-daily.manifest.json.enc",
|
||||
"database/healthlog-20261024T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
|
||||
// The newest dump whose manifest never arrived, and things that are
|
||||
// not dumps at all.
|
||||
"database/healthlog-20261025T013000Z-daily.dump.enc",
|
||||
"2026-10-25/user-abc.json.enc",
|
||||
"database/notes.txt",
|
||||
];
|
||||
|
||||
it("takes the newest dump whose manifest is there too", () => {
|
||||
expect(pickBucketDump(keys)).toEqual({
|
||||
dumpKey: "database/healthlog-20261024T013000Z-daily.dump.enc",
|
||||
manifestKey:
|
||||
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
|
||||
});
|
||||
});
|
||||
|
||||
it("finds a named one with or without its suffix, and nothing else", () => {
|
||||
for (const name of [
|
||||
"healthlog-20261023T013000Z-daily.dump",
|
||||
"healthlog-20261023T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261023T013000Z-daily.dump.enc",
|
||||
]) {
|
||||
expect(pickBucketDump(keys, name)?.dumpKey).toBe(
|
||||
"database/healthlog-20261023T013000Z-daily.dump.enc",
|
||||
);
|
||||
}
|
||||
expect(
|
||||
pickBucketDump(keys, "healthlog-20261025T013000Z-daily.dump"),
|
||||
).toBeNull();
|
||||
expect(pickBucketDump([])).toBeNull();
|
||||
});
|
||||
|
||||
it("reads the bucket from the four required variables", () => {
|
||||
const env = {
|
||||
BACKUP_S3_ENDPOINT: "https://s3.example.test",
|
||||
BACKUP_S3_BUCKET: "b",
|
||||
BACKUP_S3_ACCESS_KEY: "a",
|
||||
BACKUP_S3_SECRET_KEY: "s",
|
||||
};
|
||||
expect(bucketConfigFromEnv(env)).toEqual({
|
||||
endpoint: "https://s3.example.test",
|
||||
bucket: "b",
|
||||
accessKey: "a",
|
||||
secretKey: "s",
|
||||
region: "auto",
|
||||
});
|
||||
expect(
|
||||
bucketConfigFromEnv({ ...env, BACKUP_S3_SECRET_KEY: "" }),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* Whether the nightly whole-database dump is happening, as facts a page or an
|
||||
* alert can act on.
|
||||
*
|
||||
* Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of
|
||||
* this process: whether `DB_BACKUP_DIR` is writable here, and whether the
|
||||
* off-host bucket is configured. A ledger row that says a copy went off-host
|
||||
* outranks this process's environment, for the reason the off-host card gives
|
||||
* (a split install can hand the bucket variables to the worker only).
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
|
||||
import {
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
resolveBackupDir,
|
||||
resolveDailyKeep,
|
||||
} from "./dump-core.mjs";
|
||||
|
||||
export interface DatabaseBackupHealth {
|
||||
/** `DB_BACKUP_ENABLED` is not off. */
|
||||
enabled: boolean;
|
||||
/** The directory dumps are kept in (`DB_BACKUP_DIR`). */
|
||||
directory: string;
|
||||
/**
|
||||
* Nightly dumps kept in that directory (`DB_BACKUP_KEEP`). Also how long a
|
||||
* deleted account stays in the dumps on this server, which the backups
|
||||
* page says.
|
||||
*/
|
||||
keepNightly: number;
|
||||
/** This process can write there. False means dumps only go off-host. */
|
||||
localDirWritable: boolean;
|
||||
/** The off-host bucket is configured, or a dump has been uploaded to it. */
|
||||
offhostConfigured: boolean;
|
||||
/** The newest dump that was kept on the server or reached the bucket. */
|
||||
lastGood: {
|
||||
at: string;
|
||||
fileName: string;
|
||||
sizeBytes: number | null;
|
||||
local: boolean;
|
||||
offhost: boolean;
|
||||
} | null;
|
||||
/** The newest run that failed, or that a dying process left unfinished. */
|
||||
lastFailure: { at: string; error: string } | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A run still `running` this long after it started was cut off by its
|
||||
* process: the job's expiry, past which pg-boss has failed it anyway.
|
||||
*/
|
||||
const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000;
|
||||
|
||||
export async function getDatabaseBackupHealth(
|
||||
now: Date = new Date(),
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<DatabaseBackupHealth> {
|
||||
const directory = resolveBackupDir(env);
|
||||
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
|
||||
[
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: {
|
||||
fileName: { not: null },
|
||||
OR: [{ localResult: "written" }, { offhostResult: "uploaded" }],
|
||||
},
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: {
|
||||
startedAt: true,
|
||||
finishedAt: true,
|
||||
fileName: true,
|
||||
dumpBytes: true,
|
||||
localResult: true,
|
||||
offhostResult: true,
|
||||
},
|
||||
}),
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: { status: "failed" },
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true, finishedAt: true, error: true },
|
||||
}),
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: {
|
||||
status: "running",
|
||||
startedAt: { lt: new Date(now.getTime() - ABANDONED_AFTER_MS) },
|
||||
},
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true },
|
||||
}),
|
||||
prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }),
|
||||
],
|
||||
);
|
||||
|
||||
const failures = [
|
||||
lastFailed && {
|
||||
at: (lastFailed.finishedAt ?? lastFailed.startedAt).toISOString(),
|
||||
error: lastFailed.error ?? "failed without a reason",
|
||||
},
|
||||
lastAbandoned && {
|
||||
at: lastAbandoned.startedAt.toISOString(),
|
||||
error: "the run did not finish (the process stopped under it)",
|
||||
},
|
||||
]
|
||||
.filter((f): f is { at: string; error: string } => Boolean(f))
|
||||
.sort((a, b) => b.at.localeCompare(a.at));
|
||||
|
||||
return {
|
||||
enabled: databaseBackupEnabled(env),
|
||||
directory,
|
||||
keepNightly: resolveDailyKeep(env),
|
||||
localDirWritable: isWritableDirectory(directory),
|
||||
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
|
||||
lastGood: lastGood
|
||||
? {
|
||||
at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(),
|
||||
fileName: lastGood.fileName ?? "",
|
||||
sizeBytes:
|
||||
lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes),
|
||||
local: lastGood.localResult === "written",
|
||||
offhost: lastGood.offhostResult === "uploaded",
|
||||
}
|
||||
: null,
|
||||
lastFailure: failures[0] ?? null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* One run of the nightly whole-database dump.
|
||||
*
|
||||
* The dump is the artifact that rebuilds a lost server: every account, every
|
||||
* login, every grant and managed profile, the job queue and the instance
|
||||
* settings, in one `pg_dump --format=custom` file with a manifest beside it
|
||||
* (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the
|
||||
* bundled compose file) and, when the off-host bucket is configured, uploaded
|
||||
* under `database/` in the same envelope the per-account copies use, sealed
|
||||
* with `BACKUP_ENCRYPTION_KEY`.
|
||||
*
|
||||
* Without a writable directory (an install that runs the image without the
|
||||
* volume) the dump is written to a temporary directory, uploaded and removed:
|
||||
* the off-host copy still happens, nothing is kept on the server, and the
|
||||
* ledger and the boot summary say so. With neither, the run is skipped and
|
||||
* recorded as such.
|
||||
*
|
||||
* Every run leaves one `DatabaseBackupRun` row: started, then finished with
|
||||
* what happened on each leg. `getDatabaseBackupHealth` reads them.
|
||||
*/
|
||||
import { createReadStream } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, join } from "node:path";
|
||||
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import {
|
||||
getS3Client,
|
||||
loadOffhostConfig,
|
||||
uploadEncryptedBackup,
|
||||
type S3Like,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import packageJson from "../../../package.json";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
createDatabaseDump,
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
offhostObjectKey,
|
||||
resolveBackupDir,
|
||||
resolveDailyKeep,
|
||||
} from "./dump-core.mjs";
|
||||
|
||||
/** How long ledger rows are kept. A run a day; a year and a bit of history. */
|
||||
const LEDGER_RETENTION_DAYS = 400;
|
||||
|
||||
export type DatabaseBackupLocalResult =
|
||||
"written" | "no_directory" | "failed" | "skipped";
|
||||
export type DatabaseBackupOffhostResult =
|
||||
"uploaded" | "not_configured" | "failed" | "skipped";
|
||||
|
||||
export interface DatabaseBackupReport {
|
||||
status: "succeeded" | "failed" | "skipped";
|
||||
fileName: string | null;
|
||||
dumpBytes: number | null;
|
||||
local: DatabaseBackupLocalResult;
|
||||
offhost: DatabaseBackupOffhostResult;
|
||||
/** Dumps and manifests retention removed. */
|
||||
removed: number;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface RunDatabaseBackupOptions {
|
||||
now?: Date;
|
||||
/** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */
|
||||
deadline?: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** A bucket double for tests; the configured bucket otherwise. */
|
||||
s3?: S3Like;
|
||||
}
|
||||
|
||||
/** The running build's version, as `/api/version` answers it. */
|
||||
function appVersion(env: NodeJS.ProcessEnv): string {
|
||||
return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version;
|
||||
}
|
||||
|
||||
/** An error as one line for the ledger and the admin page. */
|
||||
function reason(err: unknown): string {
|
||||
const text = err instanceof Error ? err.message : String(err);
|
||||
return text.replace(/\s+/g, " ").trim().slice(0, 500);
|
||||
}
|
||||
|
||||
/** Upload one local file as an off-host object, streamed. */
|
||||
async function uploadFile(
|
||||
s3: S3Like,
|
||||
key: Buffer,
|
||||
path: string,
|
||||
): Promise<number> {
|
||||
return uploadEncryptedBackup(
|
||||
s3,
|
||||
offhostObjectKey(`${basename(path)}.enc`),
|
||||
key,
|
||||
async (write) => {
|
||||
for await (const chunk of createReadStream(path)) {
|
||||
await write(chunk as Buffer);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Take tonight's dump, keep it, copy it off-host, prune the old ones, and
|
||||
* record what happened. Never throws: a failure is the report's `status`.
|
||||
*/
|
||||
export async function runDatabaseBackup(
|
||||
prisma: PrismaClient,
|
||||
options: RunDatabaseBackupOptions = {},
|
||||
): Promise<DatabaseBackupReport> {
|
||||
const env = options.env ?? process.env;
|
||||
const now = options.now ?? new Date();
|
||||
const dir = resolveBackupDir(env);
|
||||
const localWritable = isWritableDirectory(dir);
|
||||
const offhostConfig = (() => {
|
||||
try {
|
||||
return loadOffhostConfig();
|
||||
} catch {
|
||||
// A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the
|
||||
// same value with their own message; here it reads as an upload error.
|
||||
return "invalid" as const;
|
||||
}
|
||||
})();
|
||||
|
||||
// Each leg reads "failed" until it has succeeded, so a run the process died
|
||||
// under says so once its expiry has passed.
|
||||
const report: DatabaseBackupReport = {
|
||||
status: "failed",
|
||||
fileName: null,
|
||||
dumpBytes: null,
|
||||
local: localWritable ? "failed" : "no_directory",
|
||||
offhost: offhostConfig ? "failed" : "not_configured",
|
||||
removed: 0,
|
||||
error: null,
|
||||
};
|
||||
const run = await prisma.databaseBackupRun.create({
|
||||
data: {
|
||||
startedAt: now,
|
||||
status: "running",
|
||||
localResult: report.local,
|
||||
offhostResult: report.offhost,
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
let scratch: string | null = null;
|
||||
try {
|
||||
if (!databaseBackupEnabled(env)) {
|
||||
report.status = "skipped";
|
||||
report.local = "skipped";
|
||||
report.offhost = "skipped";
|
||||
report.error = "DB_BACKUP_ENABLED is false";
|
||||
} else if (!localWritable && !offhostConfig) {
|
||||
report.status = "skipped";
|
||||
report.error = `no writable backup directory (${dir}) and no off-host bucket configured`;
|
||||
} else {
|
||||
const target = localWritable
|
||||
? dir
|
||||
: (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-")));
|
||||
const dump = await createDatabaseDump({
|
||||
databaseUrl: env.DATABASE_URL ?? "",
|
||||
dir: target,
|
||||
kind: "daily",
|
||||
now,
|
||||
env,
|
||||
appVersion: appVersion(env),
|
||||
buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null,
|
||||
deadline: options.deadline,
|
||||
});
|
||||
if (localWritable) report.local = "written";
|
||||
report.fileName = basename(dump.dumpPath);
|
||||
report.dumpBytes = dump.manifest.dump.bytes;
|
||||
|
||||
let uploadError: string | null = null;
|
||||
if (offhostConfig === "invalid") {
|
||||
uploadError =
|
||||
"BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64";
|
||||
} else if (offhostConfig) {
|
||||
try {
|
||||
const s3 = options.s3 ?? (await getS3Client(offhostConfig));
|
||||
// The manifest goes up first: a dump object in the bucket always
|
||||
// has the manifest a restore checks it against.
|
||||
await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath);
|
||||
await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath);
|
||||
report.offhost = "uploaded";
|
||||
} catch (err) {
|
||||
uploadError = `off-host upload failed: ${reason(err)}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (localWritable) {
|
||||
const removed = await applyRetention(dir, {
|
||||
dailyKeep: resolveDailyKeep(env),
|
||||
now: now.getTime(),
|
||||
});
|
||||
report.removed = removed.length;
|
||||
}
|
||||
|
||||
// Good when the dump exists somewhere it survives: on the volume, or in
|
||||
// the bucket. An upload that failed beside a kept dump still fails the
|
||||
// run, because the operator asked for both.
|
||||
report.status = uploadError ? "failed" : "succeeded";
|
||||
report.error = uploadError;
|
||||
}
|
||||
} catch (err) {
|
||||
report.status = "failed";
|
||||
report.error = reason(err);
|
||||
} finally {
|
||||
if (scratch) await rm(scratch, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
await prisma.databaseBackupRun.update({
|
||||
where: { id: run.id },
|
||||
data: {
|
||||
finishedAt: new Date(),
|
||||
status: report.status,
|
||||
fileName: report.fileName,
|
||||
dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes),
|
||||
localResult: report.local,
|
||||
offhostResult: report.offhost,
|
||||
error: report.error,
|
||||
},
|
||||
});
|
||||
await prisma.databaseBackupRun.deleteMany({
|
||||
where: {
|
||||
startedAt: {
|
||||
lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
annotate({
|
||||
action: { name: "database.backup.run" },
|
||||
meta: {
|
||||
status: report.status,
|
||||
local: report.local,
|
||||
offhost: report.offhost,
|
||||
dump_bytes: report.dumpBytes ?? 0,
|
||||
removed: report.removed,
|
||||
},
|
||||
});
|
||||
return report;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Which lifecycle rules retire the database dumps under `database/`, apart
|
||||
* from the account copies: a rule scoped to another prefix, or to a tag, does
|
||||
* not touch them.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { LifecycleRule } from "@aws-sdk/client-s3";
|
||||
|
||||
import { lifecycleFromRules, ruleAppliesTo } from "../offhost-lifecycle";
|
||||
|
||||
const DUMP = "database/healthlog-20261025T013000Z-daily.dump.enc";
|
||||
|
||||
function rule(days: number, extra: Partial<LifecycleRule> = {}): LifecycleRule {
|
||||
return { Status: "Enabled", Expiration: { Days: days }, ...extra };
|
||||
}
|
||||
|
||||
describe("lifecycle rules and the database dumps", () => {
|
||||
it("counts a rule for the whole bucket, or for database/, for both", () => {
|
||||
expect(lifecycleFromRules([rule(30)])).toEqual({
|
||||
state: "configured",
|
||||
expirationDays: 30,
|
||||
database: { state: "configured", expirationDays: 30 },
|
||||
});
|
||||
expect(
|
||||
lifecycleFromRules([
|
||||
rule(30, { Filter: { Prefix: "2026-" } }),
|
||||
rule(14, { Filter: { Prefix: "database/" } }),
|
||||
]).database,
|
||||
).toEqual({ state: "configured", expirationDays: 14 });
|
||||
});
|
||||
|
||||
it("says database/ is not covered when the only rule is scoped elsewhere", () => {
|
||||
const result = lifecycleFromRules([
|
||||
rule(30, { Filter: { Prefix: "2026-" } }),
|
||||
]);
|
||||
expect(result.state).toBe("configured");
|
||||
expect(result.database).toEqual({ state: "missing", expirationDays: null });
|
||||
});
|
||||
|
||||
it("reads the legacy prefix, the And filter, tags and disabled rules", () => {
|
||||
expect(ruleAppliesTo(rule(5, { Prefix: "database/" }), DUMP)).toBe(true);
|
||||
expect(ruleAppliesTo(rule(5, { Prefix: "other/" }), DUMP)).toBe(false);
|
||||
expect(
|
||||
ruleAppliesTo(
|
||||
rule(5, {
|
||||
Filter: { And: { Prefix: "database/", ObjectSizeGreaterThan: 1 } },
|
||||
}),
|
||||
DUMP,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
ruleAppliesTo(
|
||||
rule(5, { Filter: { Tag: { Key: "k", Value: "v" } } }),
|
||||
DUMP,
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
lifecycleFromRules([rule(7, { Status: "Disabled" })]).database.state,
|
||||
).toBe("missing");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* The queue facts of the nightly whole-database dump: its name, its schedule
|
||||
* and its send options. Apart from the job so the worker registry can read
|
||||
* them without loading it (`job-registry-graph.test.ts`).
|
||||
*/
|
||||
import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy";
|
||||
import { resolveBackupSchedule } from "@/lib/database-backup/dump-core.mjs";
|
||||
|
||||
export const DATABASE_BACKUP_QUEUE = "database-backup";
|
||||
|
||||
/**
|
||||
* `DB_BACKUP_SCHEDULE`, read once when the worker loads its registry, in the
|
||||
* zone every cron runs in (`createAndSchedule`, Europe/Berlin). Default 01:30,
|
||||
* an hour before the off-host account copies so the two uploads do not meet.
|
||||
* A value that is not a five-field cron keeps the default, and the boot
|
||||
* readiness summary says so.
|
||||
*/
|
||||
export const DATABASE_BACKUP_CRON = resolveBackupSchedule(process.env).cron;
|
||||
|
||||
/**
|
||||
* Four hours. `pg_dump` of a large instance takes minutes, the upload of the
|
||||
* dump about as long as the dump; the window is there to be impossible to hit
|
||||
* for a working run. The handler kills `pg_dump` at three quarters of it
|
||||
* (`jobDeadline`), so a hung dump ends as a failed run with a reason rather
|
||||
* than an expired job.
|
||||
*/
|
||||
export const DATABASE_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60;
|
||||
|
||||
/**
|
||||
* What the nightly schedule sends with: the expiry, the backups' heartbeat,
|
||||
* and one retry. A dump that failed for a passing reason (a lock it waited
|
||||
* out, a bucket that answered 503) gets a second attempt; one that fails
|
||||
* again waits for the next night, and the failure stays on the admin page.
|
||||
*/
|
||||
export const DATABASE_BACKUP_SEND_OPTIONS = {
|
||||
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
|
||||
heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS,
|
||||
retryLimit: 1,
|
||||
retryDelay: 15 * 60,
|
||||
} as const;
|
||||
@@ -78,6 +78,12 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"continued",
|
||||
"conversation_titles_migrated",
|
||||
"created",
|
||||
// The nightly whole-database dump: where it went and how big.
|
||||
"database_backup_bytes",
|
||||
"database_backup_local",
|
||||
"database_backup_offhost",
|
||||
"database_backup_removed",
|
||||
"database_backup_status",
|
||||
"daily_rows_retired",
|
||||
"days_consolidated",
|
||||
"days_rebuilt",
|
||||
|
||||
@@ -44,6 +44,12 @@ import {
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import { envValue } from "@/lib/env";
|
||||
import { OFFHOST_BACKUP_QUEUE } from "./offhost-backup-queue";
|
||||
import {
|
||||
LIFECYCLE_MISSING,
|
||||
LIFECYCLE_UNKNOWN,
|
||||
lifecycleFromRules,
|
||||
type OffhostLifecycle,
|
||||
} from "./offhost-lifecycle";
|
||||
|
||||
const ALGORITHM = "aes-256-gcm";
|
||||
const IV_LENGTH = 12;
|
||||
@@ -494,9 +500,9 @@ function createEnvelopeStream(
|
||||
return { stream, bytes: () => written };
|
||||
}
|
||||
|
||||
/** Produces the backup JSON in pieces. Every piece is written in order. */
|
||||
/** Produces the plaintext (JSON text, or a dump's bytes) in ordered pieces. */
|
||||
export type BackupJsonProducer = (
|
||||
write: (chunk: string) => Promise<void>,
|
||||
write: (chunk: string | Buffer) => Promise<void>,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export interface UploadBackupOptions {
|
||||
@@ -572,9 +578,9 @@ export async function uploadEncryptedBackup(
|
||||
},
|
||||
);
|
||||
|
||||
const write = async (chunk: string): Promise<void> => {
|
||||
const write = async (chunk: string | Buffer): Promise<void> => {
|
||||
if (failure) throw failure;
|
||||
if (gzip.write(chunk, "utf8")) return;
|
||||
if (gzip.write(chunk, "utf8")) return; // a Buffer ignores the encoding
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = (): void => {
|
||||
gzip.off("error", onError);
|
||||
@@ -634,18 +640,7 @@ export interface S3Like {
|
||||
getLifecycle?(): Promise<OffhostLifecycle>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the bucket retires old copies by itself.
|
||||
*
|
||||
* `configured` names the shortest expiry of an enabled rule; `missing` means
|
||||
* the bucket answered that it has no rule; `unknown` means it would not say
|
||||
* (a credential without `s3:GetLifecycleConfiguration`, a target that does not
|
||||
* implement the call, a network failure) and says nothing either way.
|
||||
*/
|
||||
export interface OffhostLifecycle {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
}
|
||||
export type { OffhostLifecycle } from "./offhost-lifecycle";
|
||||
|
||||
export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
// Dynamic import so unit tests + dev environments without the SDK don't fail.
|
||||
@@ -765,13 +760,7 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(5_000) },
|
||||
);
|
||||
const days = (out.Rules ?? [])
|
||||
.filter((rule) => rule.Status === "Enabled")
|
||||
.map((rule) => rule.Expiration?.Days)
|
||||
.filter((d): d is number => typeof d === "number" && d > 0);
|
||||
return days.length > 0
|
||||
? { state: "configured", expirationDays: Math.min(...days) }
|
||||
: { state: "missing", expirationDays: null };
|
||||
return lifecycleFromRules(out.Rules ?? []);
|
||||
} catch (err) {
|
||||
const name = (err as { name?: string; Code?: string }).name;
|
||||
const code = (err as { Code?: string }).Code;
|
||||
@@ -779,9 +768,9 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
name === "NoSuchLifecycleConfiguration" ||
|
||||
code === "NoSuchLifecycleConfiguration"
|
||||
) {
|
||||
return { state: "missing", expirationDays: null };
|
||||
return LIFECYCLE_MISSING;
|
||||
}
|
||||
return { state: "unknown", expirationDays: null };
|
||||
return LIFECYCLE_UNKNOWN;
|
||||
}
|
||||
},
|
||||
};
|
||||
@@ -806,15 +795,13 @@ export async function probeOffhostLifecycle(
|
||||
return lifecycleCache.value;
|
||||
}
|
||||
const cfg = loadOffhostConfigSafe();
|
||||
if (!cfg) return { state: "unknown", expirationDays: null };
|
||||
if (!cfg) return LIFECYCLE_UNKNOWN;
|
||||
let value: OffhostLifecycle;
|
||||
try {
|
||||
const s3 = s3Override ?? (await getS3Client(cfg));
|
||||
value = s3.getLifecycle
|
||||
? await s3.getLifecycle()
|
||||
: { state: "unknown", expirationDays: null };
|
||||
value = s3.getLifecycle ? await s3.getLifecycle() : LIFECYCLE_UNKNOWN;
|
||||
} catch {
|
||||
value = { state: "unknown", expirationDays: null };
|
||||
value = LIFECYCLE_UNKNOWN;
|
||||
}
|
||||
if (!s3Override) lifecycleCache = { at: now, value };
|
||||
return value;
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Whether the operator's bucket retires old copies by itself, read from its
|
||||
* lifecycle rules (`GetBucketLifecycleConfiguration`).
|
||||
*
|
||||
* Two answers, because the bucket holds two kinds of object with different
|
||||
* keys: the per-account copies (`<date>/user-<id>.json.enc`) and the database
|
||||
* dumps (`database/…`). A rule scoped to a prefix covers only the keys under
|
||||
* it, so a bucket whose one rule expires `2026-` keys ages the account copies
|
||||
* out and keeps every database dump forever, and a database dump keeps every
|
||||
* deleted account's data for as long as it exists. The account answer keeps
|
||||
* its earlier reading (the shortest expiry of any enabled rule); the database
|
||||
* answer counts only the rules that apply to a key under `database/`.
|
||||
*
|
||||
* `configured` names the shortest expiry; `missing` means the bucket answered
|
||||
* that no rule applies; `unknown` means it would not say (a credential
|
||||
* without `s3:GetLifecycleConfiguration`, a target that does not implement the
|
||||
* call, a network failure) and says nothing either way.
|
||||
*/
|
||||
import type { LifecycleRule } from "@aws-sdk/client-s3";
|
||||
|
||||
export interface LifecycleVerdict {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
}
|
||||
|
||||
export interface OffhostLifecycle extends LifecycleVerdict {
|
||||
/** The same question for the database dumps under `database/`. */
|
||||
database: LifecycleVerdict;
|
||||
}
|
||||
|
||||
export const LIFECYCLE_UNKNOWN: OffhostLifecycle = {
|
||||
state: "unknown",
|
||||
expirationDays: null,
|
||||
database: { state: "unknown", expirationDays: null },
|
||||
};
|
||||
|
||||
export const LIFECYCLE_MISSING: OffhostLifecycle = {
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
database: { state: "missing", expirationDays: null },
|
||||
};
|
||||
|
||||
/** A key a database dump is stored under, to test the rules against. */
|
||||
const DATABASE_SAMPLE_KEY =
|
||||
"database/healthlog-20260101T013000Z-daily.dump.enc";
|
||||
|
||||
/**
|
||||
* Whether a rule's filter takes in `key`. Our objects carry no tags, so a
|
||||
* rule that filters on a tag never applies to them; a size bound is not a
|
||||
* reason to call a rule inapplicable, so it is read as applying.
|
||||
*/
|
||||
export function ruleAppliesTo(rule: LifecycleRule, key: string): boolean {
|
||||
const filter = rule.Filter;
|
||||
if (filter?.Tag || filter?.And?.Tags?.length) return false;
|
||||
const prefix = filter?.Prefix ?? filter?.And?.Prefix ?? rule.Prefix ?? "";
|
||||
return key.startsWith(prefix);
|
||||
}
|
||||
|
||||
function verdict(rules: readonly LifecycleRule[]): LifecycleVerdict {
|
||||
const days = rules
|
||||
.filter((rule) => rule.Status === "Enabled")
|
||||
.map((rule) => rule.Expiration?.Days)
|
||||
.filter((d): d is number => typeof d === "number" && d > 0);
|
||||
return days.length > 0
|
||||
? { state: "configured", expirationDays: Math.min(...days) }
|
||||
: { state: "missing", expirationDays: null };
|
||||
}
|
||||
|
||||
/** Both answers from the rules a bucket returned. */
|
||||
export function lifecycleFromRules(
|
||||
rules: readonly LifecycleRule[],
|
||||
): OffhostLifecycle {
|
||||
return {
|
||||
...verdict(rules),
|
||||
database: verdict(
|
||||
rules.filter((rule) => ruleAppliesTo(rule, DATABASE_SAMPLE_KEY)),
|
||||
),
|
||||
};
|
||||
}
|
||||
@@ -42,6 +42,7 @@ import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key-
|
||||
import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill";
|
||||
import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run";
|
||||
import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy";
|
||||
import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue";
|
||||
import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue";
|
||||
import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore";
|
||||
import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue";
|
||||
@@ -332,6 +333,17 @@ export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
|
||||
exclusive: "lockedPass",
|
||||
why: "Serialises every account's whole record.",
|
||||
},
|
||||
"database-backup": {
|
||||
runtime: "long",
|
||||
expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS,
|
||||
expiryVia: ["DATABASE_BACKUP_SEND_OPTIONS"],
|
||||
stop: {
|
||||
file: "lib/jobs/reminder/backup-handlers.ts",
|
||||
fn: "handleDatabaseBackup",
|
||||
},
|
||||
exclusive: "lockedPass",
|
||||
why: "Runs pg_dump over the whole database and uploads the dump off-host.",
|
||||
},
|
||||
"data-backup-offhost": {
|
||||
runtime: "long",
|
||||
expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* The housekeeping queues: backups (weekly in-database, nightly off-host),
|
||||
* The housekeeping queues: backups (the nightly whole-database dump, weekly
|
||||
* in-database account copies, nightly off-host),
|
||||
* the restore drill and restores, every ledger and retention cleanup, the
|
||||
* medication maintenance passes, the imports (Apple Health, Health Connect,
|
||||
* medication intake), the converging encryption backfills, the document vault
|
||||
@@ -47,6 +48,11 @@ import {
|
||||
DATA_BACKUP_QUEUE,
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/data-backup-policy";
|
||||
import {
|
||||
DATABASE_BACKUP_CRON,
|
||||
DATABASE_BACKUP_QUEUE,
|
||||
DATABASE_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/database-backup-queue";
|
||||
import {
|
||||
DAWARICH_SYNC_CRON,
|
||||
DAWARICH_SYNC_QUEUE,
|
||||
@@ -300,6 +306,16 @@ const jobs: readonly JobDefinition[] = [
|
||||
lane: true,
|
||||
load: async () => (await backups()).handleDataBackup,
|
||||
}),
|
||||
// The whole-database dump: the copy that rebuilds a lost server. Locked, so
|
||||
// a retry that starts while a run is still dumping waits its turn.
|
||||
defineJob({
|
||||
queue: DATABASE_BACKUP_QUEUE,
|
||||
cron: DATABASE_BACKUP_CRON,
|
||||
scheduleOptions: DATABASE_BACKUP_SEND_OPTIONS,
|
||||
work: { localConcurrency: 1 },
|
||||
lock: () => WHOLE_PASS,
|
||||
load: async () => (await backups()).handleDatabaseBackup,
|
||||
}),
|
||||
prune(
|
||||
RATE_LIMIT_CLEANUP_QUEUE,
|
||||
RATE_LIMIT_CLEANUP_CRON,
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* schedules, and boss.work registrations.
|
||||
*/
|
||||
import { type Job, type JobWithMetadata } from "pg-boss";
|
||||
import { jobBudget } from "@/lib/jobs/job-budget";
|
||||
import { jobBudget, jobDeadline } from "@/lib/jobs/job-budget";
|
||||
import { recordError } from "@/lib/jobs/worker-status";
|
||||
import {
|
||||
BackupBlobTooLargeError,
|
||||
@@ -31,6 +31,37 @@ import {
|
||||
} from "@/lib/jobs/backup-pass-attempts";
|
||||
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
||||
|
||||
/**
|
||||
* The nightly whole-database dump (`database-backup`).
|
||||
*
|
||||
* One `pg_dump` in a child process, so it is not in the heavy lane: the heap
|
||||
* holds a part of the upload at a time, not the database. `pg_dump` is stopped
|
||||
* at three quarters of the job's expiry (`jobDeadline`), so a dump that hangs
|
||||
* on a lock ends as a failed run with its reason rather than an expired job.
|
||||
* A skipped run (switched off, nowhere to put it) completes; the admin page
|
||||
* and the boot summary carry why.
|
||||
*/
|
||||
export async function handleDatabaseBackup(
|
||||
jobs: Job<Record<string, never>>[],
|
||||
): Promise<JobOutcome> {
|
||||
return withBackgroundEvent("job.database_backup", async () => {
|
||||
const { runDatabaseBackup } = await import("@/lib/database-backup/run");
|
||||
const report = await runDatabaseBackup(getWorkerPrisma(), {
|
||||
deadline: jobs[0] ? jobDeadline(jobs[0]) : undefined,
|
||||
});
|
||||
const did = {
|
||||
database_backup_status: report.status,
|
||||
database_backup_local: report.local,
|
||||
database_backup_offhost: report.offhost,
|
||||
database_backup_bytes: report.dumpBytes ?? 0,
|
||||
database_backup_removed: report.removed,
|
||||
};
|
||||
return report.status === "failed"
|
||||
? jobFailed("database dump failed", report.error ?? undefined, did)
|
||||
: jobDone(did);
|
||||
});
|
||||
}
|
||||
|
||||
export interface DataBackupPayload {
|
||||
triggeredAt: string;
|
||||
}
|
||||
|
||||
@@ -99,6 +99,7 @@ const subsystemSurface = {
|
||||
"audit-log-cleanup": { audience: "system" },
|
||||
"step-up-elevation-cleanup": { audience: "system" },
|
||||
"data-backup-offhost": { audience: "system" },
|
||||
"database-backup": { audience: "system" },
|
||||
"data-restore-drill": { audience: "system" },
|
||||
"offhost-backup-purge": { audience: "system" },
|
||||
"backup-restore": { audience: "system" },
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
* both sides a route-handler-independent shared type home; the route
|
||||
* keeps owning the HTTP contract, the component keeps owning the UI.
|
||||
*/
|
||||
import type { DatabaseBackupHealth } from "@/lib/database-backup/health";
|
||||
import type { OffhostLifecycle } from "@/lib/jobs/offhost-lifecycle";
|
||||
import type { BackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
|
||||
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
|
||||
import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness";
|
||||
@@ -76,10 +78,7 @@ export interface OffhostBackupOverview {
|
||||
* (`GetBucketLifecycleConfiguration`), cached for ten minutes; `unknown`
|
||||
* when the credential may not read it.
|
||||
*/
|
||||
lifecycle: {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
};
|
||||
lifecycle: OffhostLifecycle;
|
||||
}
|
||||
|
||||
export interface BackupsList {
|
||||
@@ -101,6 +100,12 @@ export interface BackupsList {
|
||||
* the operator's bucket, and those two fail independently.
|
||||
*/
|
||||
offhost: OffhostBackupOverview;
|
||||
/**
|
||||
* The nightly whole-database dump, the copy that rebuilds a lost server:
|
||||
* the newest good dump (kept here, off-host, or both), the newest failed
|
||||
* run, and whether this server can keep dumps at all.
|
||||
*/
|
||||
database: DatabaseBackupHealth;
|
||||
/**
|
||||
* Soft retention hint — the worker is configured for weekly backups
|
||||
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and
|
||||
|
||||
Reference in new issue
Block a user