Merge remote-tracking branch 'origin/release/v1.46.0' into fix/v146-move-safety

This commit is contained in:
Marc-André Bombeck committed 2026-10-25 16:49:06 +01:00
commit 0085915a56
53 files changed
+1620 -246

No files matched your search

@@ -1,7 +1,7 @@
{
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 84,
"total": 94948,
"total": 94928,
"entries": {
"app/medications/page-client.tsx": 839,
"components/charts/mood-chart.tsx": 1166,
@@ -66,7 +66,7 @@
"lib/jobs/coach-nudge.ts": 1084,
"lib/jobs/insight-pregenerate.ts": 1189,
"lib/jobs/medication-intake-import.ts": 819,
"lib/jobs/offhost-backup.ts": 1114,
"lib/jobs/offhost-backup.ts": 1094,
"lib/jobs/reaction-line.ts": 803,
"lib/managed-profiles/handover.ts": 986,
"lib/mcp/record-search.ts": 1467,
+2 -2
View File
@@ -1,7 +1,7 @@
{
"$comment": "as never casts in tests, per test file under src/. Maintained by src/__tests__/test-realism-ratchet.test.ts.",
"count": 524,
"total": 5489,
"total": 5488,
"entries": {
"__tests__/api/auth/login/mfa-branch.test.ts": 10,
"__tests__/api/auth/mfa/verify.test.ts": 9,
@@ -361,7 +361,7 @@
"lib/documents/__tests__/assist-kinds.test.ts": 1,
"lib/documents/__tests__/commit.test.ts": 9,
"lib/documents/__tests__/fenced-chat-pipeline.test.ts": 6,
"lib/documents/__tests__/index-document.test.ts": 23,
"lib/documents/__tests__/index-document.test.ts": 22,
"lib/documents/__tests__/index-outcome-record.test.ts": 14,
"lib/documents/__tests__/prepare-vision-raster-failure.test.ts": 6,
"lib/documents/__tests__/provider-order.test.ts": 1,
@@ -63,6 +63,8 @@ const databaseHealth = {
directory: "/var/lib/healthlog/backups",
keepNightly: 7,
localDirWritable: true,
localPersistent: true,
localReason: "volume",
offhostConfigured: true,
lastGood: {
at: "2026-09-21T23:31:40.000Z",
@@ -4,7 +4,8 @@
* Every backup signal with its verdict, its time and where to fix it: the
* database dump, the weekly copy, the off-host copies, the last restore drill
* (no time window: a failed drill stays failed until a later one passes), the
* off-host setup and the recovery kit. The same judgement the hourly
* off-host setup, whether the bucket ever removes the database dumps, and the
* recovery kit. The same judgement the hourly
* evaluation pushes to the admins (`src/lib/recovery/readiness-model.ts`).
* Cookie-only admin.
*/
@@ -17,6 +17,8 @@ const HEALTHY: DatabaseBackupHealth = {
directory: "/var/lib/healthlog/backups",
keepNightly: 7,
localDirWritable: true,
localPersistent: true,
localReason: "volume",
offhostConfigured: true,
lastGood: {
at: "2026-10-24T23:31:00.000Z",
@@ -77,17 +79,36 @@ describe("<DatabaseDumpRow>", () => {
});
it("says when dumps go off-host only, or cannot be taken at all", () => {
expect(render({ ...HEALTHY, localDirWritable: false }, null)).toContain(
"dumps go off-host only",
const unwritable = {
...HEALTHY,
localDirWritable: false,
localPersistent: false,
localReason: "not_writable" as const,
};
expect(render(unwritable, null)).toContain("dumps go off-host only");
expect(render({ ...unwritable, offhostConfigured: false }, null)).toContain(
"no database dump can be taken",
);
expect(
render(
{ ...HEALTHY, localDirWritable: false, offhostConfigured: false },
null,
),
).toContain("no database dump can be taken");
expect(render({ ...HEALTHY, lastGood: null }, null)).toContain(
"No database dump yet",
);
});
it("does not promise dumps on this server from a directory inside the container", () => {
const layer = {
...HEALTHY,
localPersistent: false,
localReason: "container_layer" as const,
lastGood: { ...HEALTHY.lastGood!, local: false },
};
const html = render(layer);
expect(html).toContain('data-slot="database-dump-setup"');
expect(html).toContain("lost at the next update");
expect(html).toContain("Dumps go off-host only");
expect(html).not.toContain("kept on this server");
expect(html).toContain("removed after 30 days");
const nowhere = render({ ...layer, offhostConfigured: false }, null);
expect(nowhere).toContain("no database dump is kept anywhere");
expect(nowhere).not.toContain('data-slot="database-dump-retention"');
});
});
@@ -6,7 +6,10 @@
* The account copies below it restore one account on this server; the dump is
* the copy that rebuilds the server. So the row leads the card and says three
* things: when the newest good dump was taken and where it is, whether the
* newest run failed, and whether this server can keep a dump at all.
* newest run failed, and whether this server can keep a dump at all. A
* directory inside the container (the image's own, without the volume) is
* writable and still keeps nothing past the next update, so "kept on this
* server" is only said of a directory that survives it.
*/
import { AlertTriangle, HardDrive } from "lucide-react";
@@ -45,7 +48,15 @@ export function DatabaseDumpRow({
let setupProblem: string | null = null;
if (!health.enabled) {
setupProblem = t("admin.section.backups.database.disabled");
} else if (!health.localDirWritable) {
} else if (health.localDirWritable && !health.localPersistent) {
setupProblem = health.offhostConfigured
? t("admin.section.backups.database.notPersistentOffhost", {
dir: health.directory,
})
: t("admin.section.backups.database.notPersistent", {
dir: health.directory,
});
} else if (!health.localPersistent) {
setupProblem = health.offhostConfigured
? t("admin.section.backups.database.noDirectoryOffhost", {
dir: health.directory,
@@ -55,6 +66,19 @@ export function DatabaseDumpRow({
});
}
const retention = [
health.localPersistent
? t("admin.section.backups.database.retentionLocal", {
nights: health.keepNightly,
})
: null,
bucketLifecycle?.state === "configured"
? t("admin.section.backups.database.retentionOffhost", {
days: bucketLifecycle.expirationDays ?? 0,
})
: null,
].filter((line): line is string => line !== null);
return (
<div className="space-y-2" data-slot="database-dump">
<div className="flex items-start gap-2">
@@ -80,19 +104,14 @@ export function DatabaseDumpRow({
</p>
{/* How long a deleted account stays in the dumps: the operator's
side of the deletion promise (GDPR retention of backups). */}
<p
className="text-muted-foreground text-xs"
data-slot="database-dump-retention"
>
{t("admin.section.backups.database.retentionLocal", {
nights: health.keepNightly,
})}
{bucketLifecycle?.state === "configured"
? ` ${t("admin.section.backups.database.retentionOffhost", {
days: bucketLifecycle.expirationDays ?? 0,
})}`
: null}
</p>
{retention.length > 0 ? (
<p
className="text-muted-foreground text-xs"
data-slot="database-dump-retention"
>
{retention.join(" ")}
</p>
) : null}
</div>
</div>
@@ -3,7 +3,8 @@
/**
* `<RecoveryReadinessCard>` — every backup signal on one card: the database
* dump, the weekly account copies, the off-host copies, the last restore
* drill, the off-host setup and the recovery kit. Each row says its verdict,
* drill, the off-host setup, the bucket's rule for the dumps and the recovery
* kit. Each row says its verdict,
* the time it is about and where to fix it.
*
* No time window anywhere: a failed drill stays on this card until a later
+42
View File
@@ -407,6 +407,48 @@ describe("getClientIp against the connection's own peer", () => {
).toBe("203.0.113.6");
});
// The ranges docs/self-hosting/reverse-proxy.md lists, one address each.
// Reserved and multicast stay in: no host on the internet can complete a
// TCP handshake from them, so a peer there is on a network the operator
// runs, like a private one.
it("reads forwarding headers from every range the reverse-proxy page lists", () => {
for (const proxy of [
"127.0.0.2", // loopback
"0.0.0.1", // 0.0.0.0/8
"10.1.2.3", // private
"172.31.0.9",
"192.168.1.20",
"100.64.0.1", // CGNAT (Tailscale)
"169.254.10.10", // link-local
"192.0.0.8", // IETF protocol assignments
"198.18.0.4", // benchmarking
"240.0.0.1", // reserved
"fd7a:115c:a1e0::1", // unique-local (Tailscale)
"fe80::1", // link-local
"fec0::1", // site-local
"::ffff:10.0.0.3", // IPv4-mapped private
]) {
expect(
getClientIp(stamped(proxy, { "x-forwarded-for": "203.0.113.7" })),
proxy,
).toBe("203.0.113.7");
}
for (const outside of ["8.8.8.8", "2001:4860:4860::8888"]) {
expect(
getClientIp(stamped(outside, { "x-forwarded-for": "203.0.113.7" })),
).toBe(outside);
}
});
it("believes cf-connecting-ip from any peer once the flag is on, which is why the origin must take Cloudflare only", () => {
process.env.TRUST_CF_CONNECTING_IP = "1";
expect(
getClientIp(
stamped("198.51.100.7", { "cf-connecting-ip": "203.0.113.9" }),
),
).toBe("203.0.113.9");
});
it("reads an IPv4 peer that the socket reports in IPv6 form", () => {
expect(
getClientIp(
@@ -158,43 +158,67 @@ describe("collectReadiness — the database dump", () => {
BACKUP_S3_SECRET_KEY: "s",
BACKUP_ENCRYPTION_KEY: HEX32,
};
const STATUS = {
volume: { writable: true, persistent: true, reason: "volume" },
layer: { writable: true, persistent: false, reason: "container_layer" },
memory: { writable: true, persistent: false, reason: "memory" },
none: { writable: false, persistent: false, reason: "not_writable" },
} as const;
const dumpLine = (
env: Record<string, string | undefined>,
writable: boolean,
status: keyof typeof STATUS,
) =>
collectReadiness(env, () => writable).lines.find(
collectReadiness(env, () => STATUS[status]).lines.find(
(l) => l.label === "Database dump",
);
it("names the schedule, the directory and the copies kept", () => {
const line = dumpLine(baseEnv(), true);
const line = dumpLine(baseEnv(), "volume");
expect(line?.status).toBe("ok");
expect(line?.detail).toBe(
"nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7",
);
expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain(
expect(dumpLine({ ...baseEnv(), ...offhost }, "volume")?.detail).toContain(
"copied off-host",
);
});
it("warns when nothing can be kept on the server", () => {
expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({
expect(dumpLine({ ...baseEnv(), ...offhost }, "none")).toMatchObject({
status: "warn",
detail: expect.stringContaining("goes off-host only"),
});
expect(dumpLine(baseEnv(), false)).toMatchObject({
expect(dumpLine(baseEnv(), "none")).toMatchObject({
status: "warn",
detail: expect.stringContaining("no database dump can be taken"),
detail: expect.stringContaining("no database dump is kept anywhere"),
});
});
it("does not call a directory inside the container kept: it is writable and lost at the next update", () => {
const offhostOnly = dumpLine({ ...baseEnv(), ...offhost }, "layer");
expect(offhostOnly?.status).toBe("warn");
expect(offhostOnly?.detail).toContain(
"/var/lib/healthlog/backups is inside the container, not on a mounted volume",
);
expect(offhostOnly?.detail).toContain("none is kept on this server");
expect(offhostOnly?.detail).toContain("none is taken before a migration");
expect(offhostOnly?.detail).not.toContain("keeping 7");
expect(dumpLine(baseEnv(), "layer")).toMatchObject({
status: "warn",
detail: expect.stringContaining("no database dump is kept anywhere"),
});
expect(dumpLine(baseEnv(), "memory")?.detail).toContain(
"lost at the next restart",
);
});
it("warns when it is switched off, and names a schedule it ignored", () => {
expect(
dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true),
dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, "volume"),
).toMatchObject({ status: "warn" });
const report = collectReadiness(
{ ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" },
() => true,
() => STATUS.volume,
);
expect(
report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail,
+28 -18
View File
@@ -20,11 +20,12 @@ import {
legacyAnyHostConfigured,
} from "@/lib/ai/local-host-allowlist";
import {
backupDirectoryStatus,
databaseBackupEnabled,
isWritableDirectory,
resolveBackupDir,
resolveBackupSchedule,
resolveDailyKeep,
type BackupDirectoryStatus,
} from "@/lib/database-backup/dump-core.mjs";
export interface ReadinessLine {
@@ -74,7 +75,7 @@ function anySet(env: Env, keys: string[]): boolean {
*/
export function databaseBackupReadiness(
env: Env,
writable: (dir: string) => boolean,
directoryStatus: (dir: string) => BackupDirectoryStatus,
): ReadinessLine[] {
const label = "Database dump";
if (!databaseBackupEnabled(env)) {
@@ -104,35 +105,44 @@ export function databaseBackupReadiness(
"BACKUP_S3_SECRET_KEY",
"BACKUP_ENCRYPTION_KEY",
]);
if (writable(dir)) {
const status = directoryStatus(dir);
if (status.persistent) {
lines.push({
label,
status: "ok",
detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`,
});
} else if (offhost) {
lines.push({
label,
status: "warn",
detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
});
} else {
lines.push({
label,
status: "warn",
detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
});
return lines;
}
// Writable but thrown away with the container (the image's own directory
// without the volume) or at a restart (a tmpfs): no dump is kept there,
// because one that is would be counted as kept and then lost.
const where = !status.writable
? `${dir} is not writable`
: status.reason === "memory"
? `${dir} is in memory (tmpfs), so a dump there is lost at the next restart`
: `${dir} is inside the container, not on a mounted volume, so a dump there is lost at the next update`;
const fix =
"Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR to a mounted directory";
lines.push({
label,
status: "warn",
detail: offhost
? `${where}: the nightly dump goes off-host only, none is kept on this server and none is taken before a migration. ${fix}`
: `${where} and no off-host bucket is configured: no database dump is kept anywhere. ${fix}`,
});
return lines;
}
/**
* Inspect the environment and build the readiness report. Pure apart from
* `writable`, the probe of the dump directory, which tests replace.
* `directoryStatus`, the probe of the dump directory, which tests replace.
*/
export function collectReadiness(
env: Env = process.env,
writable: (dir: string) => boolean = isWritableDirectory,
directoryStatus: (
dir: string,
) => BackupDirectoryStatus = backupDirectoryStatus,
): ReadinessReport {
const lines: ReadinessLine[] = [];
@@ -211,7 +221,7 @@ export function collectReadiness(
});
}
lines.push(...databaseBackupReadiness(env, writable));
lines.push(...databaseBackupReadiness(env, directoryStatus));
// --- Optional subsystems (informational) --------------------------------
const optional: Array<[string, boolean]> = [
@@ -25,6 +25,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import {
applyRetention,
backupDirectoryStatus,
bucketConfigFromEnv,
pickBucketDump,
buildManifest,
@@ -37,13 +38,17 @@ import {
formatDumpTimestamp,
keyFingerprints,
listDumpSets,
mountContaining,
offhostObjectKey,
parseMountInfo,
parseDumpFileName,
parseManifest,
resolveBackupSchedule,
resolveDailyKeep,
selectForDeletion,
type DirectoryProbe,
type DumpSet,
type Env,
} from "../dump-core.mjs";
import type { S3Like } from "@/lib/jobs/offhost-backup";
@@ -687,3 +692,168 @@ describe("finding a dump in the bucket", () => {
).toBeNull();
});
});
describe("whether the dump directory survives the container", () => {
// /proc/self/mountinfo of the app container under the bundled compose
// file, trimmed to the lines that matter, with and without the volume.
const ROOT =
"1470 1351 0:330 / / rw,relatime master:396 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/A:/var/lib/docker/overlay2/l/B,upperdir=/var/lib/docker/overlay2/x/diff,workdir=/var/lib/docker/overlay2/x/work";
const COMMON = [
"1471 1470 0:333 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw",
"1472 1470 0:334 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755",
"1481 1470 8:1 /var/lib/docker/containers/c/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/sda1 rw",
"1483 1470 8:1 /var/lib/docker/containers/c/hosts /etc/hosts rw,relatime - ext4 /dev/sda1 rw",
];
const VOLUME =
"1480 1470 8:1 /var/lib/docker/volumes/healthlog_backups/_data /var/lib/healthlog/backups rw,relatime master:1 - ext4 /dev/sda1 rw,discard";
const DIR = "/var/lib/healthlog/backups";
function probe(
over: Partial<DirectoryProbe> & { files?: Record<string, string> } = {},
): DirectoryProbe {
const { files = {}, ...rest } = over;
return {
writable: () => true,
realpath: (p) => p,
readText: (p) => files[p] ?? null,
exists: (p) => p in files,
device: () => null,
env: {},
...rest,
};
}
const inDocker = (...lines: string[]) =>
probe({
files: {
"/.dockerenv": "",
"/proc/self/mountinfo": [ROOT, ...COMMON, ...lines].join("\n"),
},
});
it("reads mount points and types, escapes undone, odd lines skipped", () => {
const mounts = parseMountInfo(
[
ROOT,
"90 1470 8:1 / /mnt/my\\040disk rw - ext4 /dev/sdb1 rw",
"not a mount line",
"",
].join("\n"),
);
expect(mounts).toEqual([
{ mountPoint: "/", fsType: "overlay" },
{ mountPoint: "/mnt/my disk", fsType: "ext4" },
]);
});
it("finds the longest mount that holds a path, by whole segments", () => {
const mounts = [
{ mountPoint: "/", fsType: "overlay" },
{ mountPoint: "/var/lib/healthlog/back", fsType: "ext4" },
{ mountPoint: "/var/lib/healthlog", fsType: "xfs" },
];
expect(mountContaining(DIR, mounts)?.mountPoint).toBe("/var/lib/healthlog");
expect(mountContaining("/var/lib/other", mounts)?.mountPoint).toBe("/");
expect(mountContaining("/var/lib/healthlog", mounts)?.fsType).toBe("xfs");
});
it("counts the backups volume as kept", () => {
expect(backupDirectoryStatus(DIR, inDocker(VOLUME))).toEqual({
writable: true,
persistent: true,
reason: "volume",
});
});
it("does not count the image's own directory without the volume, though it is writable", () => {
expect(backupDirectoryStatus(DIR, inDocker())).toEqual({
writable: true,
persistent: false,
reason: "container_layer",
});
});
it("counts a volume on a parent directory, and follows a symlink to where the files land", () => {
const parent =
"1480 1470 8:1 /srv/healthlog /var/lib/healthlog rw - ext4 /dev/sda1 rw";
expect(backupDirectoryStatus(DIR, inDocker(parent)).persistent).toBe(true);
const linked = {
...inDocker(VOLUME),
realpath: (p: string) => (p === "/backups" ? DIR : p),
};
expect(backupDirectoryStatus("/backups", linked).reason).toBe("volume");
expect(backupDirectoryStatus("/elsewhere", inDocker(VOLUME)).reason).toBe(
"container_layer",
);
});
it("does not count a tmpfs as kept", () => {
const tmpfs =
"1480 1470 0:400 / /var/lib/healthlog/backups rw - tmpfs tmpfs rw,size=1g";
expect(backupDirectoryStatus(DIR, inDocker(tmpfs))).toEqual({
writable: true,
persistent: false,
reason: "memory",
});
});
it("recognises a container by Podman's marker, Kubernetes, the cgroup or an overlay root", () => {
const table = [ROOT, ...COMMON].join("\n");
const cases: Array<{ files?: Record<string, string>; env?: Env }> = [
{ files: { "/run/.containerenv": "" } },
{ env: { KUBERNETES_SERVICE_HOST: "10.0.0.1" } },
{ files: { "/proc/self/cgroup": "12:pids:/docker/0123abcd\n" } },
{},
];
for (const extra of cases) {
const p = probe({
env: extra.env ?? {},
files: { ...extra.files, "/proc/self/mountinfo": table },
});
expect(backupDirectoryStatus(DIR, p).reason).toBe("container_layer");
}
});
it("treats a writable directory outside a container as kept: a host, a VM", () => {
const host = probe({
files: {
"/proc/self/cgroup": "0::/user.slice/user-1000.slice\n",
"/proc/self/mountinfo":
"22 1 259:2 / / rw,relatime shared:1 - ext4 /dev/nvme0n1p2 rw",
},
});
expect(backupDirectoryStatus(DIR, host)).toEqual({
writable: true,
persistent: true,
reason: "host",
});
// No /proc at all (macOS, a BSD): a host too.
expect(backupDirectoryStatus(DIR, probe()).reason).toBe("host");
});
it("falls back to the device numbers in a container without a mount table", () => {
const same = probe({ files: { "/.dockerenv": "" }, device: () => 42 });
expect(backupDirectoryStatus(DIR, same).reason).toBe("container_layer");
const apart = probe({
files: { "/.dockerenv": "" },
device: (p) => (p === "/" ? 42 : 7),
});
expect(backupDirectoryStatus(DIR, apart).reason).toBe("volume");
});
it("says when nothing can be written at all, before anything else", () => {
expect(
backupDirectoryStatus(DIR, {
...inDocker(VOLUME),
writable: () => false,
}),
).toEqual({ writable: false, persistent: false, reason: "not_writable" });
});
it("answers for a real directory on this machine without throwing", () => {
const status = backupDirectoryStatus(tempDir());
expect(status.writable).toBe(true);
expect(["host", "volume", "container_layer", "memory"]).toContain(
status.reason,
);
});
});
+190
View File
@@ -33,10 +33,13 @@ import {
constants as fsConstants,
createReadStream,
createWriteStream,
existsSync,
mkdirSync,
openSync,
readFileSync,
readSync,
readdirSync,
realpathSync,
statSync,
} from "node:fs";
import { readFile, rename, rm, stat, writeFile } from "node:fs/promises";
@@ -222,6 +225,193 @@ export function isWritableDirectory(dir) {
}
}
// ── Whether a dump kept there survives ─────────────────────────────────────
/**
* File systems that live in memory: a dump on one is gone at the next restart.
*/
const EPHEMERAL_FS_TYPES = new Set(["tmpfs", "ramfs"]);
/**
* cgroup paths that name a container runtime (cgroup v1; under v2 a container
* sees `0::/`, and the marker files or the root mount answer instead).
*/
const CONTAINER_CGROUP = /\/(docker|kubepods|containerd|libpod|podman)[/-]/;
/**
* @typedef {{ mountPoint: string; fsType: string }} MountEntry
*/
/** Undo the octal escapes the kernel writes for space, tab, newline, `\`. */
function unescapeMountPath(raw) {
return raw.replace(/\\([0-7]{3})/g, (_, oct) =>
String.fromCharCode(Number.parseInt(oct, 8)),
);
}
/**
* The mount points and file system types in `/proc/self/mountinfo` text.
* Lines that do not have the documented shape are skipped.
*
* @param {string} text
* @returns {MountEntry[]}
*/
export function parseMountInfo(text) {
/** @type {MountEntry[]} */
const mounts = [];
for (const line of text.split("\n")) {
const sep = line.indexOf(" - ");
if (sep < 0) continue;
const left = line.slice(0, sep).split(" ");
const right = line.slice(sep + 3).split(" ");
if (left.length < 6 || !right[0]) continue;
mounts.push({ mountPoint: unescapeMountPath(left[4]), fsType: right[0] });
}
return mounts;
}
/**
* The mount `path` lives on: the longest mount point that is `path` or one of
* its parents. A later entry for the same point wins, as it covers the earlier
* one. Null when nothing matches (no `/` in the list).
*
* @param {string} path an absolute path, symlinks resolved
* @param {MountEntry[]} mounts
* @returns {MountEntry | null}
*/
export function mountContaining(path, mounts) {
/** @type {MountEntry | null} */
let best = null;
for (const mount of mounts) {
const point = mount.mountPoint;
const covers =
point === "/" ||
path === point ||
path.startsWith(point.endsWith("/") ? point : `${point}/`);
if (covers && (best === null || point.length >= best.mountPoint.length)) {
best = mount;
}
}
return best;
}
/**
* @typedef {{
* writable: (dir: string) => boolean;
* realpath: (path: string) => string;
* readText: (path: string) => string | null;
* exists: (path: string) => boolean;
* device: (path: string) => number | null;
* env: Env;
* }} DirectoryProbe
*/
/** @type {DirectoryProbe} */
const SYSTEM_PROBE = {
writable: isWritableDirectory,
realpath: (path) => {
try {
return realpathSync(path);
} catch {
return path;
}
},
readText: (path) => {
try {
return readFileSync(path, "utf8");
} catch {
return null;
}
},
exists: (path) => existsSync(path),
device: (path) => {
try {
return statSync(path).dev;
} catch {
return null;
}
},
env: process.env,
};
/**
* Whether this process runs in a container whose root file system is thrown
* away when the container is replaced: Docker (`/.dockerenv`), Podman
* (`/run/.containerenv`), Kubernetes, a container cgroup, or a root file
* system that is an overlay. A host, a VM or a system container with a disk
* of its own answers false.
*
* @param {DirectoryProbe} probe
* @param {MountEntry[] | null} mounts
*/
export function runsInContainer(probe, mounts) {
if (probe.exists("/.dockerenv") || probe.exists("/run/.containerenv")) {
return true;
}
if ((probe.env.KUBERNETES_SERVICE_HOST ?? "").trim()) return true;
if (CONTAINER_CGROUP.test(probe.readText("/proc/self/cgroup") ?? "")) {
return true;
}
const root = mounts ? mountContaining("/", mounts) : null;
return root?.mountPoint === "/" && root.fsType === "overlay";
}
/**
* @typedef {{
* writable: boolean;
* persistent: boolean;
* reason: "volume" | "host" | "container_layer" | "memory" | "not_writable";
* }} BackupDirectoryStatus
*/
/**
* Whether a dump written to `dir` is still there after the container is
* replaced, which is what "kept on this server" promises.
*
* volume — in a container, on a mount of its own (the `backups`
* volume, a bind mount): it survives.
* host — not in a container: a writable directory is a disk.
* container_layer — in a container, on its root file system: an update
* replaces the container and the dump with it. The image
* creates the default directory, so without the volume it
* is writable and looks fine; this is the case to catch.
* memory — on a tmpfs: gone at the next restart.
* not_writable — nothing can be written there at all.
*
* Never throws. `probe` is the file system; tests hand in their own.
*
* @param {string} dir
* @param {DirectoryProbe} [probe]
* @returns {BackupDirectoryStatus}
*/
export function backupDirectoryStatus(dir, probe = SYSTEM_PROBE) {
if (!probe.writable(dir)) {
return { writable: false, persistent: false, reason: "not_writable" };
}
const text = probe.readText("/proc/self/mountinfo");
const mounts = text === null ? null : parseMountInfo(text);
if (!runsInContainer(probe, mounts)) {
return { writable: true, persistent: true, reason: "host" };
}
const real = probe.realpath(dir);
const mount = mounts ? mountContaining(real, mounts) : null;
if (mount) {
if (EPHEMERAL_FS_TYPES.has(mount.fsType)) {
return { writable: true, persistent: false, reason: "memory" };
}
return mount.mountPoint === "/"
? { writable: true, persistent: false, reason: "container_layer" }
: { writable: true, persistent: true, reason: "volume" };
}
// No mount table to read: a directory on the root's own device is in the
// container's layer.
const own = probe.device(real);
const root = probe.device("/");
return own !== null && own === root
? { writable: true, persistent: false, reason: "container_layer" }
: { writable: true, persistent: true, reason: "volume" };
}
// ── Keys ───────────────────────────────────────────────────────────────────
/**
+32 -10
View File
@@ -3,8 +3,8 @@
* alert can act on.
*
* Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of
* this process: whether `DB_BACKUP_DIR` is writable here, and whether the
* off-host bucket is configured. A ledger row that says a copy went off-host
* this process: whether `DB_BACKUP_DIR` is writable here and survives the
* container being replaced, and whether the off-host bucket is configured. A ledger row that says a copy went off-host
* outranks this process's environment, for the reason the off-host card gives
* (a split install can hand the bucket variables to the worker only).
*/
@@ -13,10 +13,11 @@ import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
import {
backupDirectoryStatus,
databaseBackupEnabled,
isWritableDirectory,
resolveBackupDir,
resolveDailyKeep,
type BackupDirectoryStatus,
} from "./dump-core.mjs";
export interface DatabaseBackupHealth {
@@ -30,8 +31,20 @@ export interface DatabaseBackupHealth {
* page says.
*/
keepNightly: number;
/** This process can write there. False means dumps only go off-host. */
/** This process can write there. */
localDirWritable: boolean;
/**
* A dump written there survives an update: the directory is on a volume or
* a bind mount, or the app does not run in a container. False means no dump
* is kept on this server, whatever an older ledger row says, and dumps only
* go off-host. `localReason` says why.
*/
localPersistent: boolean;
/**
* `volume`, `host`, `container_layer` (the image's own directory, without
* the volume mounted), `memory` (a tmpfs) or `not_writable`.
*/
localReason: BackupDirectoryStatus["reason"];
/** The off-host bucket is configured, or a dump has been uploaded to it. */
offhostConfigured: boolean;
/** The newest dump that was kept on the server or reached the bucket. */
@@ -55,15 +68,22 @@ const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000;
export async function getDatabaseBackupHealth(
now: Date = new Date(),
env: NodeJS.ProcessEnv = process.env,
directoryStatus: (
dir: string,
) => BackupDirectoryStatus = backupDirectoryStatus,
): Promise<DatabaseBackupHealth> {
const directory = resolveBackupDir(env);
const local = directoryStatus(directory);
// A dump recorded as written into a directory that does not survive the
// container was lost with the container that wrote it: only a copy in the
// bucket still counts.
const keptWhere = local.persistent
? [{ localResult: "written" }, { offhostResult: "uploaded" }]
: [{ offhostResult: "uploaded" }];
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
[
prisma.databaseBackupRun.findFirst({
where: {
fileName: { not: null },
OR: [{ localResult: "written" }, { offhostResult: "uploaded" }],
},
where: { fileName: { not: null }, OR: keptWhere },
orderBy: { startedAt: "desc" },
select: {
startedAt: true,
@@ -108,7 +128,9 @@ export async function getDatabaseBackupHealth(
enabled: databaseBackupEnabled(env),
directory,
keepNightly: resolveDailyKeep(env),
localDirWritable: isWritableDirectory(directory),
localDirWritable: local.writable,
localPersistent: local.persistent,
localReason: local.reason,
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
lastGood: lastGood
? {
@@ -116,7 +138,7 @@ export async function getDatabaseBackupHealth(
fileName: lastGood.fileName ?? "",
sizeBytes:
lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes),
local: lastGood.localResult === "written",
local: local.persistent && lastGood.localResult === "written",
offhost: lastGood.offhostResult === "uploaded",
}
: null,
+28 -14
View File
@@ -9,11 +9,14 @@
* under `database/` in the same envelope the per-account copies use, sealed
* with `BACKUP_ENCRYPTION_KEY`.
*
* Without a writable directory (an install that runs the image without the
* volume) the dump is written to a temporary directory, uploaded and removed:
* the off-host copy still happens, nothing is kept on the server, and the
* ledger and the boot summary say so. With neither, the run is skipped and
* recorded as such.
* Without a directory that survives the container (an install that runs the
* image without the volume: the image's own directory is writable but sits in
* the container's layer, which the next update throws away) the dump is
* written to a temporary directory, uploaded and removed: the off-host copy
* still happens, nothing is kept on the server, and the ledger and the boot
* summary say so. A dump left in the container's layer would be counted as
* kept and lost with the container, so none is. With neither, the run is
* skipped and recorded as such.
*
* Every run leaves one `DatabaseBackupRun` row: started, then finished with
* what happened on each leg. `getDatabaseBackupHealth` reads them.
@@ -35,19 +38,20 @@ import packageJson from "../../../package.json";
import {
applyRetention,
backupDirectoryStatus,
createDatabaseDump,
databaseBackupEnabled,
isWritableDirectory,
offhostObjectKey,
resolveBackupDir,
resolveDailyKeep,
type BackupDirectoryStatus,
} from "./dump-core.mjs";
/** How long ledger rows are kept. A run a day; a year and a bit of history. */
const LEDGER_RETENTION_DAYS = 400;
export type DatabaseBackupLocalResult =
"written" | "no_directory" | "failed" | "skipped";
"written" | "no_directory" | "not_persistent" | "failed" | "skipped";
export type DatabaseBackupOffhostResult =
"uploaded" | "not_configured" | "failed" | "skipped";
@@ -69,6 +73,8 @@ export interface RunDatabaseBackupOptions {
env?: NodeJS.ProcessEnv;
/** A bucket double for tests; the configured bucket otherwise. */
s3?: S3Like;
/** Where `DB_BACKUP_DIR` stands; the file system's answer otherwise. */
directoryStatus?: (dir: string) => BackupDirectoryStatus;
}
/** The running build's version, as `/api/version` answers it. */
@@ -111,7 +117,9 @@ export async function runDatabaseBackup(
const env = options.env ?? process.env;
const now = options.now ?? new Date();
const dir = resolveBackupDir(env);
const localWritable = isWritableDirectory(dir);
const directory = (options.directoryStatus ?? backupDirectoryStatus)(dir);
// Kept here only where it survives the container being replaced.
const keepLocal = directory.persistent;
const offhostConfig = (() => {
try {
return loadOffhostConfig();
@@ -128,7 +136,11 @@ export async function runDatabaseBackup(
status: "failed",
fileName: null,
dumpBytes: null,
local: localWritable ? "failed" : "no_directory",
local: keepLocal
? "failed"
: directory.writable
? "not_persistent"
: "no_directory",
offhost: offhostConfig ? "failed" : "not_configured",
removed: 0,
error: null,
@@ -150,11 +162,13 @@ export async function runDatabaseBackup(
report.local = "skipped";
report.offhost = "skipped";
report.error = "DB_BACKUP_ENABLED is false";
} else if (!localWritable && !offhostConfig) {
} else if (!keepLocal && !offhostConfig) {
report.status = "skipped";
report.error = `no writable backup directory (${dir}) and no off-host bucket configured`;
report.error = directory.writable
? `the backup directory (${dir}) is not on persistent storage, so a dump there would be lost with the container, and no off-host bucket is configured`
: `no writable backup directory (${dir}) and no off-host bucket configured`;
} else {
const target = localWritable
const target = keepLocal
? dir
: (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-")));
const dump = await createDatabaseDump({
@@ -167,7 +181,7 @@ export async function runDatabaseBackup(
buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null,
deadline: options.deadline,
});
if (localWritable) report.local = "written";
if (keepLocal) report.local = "written";
report.fileName = basename(dump.dumpPath);
report.dumpBytes = dump.manifest.dump.bytes;
@@ -188,7 +202,7 @@ export async function runDatabaseBackup(
}
}
if (localWritable) {
if (keepLocal) {
const removed = await applyRetention(dir, {
dailyKeep: resolveDailyKeep(env),
now: now.getTime(),
@@ -115,7 +115,7 @@ beforeEach(() => {
ok: true,
text: "glucose fasting creatinine values report",
source: "local-pdf",
} as never);
});
// Default: the auto-read toggle is ON (so the external-pick provider path is
// exercised); the OFF-specific tests override this explicitly.
vi.mocked(documentAutoReadEnabled).mockResolvedValue(true);
@@ -284,12 +284,44 @@ describe("indexDocumentContent — local fallback path", () => {
vi.mocked(localExtractText).mockResolvedValue({
ok: false,
reason: "empty",
} as never);
});
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toEqual({ indexed: false, reason: "local-empty" });
expect(upsertContentIndex).not.toHaveBeenCalled();
});
it("reports a PDF past the reader's limits as resource-limit, not as a file that would not open", async () => {
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({
chain: [],
pick: null,
localOcrEnabled: false,
withheld: null,
});
vi.mocked(localExtractText).mockResolvedValue({
ok: false,
reason: "resource-limit",
});
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toEqual({ indexed: false, reason: "resource-limit" });
expect(upsertContentIndex).not.toHaveBeenCalled();
});
it("keeps resource-limit over a provider note: the provider never got further either", async () => {
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never);
vi.mocked(prepareVisionInput).mockResolvedValue({
ok: false,
reason: "rasterFailed",
});
vi.mocked(localExtractText).mockResolvedValue({
ok: false,
reason: "resource-limit",
});
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toEqual({ indexed: false, reason: "resource-limit" });
});
});
describe("indexDocumentContent — the documentAi capability", () => {
@@ -0,0 +1,53 @@
import { describe, expect, it, vi } from "vitest";
/**
* A PDF that crosses the reader's memory or time bound is reported as such
* (`resource-limit`), not as a read error, which the index job records as a
* stored file that would not open. The bound itself is the isolated reader's
* (`../pdf-isolation.ts`); here it is the error it throws.
*/
const { annotate, logCaught, readPdfText } = vi.hoisted(() => ({
annotate: vi.fn(),
logCaught: vi.fn(),
readPdfText: vi.fn(),
}));
vi.mock("@/lib/logging/context", () => ({ annotate }));
vi.mock("@/lib/logging/signal", () => ({ logCaught }));
vi.mock("../pdfjs", async (importOriginal) => ({
...(await importOriginal<typeof import("../pdfjs")>()),
readPdfText,
}));
import { PdfResourceLimitError } from "../pdfjs";
import { extractPdfText, localExtractText } from "../local-extract";
describe("a PDF past the reader's limits", () => {
it.each(["memory", "time", "exit"] as const)(
"answers resource-limit for the %s bound, without the error log a broken file gets",
async (limit) => {
readPdfText.mockRejectedValueOnce(new PdfResourceLimitError(limit));
annotate.mockClear();
logCaught.mockClear();
const bytes = Buffer.from("%PDF-1.7 large");
expect(await extractPdfText(bytes)).toEqual({
ok: false,
reason: "resource-limit",
});
expect(logCaught).not.toHaveBeenCalled();
expect(annotate).toHaveBeenCalledWith({
action: { name: "documents.localIndex.resourceLimit" },
meta: { limit, bytes: bytes.length },
});
},
);
it("still answers error for anything else the reader throws", async () => {
const broken = new Error("Invalid PDF structure");
broken.name = "InvalidPDFException";
readPdfText.mockRejectedValueOnce(broken);
expect(
await localExtractText(Buffer.from("%PDF-1.7"), "application/pdf"),
).toEqual({ ok: false, reason: "error" });
});
});
+7
View File
@@ -78,6 +78,7 @@ export type IndexFailureReason =
| "local-empty"
| "local-unsupported"
| "decrypt-error"
| "resource-limit"
| "raster-failed"
| "provider-error"
| "pdf-needs-anthropic"
@@ -342,6 +343,12 @@ async function tryLocalIndex(
if (result.reason === "unsupported") {
return { indexed: false, reason: "local-unsupported" };
}
if (result.reason === "resource-limit") {
// Too large or too complex for the reader's bounds: said as such, and
// final like the thumbnail's refusal of the same bytes. Nothing retries
// it on its own; a person can still ask for a read.
return { indexed: false, reason: "resource-limit" };
}
if (result.reason === "error") {
return { indexed: false, reason: "decrypt-error" };
}
+20 -3
View File
@@ -32,7 +32,7 @@ import { Buffer } from "node:buffer";
import { annotate } from "@/lib/logging/context";
import { logCaught } from "@/lib/logging/signal";
import { readPdfText } from "./pdfjs";
import { PdfResourceLimitError, readPdfText } from "./pdfjs";
/**
* Minimum joined text length for a PDF to count as a real text-layer document.
@@ -54,15 +54,23 @@ export type LocalExtractSource = "local-pdf" | "local-ocr";
* a provider vision pass is the only way to read it.
* - `unsupported` → a MIME the local path cannot read without OCR (images) or
* at all; leave un-indexed locally.
* - `resource-limit` → the PDF crossed the reader's memory or time bound
* (`PdfResourceLimitError`): too large or too complex to
* read here. The same bytes cross it again, so it is a
* final answer, as it is for thumbnails.
* - `error` → the file was malformed / unreadable; treat as un-indexed.
*/
export type LocalExtractResult =
| { ok: true; text: string; source: LocalExtractSource }
| { ok: false; reason: "empty" | "unsupported" | "error" };
| {
ok: false;
reason: "empty" | "unsupported" | "resource-limit" | "error";
};
/**
* Extract a text-layer PDF's embedded text server-side. Returns `empty` for a
* scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never
* scanned/no-text-layer PDF, `resource-limit` for one past the reader's memory
* or time bound, and `error` for a malformed/encrypted one — never
* throws, so a bad document can never abort the caller (the upload/backfill job
* just leaves it un-indexed). An `error` leaves a `warn` line naming what
* pdf.js threw, because the outcome alone does not say why a document stayed
@@ -76,6 +84,15 @@ export async function extractPdfText(
// Bound the page walk so a pathological many-page PDF costs no more.
text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim();
} catch (err) {
if (err instanceof PdfResourceLimitError) {
// Not a fault of the file's bytes being unreadable: they are readable,
// only not within the bounds this server gives one document.
annotate({
action: { name: "documents.localIndex.resourceLimit" },
meta: { limit: err.limit, bytes: buffer.length },
});
return { ok: false, reason: "resource-limit" };
}
const errorType = err instanceof Error ? err.name || "Error" : typeof err;
annotate({
action: { name: "documents.localIndex.extractFailed" },
+1
View File
@@ -300,6 +300,7 @@ const inboundDocument = z
"local-empty",
"local-unsupported",
"decrypt-error",
"resource-limit",
"raster-failed",
"provider-error",
"pdf-needs-anthropic",
@@ -12,6 +12,7 @@ import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
import {
DUMP_FAILING_AFTER_HOURS,
evaluateDatabaseDump,
evaluateDumpRetention,
evaluateOffhostCopies,
evaluateOffhostSetup,
evaluateRecoveryKit,
@@ -45,11 +46,15 @@ function dump(
lastFailureReason,
...rest
} = over;
// A directory that cannot be written keeps nothing either.
const writable = rest.localDirWritable ?? true;
return {
enabled: true,
directory: "/data/db-backups",
keepNightly: 7,
localDirWritable: true,
localPersistent: writable,
localReason: writable ? "volume" : "not_writable",
offhostConfigured: false,
lastGood: lastGoodAt
? {
@@ -79,6 +84,7 @@ function inputs(over: Partial<ReadinessInputs> = {}): ReadinessInputs {
weekly: { accounts: [{ createdAt: ago(60 * DAY), lastCopyAt: ago(DAY) }] },
offhost: {
configured: true,
databaseLifecycle: { state: "configured", expirationDays: 30 },
accounts: [
{ lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) },
],
@@ -172,6 +178,37 @@ describe("database dump", () => {
).toBe("ok");
});
it("does not count a writable directory inside the container as a place to keep dumps", () => {
const layer = {
localDirWritable: true,
localPersistent: false,
localReason: "container_layer" as const,
};
// No bucket: the dumps written there are lost at the next update.
expect(evalDump(dump(layer))).toMatchObject({
state: "failing",
reason: "not_persistent",
params: { directory: "/data/db-backups" },
});
// A bucket: fine, and the tile says the bucket holds the only copy.
expect(evalDump(dump({ ...layer, offhostConfigured: true }))).toMatchObject(
{
state: "ok",
reason: "fresh_offhost_only",
params: { directory: "/data/db-backups" },
},
);
// The same directory on a volume reads as before.
expect(evalDump(dump({ offhostConfigured: true }))).toMatchObject({
state: "ok",
reason: "fresh",
});
// Switched off with nowhere lasting to write: not news.
expect(
evalDump(dump({ ...layer, enabled: false, lastGoodAt: null })),
).toMatchObject({ state: "inactive", reason: "disabled" });
});
it("gives a newly switched-on dump 36 hours before calling it failing", () => {
const fresh = dump({ lastGoodAt: null });
expect(evalDump(fresh, ago(HOUR))).toMatchObject({
@@ -192,6 +229,76 @@ describe("database dump", () => {
});
});
describe("dump retention in the bucket", () => {
const goingOffhost = dump({ offhostConfigured: true });
const evalRule = (
databaseLifecycle: ReadinessInputs["offhost"]["databaseLifecycle"],
d: ReadinessInputs["dump"] = goingOffhost,
configured = true,
) =>
evaluateDumpRetention({
dump: d,
offhost: { configured, databaseLifecycle, accounts: [] },
});
it("asks for a rule when dumps go to a bucket that never removes them", () => {
expect(evalRule({ state: "missing", expirationDays: null })).toMatchObject({
state: "attention",
reason: "missing",
fixPath: "/admin/backups",
});
});
it("names the expiry when a rule covers database/, and stays quiet when the bucket will not say", () => {
expect(evalRule({ state: "configured", expirationDays: 30 })).toMatchObject(
{ state: "ok", reason: "configured", params: { days: 30 } },
);
expect(evalRule({ state: "unknown", expirationDays: null })).toMatchObject({
state: "ok",
reason: "unknown",
});
expect(evalRule(null)).toMatchObject({ state: "ok", reason: "unknown" });
});
it("does not apply without a bucket, or while no dump goes to it", () => {
const missing = { state: "missing", expirationDays: null } as const;
expect(evalRule(missing, goingOffhost, false)).toMatchObject({
state: "inactive",
reason: "not_configured",
});
for (const d of [
null,
"unreadable" as const,
dump({ offhostConfigured: false }),
dump({ offhostConfigured: true, enabled: false }),
]) {
expect(evalRule(missing, d)).toMatchObject({
state: "inactive",
reason: "no_offhost_dumps",
});
}
});
it("moves the overall verdict to attention on its own", () => {
const r = evaluateRecoveryReadiness(
inputs({
dump: goingOffhost,
offhost: {
configured: true,
databaseLifecycle: { state: "missing", expirationDays: null },
accounts: [
{ lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) },
],
},
}),
);
expect(
r.signals.filter((x) => x.state !== "ok" && x.state !== "inactive"),
).toEqual([expect.objectContaining({ id: "dump_retention" })]);
expect(r.state).toBe("attention");
});
});
describe("weekly copy", () => {
const weekly = (
...accounts: Array<{ createdAt: Date; lastCopyAt: Date | null }>
@@ -254,7 +361,11 @@ describe("off-host copies", () => {
lastAttemptAt: Date | null;
lastSuccessAt: Date | null;
}>
) => evaluateOffhostCopies({ now: NOW, offhost: { configured, accounts } });
) =>
evaluateOffhostCopies({
now: NOW,
offhost: { configured, databaseLifecycle: null, accounts },
});
it("does not apply while no bucket is configured", () => {
expect(offhost(false).state).toBe("inactive");
@@ -302,7 +413,10 @@ describe("off-host copies", () => {
describe("restore drill", () => {
const drill = (d: ReadinessInputs["drill"], configured = true) =>
evaluateRestoreDrill({ offhost: { configured, accounts: [] }, drill: d });
evaluateRestoreDrill({
offhost: { configured, databaseLifecycle: null, accounts: [] },
drill: d,
});
const run = {
lastRunAt: ago(5 * DAY),
lastSuccessAt: null as Date | null,
@@ -351,7 +465,7 @@ describe("off-host setup", () => {
evaluateOffhostSetup({
now: NOW,
firstAdminAt,
offhost: { configured, accounts: [] },
offhost: { configured, databaseLifecycle: null, accounts: [] },
});
it("asks for a bucket 7 days after the first admin exists", () => {
+70 -10
View File
@@ -7,8 +7,11 @@
* ordinary row. A failed monthly drill was on the failing-jobs card three days
* in thirty. An off-host bucket that was never configured showed only on the
* backups page, and the key step confirmed one key of the several a restore
* needs. This module turns them into six signals with one verdict each, which
* the readiness tile shows and the hourly evaluation pushes to every admin.
* needs. A bucket whose lifecycle rule left `database/` out kept every dump,
* and every deleted account inside it, forever, and only the backups page
* said so. This module turns them into seven signals with one verdict each,
* which the readiness tile shows and the hourly evaluation pushes to every
* admin.
*
* Pure: the inputs are read elsewhere (`readiness.ts`), so each threshold is a
* unit test away and the tile and the push cannot disagree about a verdict.
@@ -26,6 +29,7 @@ import {
classifyOffhostBackup,
type OffhostBackupFreshness,
} from "@/lib/jobs/offhost-backup-freshness";
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
import type { KitEntry } from "@/lib/recovery/kit-rules";
@@ -50,6 +54,7 @@ export const READINESS_SIGNALS = [
"offhost_copies",
"restore_drill",
"offhost_setup",
"dump_retention",
"recovery_kit",
] as const;
export type ReadinessSignalId = (typeof READINESS_SIGNALS)[number];
@@ -69,6 +74,7 @@ export const READINESS_FIX_PATHS: Record<ReadinessSignalId, string> = {
offhost_copies: "/admin/backups",
restore_drill: "/admin/backups",
offhost_setup: "/admin/backups",
dump_retention: "/admin/backups",
recovery_kit: "/admin/encryption",
};
@@ -121,6 +127,12 @@ export interface ReadinessInputs {
};
offhost: {
configured: boolean;
/**
* The bucket's lifecycle rules for keys under `database/`, as the backups
* page reads them (`probeOffhostLifecycle`). Null when no bucket is
* configured.
*/
databaseLifecycle: LifecycleVerdict | null;
accounts: ReadonlyArray<{
lastAttemptAt: Date | null;
lastSuccessAt: Date | null;
@@ -185,19 +197,24 @@ export function evaluateDatabaseDump(
// directory, no bucket) switched off what it could not do, which is not
// news.
const neverUsed = dump.lastGood === null && dump.lastFailure === null;
const cannotWrite = !dump.localDirWritable && !dump.offhostConfigured;
const cannotWrite = !dump.localPersistent && !dump.offhostConfigured;
return neverUsed && cannotWrite
? signal(id, "inactive", "disabled", null)
: signal(id, "attention", "disabled", lastGoodAt);
}
const lastFailureAt = dump.lastFailure ? new Date(dump.lastFailure.at) : null;
const failureReason = dump.lastFailure?.error ?? null;
// A directory this process cannot write is a valid setup when the dumps go
// to the bucket; with no bucket either, no dump can be kept anywhere.
if (!dump.localDirWritable && !dump.offhostConfigured) {
return signal(id, "failing", "nowhere_to_write", lastGoodAt, {
directory: dump.directory,
});
// A directory that keeps nothing (not writable, or writable inside the
// container and lost at the next update) is a valid setup when the dumps
// go to the bucket; with no bucket either, no dump is kept anywhere.
if (!dump.localPersistent && !dump.offhostConfigured) {
return signal(
id,
"failing",
dump.localDirWritable ? "not_persistent" : "nowhere_to_write",
lastGoodAt,
{ directory: dump.directory },
);
}
const limitMs = DUMP_FAILING_AFTER_HOURS * HOUR_MS;
const failedSinceGood =
@@ -242,7 +259,13 @@ export function evaluateDatabaseDump(
failureReason,
);
}
return signal(id, "ok", "fresh", lastGoodAt);
// Fine, and worth saying: the bucket holds the only copy, and no dump is
// taken before a migration.
return dump.localPersistent
? signal(id, "ok", "fresh", lastGoodAt)
: signal(id, "ok", "fresh_offhost_only", lastGoodAt, {
directory: dump.directory,
});
}
/**
@@ -374,6 +397,42 @@ export function evaluateOffhostSetup(
: signal(id, "ok", "not_configured_grace", graceEnds);
}
/**
* Whether the bucket ever removes the database dumps. Each one holds every
* account that existed when it was taken, so a bucket with no rule for
* `database/` keeps a deleted account's data for as long as the bucket
* exists. Attention, not failing: every restore still works. A bucket that
* will not say (a credential without the permission, a target without the
* call) is not news either way.
*/
export function evaluateDumpRetention(
input: Pick<ReadinessInputs, "dump" | "offhost">,
): ReadinessSignal {
const id = "dump_retention";
if (!input.offhost.configured) {
return signal(id, "inactive", "not_configured", null);
}
const { dump } = input;
const dumpsGoOffhost =
dump !== null &&
dump !== "unreadable" &&
dump.enabled &&
dump.offhostConfigured;
if (!dumpsGoOffhost) {
return signal(id, "inactive", "no_offhost_dumps", null);
}
const rule = input.offhost.databaseLifecycle;
if (rule?.state === "missing") {
return signal(id, "attention", "missing", null);
}
if (rule?.state === "configured" && rule.expirationDays !== null) {
return signal(id, "ok", "configured", null, {
days: rule.expirationDays,
});
}
return signal(id, "ok", "unknown", null);
}
export function evaluateRecoveryKit(
input: Pick<ReadinessInputs, "now" | "firstAdminAt" | "kit">,
): ReadinessSignal {
@@ -420,6 +479,7 @@ export function evaluateRecoveryReadiness(
evaluateOffhostCopies(inputs),
evaluateRestoreDrill(inputs),
evaluateOffhostSetup(inputs),
evaluateDumpRetention(inputs),
evaluateRecoveryKit(inputs),
];
const state = signals.reduce<Exclude<ReadinessState, "inactive">>(
+11 -1
View File
@@ -11,12 +11,17 @@
* - the off-host copies: the per-account ledger the nightly run writes, and
* the same "configured" rule the backups page uses (the environment, or a
* ledger row as proof that an upload happened);
* - the bucket's lifecycle rule for `database/`: the same cached probe the
* backups page shows, asked only when a bucket is configured;
* - the drill: its persisted last outcome, with no time window;
* - the kit: the same assessment the kit card shows;
* - the dump: whatever the dump module reports, through a provider.
*/
import { prisma } from "@/lib/db";
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
import {
offhostBackupConfigured,
probeOffhostLifecycle,
} from "@/lib/jobs/offhost-backup";
import {
readDatabaseBackupHealth,
type DatabaseBackupHealthProvider,
@@ -74,6 +79,10 @@ export async function readReadinessInputs(
const configured =
offhostBackupConfigured() ||
accounts.some((account) => account.offhostBackupState !== null);
// Never throws: a bucket that cannot be asked answers `unknown`.
const databaseLifecycle = configured
? (await probeOffhostLifecycle()).database
: null;
return {
now,
@@ -88,6 +97,7 @@ export async function readReadinessInputs(
},
offhost: {
configured,
databaseLifecycle,
accounts: accounts.map((account) => ({
lastAttemptAt: account.offhostBackupState?.lastAttemptAt ?? null,
lastSuccessAt: account.offhostBackupState?.lastSuccessAt ?? null,
+3 -3
View File
@@ -107,14 +107,14 @@ export type DocumentSummaryStateValue =
/**
* Refs #776 — why a document's most recent content-index attempt produced no
* index. Mirrors the free-String `lastIndexOutcome` column (closed here, not
* in the schema, so a new reason needs no migration); the detail view renders
* a plain-words line per value instead of a bare "not indexed" that cannot
* tell "never tried" from "tried and failed".
* in the schema, so a new reason needs no migration); the detail view says
* each in plain words, so "never tried" and "tried and failed" differ.
*/
export const DOCUMENT_INDEX_OUTCOMES = [
"local-empty",
"local-unsupported",
"decrypt-error",
"resource-limit",
"raster-failed",
"provider-error",
"pdf-needs-anthropic",