mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 15:23:29 +00:00
Merge remote-tracking branch 'origin/release/v1.46.0' into fix/v146-move-safety
This commit is contained in:
commit
0085915a56
53 files changed
+1620
-246
No files matched your search
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
|
||||
"count": 84,
|
||||
"total": 94948,
|
||||
"total": 94928,
|
||||
"entries": {
|
||||
"app/medications/page-client.tsx": 839,
|
||||
"components/charts/mood-chart.tsx": 1166,
|
||||
@@ -66,7 +66,7 @@
|
||||
"lib/jobs/coach-nudge.ts": 1084,
|
||||
"lib/jobs/insight-pregenerate.ts": 1189,
|
||||
"lib/jobs/medication-intake-import.ts": 819,
|
||||
"lib/jobs/offhost-backup.ts": 1114,
|
||||
"lib/jobs/offhost-backup.ts": 1094,
|
||||
"lib/jobs/reaction-line.ts": 803,
|
||||
"lib/managed-profiles/handover.ts": 986,
|
||||
"lib/mcp/record-search.ts": 1467,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "as never casts in tests, per test file under src/. Maintained by src/__tests__/test-realism-ratchet.test.ts.",
|
||||
"count": 524,
|
||||
"total": 5489,
|
||||
"total": 5488,
|
||||
"entries": {
|
||||
"__tests__/api/auth/login/mfa-branch.test.ts": 10,
|
||||
"__tests__/api/auth/mfa/verify.test.ts": 9,
|
||||
@@ -361,7 +361,7 @@
|
||||
"lib/documents/__tests__/assist-kinds.test.ts": 1,
|
||||
"lib/documents/__tests__/commit.test.ts": 9,
|
||||
"lib/documents/__tests__/fenced-chat-pipeline.test.ts": 6,
|
||||
"lib/documents/__tests__/index-document.test.ts": 23,
|
||||
"lib/documents/__tests__/index-document.test.ts": 22,
|
||||
"lib/documents/__tests__/index-outcome-record.test.ts": 14,
|
||||
"lib/documents/__tests__/prepare-vision-raster-failure.test.ts": 6,
|
||||
"lib/documents/__tests__/provider-order.test.ts": 1,
|
||||
|
||||
@@ -63,6 +63,8 @@ const databaseHealth = {
|
||||
directory: "/var/lib/healthlog/backups",
|
||||
keepNightly: 7,
|
||||
localDirWritable: true,
|
||||
localPersistent: true,
|
||||
localReason: "volume",
|
||||
offhostConfigured: true,
|
||||
lastGood: {
|
||||
at: "2026-09-21T23:31:40.000Z",
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
* Every backup signal with its verdict, its time and where to fix it: the
|
||||
* database dump, the weekly copy, the off-host copies, the last restore drill
|
||||
* (no time window: a failed drill stays failed until a later one passes), the
|
||||
* off-host setup and the recovery kit. The same judgement the hourly
|
||||
* off-host setup, whether the bucket ever removes the database dumps, and the
|
||||
* recovery kit. The same judgement the hourly
|
||||
* evaluation pushes to the admins (`src/lib/recovery/readiness-model.ts`).
|
||||
* Cookie-only admin.
|
||||
*/
|
||||
|
||||
@@ -17,6 +17,8 @@ const HEALTHY: DatabaseBackupHealth = {
|
||||
directory: "/var/lib/healthlog/backups",
|
||||
keepNightly: 7,
|
||||
localDirWritable: true,
|
||||
localPersistent: true,
|
||||
localReason: "volume",
|
||||
offhostConfigured: true,
|
||||
lastGood: {
|
||||
at: "2026-10-24T23:31:00.000Z",
|
||||
@@ -77,17 +79,36 @@ describe("<DatabaseDumpRow>", () => {
|
||||
});
|
||||
|
||||
it("says when dumps go off-host only, or cannot be taken at all", () => {
|
||||
expect(render({ ...HEALTHY, localDirWritable: false }, null)).toContain(
|
||||
"dumps go off-host only",
|
||||
const unwritable = {
|
||||
...HEALTHY,
|
||||
localDirWritable: false,
|
||||
localPersistent: false,
|
||||
localReason: "not_writable" as const,
|
||||
};
|
||||
expect(render(unwritable, null)).toContain("dumps go off-host only");
|
||||
expect(render({ ...unwritable, offhostConfigured: false }, null)).toContain(
|
||||
"no database dump can be taken",
|
||||
);
|
||||
expect(
|
||||
render(
|
||||
{ ...HEALTHY, localDirWritable: false, offhostConfigured: false },
|
||||
null,
|
||||
),
|
||||
).toContain("no database dump can be taken");
|
||||
expect(render({ ...HEALTHY, lastGood: null }, null)).toContain(
|
||||
"No database dump yet",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not promise dumps on this server from a directory inside the container", () => {
|
||||
const layer = {
|
||||
...HEALTHY,
|
||||
localPersistent: false,
|
||||
localReason: "container_layer" as const,
|
||||
lastGood: { ...HEALTHY.lastGood!, local: false },
|
||||
};
|
||||
const html = render(layer);
|
||||
expect(html).toContain('data-slot="database-dump-setup"');
|
||||
expect(html).toContain("lost at the next update");
|
||||
expect(html).toContain("Dumps go off-host only");
|
||||
expect(html).not.toContain("kept on this server");
|
||||
expect(html).toContain("removed after 30 days");
|
||||
const nowhere = render({ ...layer, offhostConfigured: false }, null);
|
||||
expect(nowhere).toContain("no database dump is kept anywhere");
|
||||
expect(nowhere).not.toContain('data-slot="database-dump-retention"');
|
||||
});
|
||||
});
|
||||
@@ -6,7 +6,10 @@
|
||||
* The account copies below it restore one account on this server; the dump is
|
||||
* the copy that rebuilds the server. So the row leads the card and says three
|
||||
* things: when the newest good dump was taken and where it is, whether the
|
||||
* newest run failed, and whether this server can keep a dump at all.
|
||||
* newest run failed, and whether this server can keep a dump at all. A
|
||||
* directory inside the container (the image's own, without the volume) is
|
||||
* writable and still keeps nothing past the next update, so "kept on this
|
||||
* server" is only said of a directory that survives it.
|
||||
*/
|
||||
import { AlertTriangle, HardDrive } from "lucide-react";
|
||||
|
||||
@@ -45,7 +48,15 @@ export function DatabaseDumpRow({
|
||||
let setupProblem: string | null = null;
|
||||
if (!health.enabled) {
|
||||
setupProblem = t("admin.section.backups.database.disabled");
|
||||
} else if (!health.localDirWritable) {
|
||||
} else if (health.localDirWritable && !health.localPersistent) {
|
||||
setupProblem = health.offhostConfigured
|
||||
? t("admin.section.backups.database.notPersistentOffhost", {
|
||||
dir: health.directory,
|
||||
})
|
||||
: t("admin.section.backups.database.notPersistent", {
|
||||
dir: health.directory,
|
||||
});
|
||||
} else if (!health.localPersistent) {
|
||||
setupProblem = health.offhostConfigured
|
||||
? t("admin.section.backups.database.noDirectoryOffhost", {
|
||||
dir: health.directory,
|
||||
@@ -55,6 +66,19 @@ export function DatabaseDumpRow({
|
||||
});
|
||||
}
|
||||
|
||||
const retention = [
|
||||
health.localPersistent
|
||||
? t("admin.section.backups.database.retentionLocal", {
|
||||
nights: health.keepNightly,
|
||||
})
|
||||
: null,
|
||||
bucketLifecycle?.state === "configured"
|
||||
? t("admin.section.backups.database.retentionOffhost", {
|
||||
days: bucketLifecycle.expirationDays ?? 0,
|
||||
})
|
||||
: null,
|
||||
].filter((line): line is string => line !== null);
|
||||
|
||||
return (
|
||||
<div className="space-y-2" data-slot="database-dump">
|
||||
<div className="flex items-start gap-2">
|
||||
@@ -80,19 +104,14 @@ export function DatabaseDumpRow({
|
||||
</p>
|
||||
{/* How long a deleted account stays in the dumps: the operator's
|
||||
side of the deletion promise (GDPR retention of backups). */}
|
||||
<p
|
||||
className="text-muted-foreground text-xs"
|
||||
data-slot="database-dump-retention"
|
||||
>
|
||||
{t("admin.section.backups.database.retentionLocal", {
|
||||
nights: health.keepNightly,
|
||||
})}
|
||||
{bucketLifecycle?.state === "configured"
|
||||
? ` ${t("admin.section.backups.database.retentionOffhost", {
|
||||
days: bucketLifecycle.expirationDays ?? 0,
|
||||
})}`
|
||||
: null}
|
||||
</p>
|
||||
{retention.length > 0 ? (
|
||||
<p
|
||||
className="text-muted-foreground text-xs"
|
||||
data-slot="database-dump-retention"
|
||||
>
|
||||
{retention.join(" ")}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
/**
|
||||
* `<RecoveryReadinessCard>` — every backup signal on one card: the database
|
||||
* dump, the weekly account copies, the off-host copies, the last restore
|
||||
* drill, the off-host setup and the recovery kit. Each row says its verdict,
|
||||
* drill, the off-host setup, the bucket's rule for the dumps and the recovery
|
||||
* kit. Each row says its verdict,
|
||||
* the time it is about and where to fix it.
|
||||
*
|
||||
* No time window anywhere: a failed drill stays on this card until a later
|
||||
|
||||
@@ -407,6 +407,48 @@ describe("getClientIp against the connection's own peer", () => {
|
||||
).toBe("203.0.113.6");
|
||||
});
|
||||
|
||||
// The ranges docs/self-hosting/reverse-proxy.md lists, one address each.
|
||||
// Reserved and multicast stay in: no host on the internet can complete a
|
||||
// TCP handshake from them, so a peer there is on a network the operator
|
||||
// runs, like a private one.
|
||||
it("reads forwarding headers from every range the reverse-proxy page lists", () => {
|
||||
for (const proxy of [
|
||||
"127.0.0.2", // loopback
|
||||
"0.0.0.1", // 0.0.0.0/8
|
||||
"10.1.2.3", // private
|
||||
"172.31.0.9",
|
||||
"192.168.1.20",
|
||||
"100.64.0.1", // CGNAT (Tailscale)
|
||||
"169.254.10.10", // link-local
|
||||
"192.0.0.8", // IETF protocol assignments
|
||||
"198.18.0.4", // benchmarking
|
||||
"240.0.0.1", // reserved
|
||||
"fd7a:115c:a1e0::1", // unique-local (Tailscale)
|
||||
"fe80::1", // link-local
|
||||
"fec0::1", // site-local
|
||||
"::ffff:10.0.0.3", // IPv4-mapped private
|
||||
]) {
|
||||
expect(
|
||||
getClientIp(stamped(proxy, { "x-forwarded-for": "203.0.113.7" })),
|
||||
proxy,
|
||||
).toBe("203.0.113.7");
|
||||
}
|
||||
for (const outside of ["8.8.8.8", "2001:4860:4860::8888"]) {
|
||||
expect(
|
||||
getClientIp(stamped(outside, { "x-forwarded-for": "203.0.113.7" })),
|
||||
).toBe(outside);
|
||||
}
|
||||
});
|
||||
|
||||
it("believes cf-connecting-ip from any peer once the flag is on, which is why the origin must take Cloudflare only", () => {
|
||||
process.env.TRUST_CF_CONNECTING_IP = "1";
|
||||
expect(
|
||||
getClientIp(
|
||||
stamped("198.51.100.7", { "cf-connecting-ip": "203.0.113.9" }),
|
||||
),
|
||||
).toBe("203.0.113.9");
|
||||
});
|
||||
|
||||
it("reads an IPv4 peer that the socket reports in IPv6 form", () => {
|
||||
expect(
|
||||
getClientIp(
|
||||
|
||||
@@ -158,43 +158,67 @@ describe("collectReadiness — the database dump", () => {
|
||||
BACKUP_S3_SECRET_KEY: "s",
|
||||
BACKUP_ENCRYPTION_KEY: HEX32,
|
||||
};
|
||||
const STATUS = {
|
||||
volume: { writable: true, persistent: true, reason: "volume" },
|
||||
layer: { writable: true, persistent: false, reason: "container_layer" },
|
||||
memory: { writable: true, persistent: false, reason: "memory" },
|
||||
none: { writable: false, persistent: false, reason: "not_writable" },
|
||||
} as const;
|
||||
const dumpLine = (
|
||||
env: Record<string, string | undefined>,
|
||||
writable: boolean,
|
||||
status: keyof typeof STATUS,
|
||||
) =>
|
||||
collectReadiness(env, () => writable).lines.find(
|
||||
collectReadiness(env, () => STATUS[status]).lines.find(
|
||||
(l) => l.label === "Database dump",
|
||||
);
|
||||
|
||||
it("names the schedule, the directory and the copies kept", () => {
|
||||
const line = dumpLine(baseEnv(), true);
|
||||
const line = dumpLine(baseEnv(), "volume");
|
||||
expect(line?.status).toBe("ok");
|
||||
expect(line?.detail).toBe(
|
||||
"nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7",
|
||||
);
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain(
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, "volume")?.detail).toContain(
|
||||
"copied off-host",
|
||||
);
|
||||
});
|
||||
|
||||
it("warns when nothing can be kept on the server", () => {
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({
|
||||
expect(dumpLine({ ...baseEnv(), ...offhost }, "none")).toMatchObject({
|
||||
status: "warn",
|
||||
detail: expect.stringContaining("goes off-host only"),
|
||||
});
|
||||
expect(dumpLine(baseEnv(), false)).toMatchObject({
|
||||
expect(dumpLine(baseEnv(), "none")).toMatchObject({
|
||||
status: "warn",
|
||||
detail: expect.stringContaining("no database dump can be taken"),
|
||||
detail: expect.stringContaining("no database dump is kept anywhere"),
|
||||
});
|
||||
});
|
||||
|
||||
it("does not call a directory inside the container kept: it is writable and lost at the next update", () => {
|
||||
const offhostOnly = dumpLine({ ...baseEnv(), ...offhost }, "layer");
|
||||
expect(offhostOnly?.status).toBe("warn");
|
||||
expect(offhostOnly?.detail).toContain(
|
||||
"/var/lib/healthlog/backups is inside the container, not on a mounted volume",
|
||||
);
|
||||
expect(offhostOnly?.detail).toContain("none is kept on this server");
|
||||
expect(offhostOnly?.detail).toContain("none is taken before a migration");
|
||||
expect(offhostOnly?.detail).not.toContain("keeping 7");
|
||||
expect(dumpLine(baseEnv(), "layer")).toMatchObject({
|
||||
status: "warn",
|
||||
detail: expect.stringContaining("no database dump is kept anywhere"),
|
||||
});
|
||||
expect(dumpLine(baseEnv(), "memory")?.detail).toContain(
|
||||
"lost at the next restart",
|
||||
);
|
||||
});
|
||||
|
||||
it("warns when it is switched off, and names a schedule it ignored", () => {
|
||||
expect(
|
||||
dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true),
|
||||
dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, "volume"),
|
||||
).toMatchObject({ status: "warn" });
|
||||
const report = collectReadiness(
|
||||
{ ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" },
|
||||
() => true,
|
||||
() => STATUS.volume,
|
||||
);
|
||||
expect(
|
||||
report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail,
|
||||
|
||||
@@ -20,11 +20,12 @@ import {
|
||||
legacyAnyHostConfigured,
|
||||
} from "@/lib/ai/local-host-allowlist";
|
||||
import {
|
||||
backupDirectoryStatus,
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
resolveBackupDir,
|
||||
resolveBackupSchedule,
|
||||
resolveDailyKeep,
|
||||
type BackupDirectoryStatus,
|
||||
} from "@/lib/database-backup/dump-core.mjs";
|
||||
|
||||
export interface ReadinessLine {
|
||||
@@ -74,7 +75,7 @@ function anySet(env: Env, keys: string[]): boolean {
|
||||
*/
|
||||
export function databaseBackupReadiness(
|
||||
env: Env,
|
||||
writable: (dir: string) => boolean,
|
||||
directoryStatus: (dir: string) => BackupDirectoryStatus,
|
||||
): ReadinessLine[] {
|
||||
const label = "Database dump";
|
||||
if (!databaseBackupEnabled(env)) {
|
||||
@@ -104,35 +105,44 @@ export function databaseBackupReadiness(
|
||||
"BACKUP_S3_SECRET_KEY",
|
||||
"BACKUP_ENCRYPTION_KEY",
|
||||
]);
|
||||
if (writable(dir)) {
|
||||
const status = directoryStatus(dir);
|
||||
if (status.persistent) {
|
||||
lines.push({
|
||||
label,
|
||||
status: "ok",
|
||||
detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`,
|
||||
});
|
||||
} else if (offhost) {
|
||||
lines.push({
|
||||
label,
|
||||
status: "warn",
|
||||
detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
|
||||
});
|
||||
} else {
|
||||
lines.push({
|
||||
label,
|
||||
status: "warn",
|
||||
detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`,
|
||||
});
|
||||
return lines;
|
||||
}
|
||||
// Writable but thrown away with the container (the image's own directory
|
||||
// without the volume) or at a restart (a tmpfs): no dump is kept there,
|
||||
// because one that is would be counted as kept and then lost.
|
||||
const where = !status.writable
|
||||
? `${dir} is not writable`
|
||||
: status.reason === "memory"
|
||||
? `${dir} is in memory (tmpfs), so a dump there is lost at the next restart`
|
||||
: `${dir} is inside the container, not on a mounted volume, so a dump there is lost at the next update`;
|
||||
const fix =
|
||||
"Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR to a mounted directory";
|
||||
lines.push({
|
||||
label,
|
||||
status: "warn",
|
||||
detail: offhost
|
||||
? `${where}: the nightly dump goes off-host only, none is kept on this server and none is taken before a migration. ${fix}`
|
||||
: `${where} and no off-host bucket is configured: no database dump is kept anywhere. ${fix}`,
|
||||
});
|
||||
return lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect the environment and build the readiness report. Pure apart from
|
||||
* `writable`, the probe of the dump directory, which tests replace.
|
||||
* `directoryStatus`, the probe of the dump directory, which tests replace.
|
||||
*/
|
||||
export function collectReadiness(
|
||||
env: Env = process.env,
|
||||
writable: (dir: string) => boolean = isWritableDirectory,
|
||||
directoryStatus: (
|
||||
dir: string,
|
||||
) => BackupDirectoryStatus = backupDirectoryStatus,
|
||||
): ReadinessReport {
|
||||
const lines: ReadinessLine[] = [];
|
||||
|
||||
@@ -211,7 +221,7 @@ export function collectReadiness(
|
||||
});
|
||||
}
|
||||
|
||||
lines.push(...databaseBackupReadiness(env, writable));
|
||||
lines.push(...databaseBackupReadiness(env, directoryStatus));
|
||||
|
||||
// --- Optional subsystems (informational) --------------------------------
|
||||
const optional: Array<[string, boolean]> = [
|
||||
|
||||
@@ -25,6 +25,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
backupDirectoryStatus,
|
||||
bucketConfigFromEnv,
|
||||
pickBucketDump,
|
||||
buildManifest,
|
||||
@@ -37,13 +38,17 @@ import {
|
||||
formatDumpTimestamp,
|
||||
keyFingerprints,
|
||||
listDumpSets,
|
||||
mountContaining,
|
||||
offhostObjectKey,
|
||||
parseMountInfo,
|
||||
parseDumpFileName,
|
||||
parseManifest,
|
||||
resolveBackupSchedule,
|
||||
resolveDailyKeep,
|
||||
selectForDeletion,
|
||||
type DirectoryProbe,
|
||||
type DumpSet,
|
||||
type Env,
|
||||
} from "../dump-core.mjs";
|
||||
import type { S3Like } from "@/lib/jobs/offhost-backup";
|
||||
|
||||
@@ -687,3 +692,168 @@ describe("finding a dump in the bucket", () => {
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("whether the dump directory survives the container", () => {
|
||||
// /proc/self/mountinfo of the app container under the bundled compose
|
||||
// file, trimmed to the lines that matter, with and without the volume.
|
||||
const ROOT =
|
||||
"1470 1351 0:330 / / rw,relatime master:396 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/A:/var/lib/docker/overlay2/l/B,upperdir=/var/lib/docker/overlay2/x/diff,workdir=/var/lib/docker/overlay2/x/work";
|
||||
const COMMON = [
|
||||
"1471 1470 0:333 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw",
|
||||
"1472 1470 0:334 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755",
|
||||
"1481 1470 8:1 /var/lib/docker/containers/c/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/sda1 rw",
|
||||
"1483 1470 8:1 /var/lib/docker/containers/c/hosts /etc/hosts rw,relatime - ext4 /dev/sda1 rw",
|
||||
];
|
||||
const VOLUME =
|
||||
"1480 1470 8:1 /var/lib/docker/volumes/healthlog_backups/_data /var/lib/healthlog/backups rw,relatime master:1 - ext4 /dev/sda1 rw,discard";
|
||||
const DIR = "/var/lib/healthlog/backups";
|
||||
|
||||
function probe(
|
||||
over: Partial<DirectoryProbe> & { files?: Record<string, string> } = {},
|
||||
): DirectoryProbe {
|
||||
const { files = {}, ...rest } = over;
|
||||
return {
|
||||
writable: () => true,
|
||||
realpath: (p) => p,
|
||||
readText: (p) => files[p] ?? null,
|
||||
exists: (p) => p in files,
|
||||
device: () => null,
|
||||
env: {},
|
||||
...rest,
|
||||
};
|
||||
}
|
||||
const inDocker = (...lines: string[]) =>
|
||||
probe({
|
||||
files: {
|
||||
"/.dockerenv": "",
|
||||
"/proc/self/mountinfo": [ROOT, ...COMMON, ...lines].join("\n"),
|
||||
},
|
||||
});
|
||||
|
||||
it("reads mount points and types, escapes undone, odd lines skipped", () => {
|
||||
const mounts = parseMountInfo(
|
||||
[
|
||||
ROOT,
|
||||
"90 1470 8:1 / /mnt/my\\040disk rw - ext4 /dev/sdb1 rw",
|
||||
"not a mount line",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(mounts).toEqual([
|
||||
{ mountPoint: "/", fsType: "overlay" },
|
||||
{ mountPoint: "/mnt/my disk", fsType: "ext4" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("finds the longest mount that holds a path, by whole segments", () => {
|
||||
const mounts = [
|
||||
{ mountPoint: "/", fsType: "overlay" },
|
||||
{ mountPoint: "/var/lib/healthlog/back", fsType: "ext4" },
|
||||
{ mountPoint: "/var/lib/healthlog", fsType: "xfs" },
|
||||
];
|
||||
expect(mountContaining(DIR, mounts)?.mountPoint).toBe("/var/lib/healthlog");
|
||||
expect(mountContaining("/var/lib/other", mounts)?.mountPoint).toBe("/");
|
||||
expect(mountContaining("/var/lib/healthlog", mounts)?.fsType).toBe("xfs");
|
||||
});
|
||||
|
||||
it("counts the backups volume as kept", () => {
|
||||
expect(backupDirectoryStatus(DIR, inDocker(VOLUME))).toEqual({
|
||||
writable: true,
|
||||
persistent: true,
|
||||
reason: "volume",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not count the image's own directory without the volume, though it is writable", () => {
|
||||
expect(backupDirectoryStatus(DIR, inDocker())).toEqual({
|
||||
writable: true,
|
||||
persistent: false,
|
||||
reason: "container_layer",
|
||||
});
|
||||
});
|
||||
|
||||
it("counts a volume on a parent directory, and follows a symlink to where the files land", () => {
|
||||
const parent =
|
||||
"1480 1470 8:1 /srv/healthlog /var/lib/healthlog rw - ext4 /dev/sda1 rw";
|
||||
expect(backupDirectoryStatus(DIR, inDocker(parent)).persistent).toBe(true);
|
||||
const linked = {
|
||||
...inDocker(VOLUME),
|
||||
realpath: (p: string) => (p === "/backups" ? DIR : p),
|
||||
};
|
||||
expect(backupDirectoryStatus("/backups", linked).reason).toBe("volume");
|
||||
expect(backupDirectoryStatus("/elsewhere", inDocker(VOLUME)).reason).toBe(
|
||||
"container_layer",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not count a tmpfs as kept", () => {
|
||||
const tmpfs =
|
||||
"1480 1470 0:400 / /var/lib/healthlog/backups rw - tmpfs tmpfs rw,size=1g";
|
||||
expect(backupDirectoryStatus(DIR, inDocker(tmpfs))).toEqual({
|
||||
writable: true,
|
||||
persistent: false,
|
||||
reason: "memory",
|
||||
});
|
||||
});
|
||||
|
||||
it("recognises a container by Podman's marker, Kubernetes, the cgroup or an overlay root", () => {
|
||||
const table = [ROOT, ...COMMON].join("\n");
|
||||
const cases: Array<{ files?: Record<string, string>; env?: Env }> = [
|
||||
{ files: { "/run/.containerenv": "" } },
|
||||
{ env: { KUBERNETES_SERVICE_HOST: "10.0.0.1" } },
|
||||
{ files: { "/proc/self/cgroup": "12:pids:/docker/0123abcd\n" } },
|
||||
{},
|
||||
];
|
||||
for (const extra of cases) {
|
||||
const p = probe({
|
||||
env: extra.env ?? {},
|
||||
files: { ...extra.files, "/proc/self/mountinfo": table },
|
||||
});
|
||||
expect(backupDirectoryStatus(DIR, p).reason).toBe("container_layer");
|
||||
}
|
||||
});
|
||||
|
||||
it("treats a writable directory outside a container as kept: a host, a VM", () => {
|
||||
const host = probe({
|
||||
files: {
|
||||
"/proc/self/cgroup": "0::/user.slice/user-1000.slice\n",
|
||||
"/proc/self/mountinfo":
|
||||
"22 1 259:2 / / rw,relatime shared:1 - ext4 /dev/nvme0n1p2 rw",
|
||||
},
|
||||
});
|
||||
expect(backupDirectoryStatus(DIR, host)).toEqual({
|
||||
writable: true,
|
||||
persistent: true,
|
||||
reason: "host",
|
||||
});
|
||||
// No /proc at all (macOS, a BSD): a host too.
|
||||
expect(backupDirectoryStatus(DIR, probe()).reason).toBe("host");
|
||||
});
|
||||
|
||||
it("falls back to the device numbers in a container without a mount table", () => {
|
||||
const same = probe({ files: { "/.dockerenv": "" }, device: () => 42 });
|
||||
expect(backupDirectoryStatus(DIR, same).reason).toBe("container_layer");
|
||||
const apart = probe({
|
||||
files: { "/.dockerenv": "" },
|
||||
device: (p) => (p === "/" ? 42 : 7),
|
||||
});
|
||||
expect(backupDirectoryStatus(DIR, apart).reason).toBe("volume");
|
||||
});
|
||||
|
||||
it("says when nothing can be written at all, before anything else", () => {
|
||||
expect(
|
||||
backupDirectoryStatus(DIR, {
|
||||
...inDocker(VOLUME),
|
||||
writable: () => false,
|
||||
}),
|
||||
).toEqual({ writable: false, persistent: false, reason: "not_writable" });
|
||||
});
|
||||
|
||||
it("answers for a real directory on this machine without throwing", () => {
|
||||
const status = backupDirectoryStatus(tempDir());
|
||||
expect(status.writable).toBe(true);
|
||||
expect(["host", "volume", "container_layer", "memory"]).toContain(
|
||||
status.reason,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -33,10 +33,13 @@ import {
|
||||
constants as fsConstants,
|
||||
createReadStream,
|
||||
createWriteStream,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
readSync,
|
||||
readdirSync,
|
||||
realpathSync,
|
||||
statSync,
|
||||
} from "node:fs";
|
||||
import { readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
||||
@@ -222,6 +225,193 @@ export function isWritableDirectory(dir) {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Whether a dump kept there survives ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
* File systems that live in memory: a dump on one is gone at the next restart.
|
||||
*/
|
||||
const EPHEMERAL_FS_TYPES = new Set(["tmpfs", "ramfs"]);
|
||||
|
||||
/**
|
||||
* cgroup paths that name a container runtime (cgroup v1; under v2 a container
|
||||
* sees `0::/`, and the marker files or the root mount answer instead).
|
||||
*/
|
||||
const CONTAINER_CGROUP = /\/(docker|kubepods|containerd|libpod|podman)[/-]/;
|
||||
|
||||
/**
|
||||
* @typedef {{ mountPoint: string; fsType: string }} MountEntry
|
||||
*/
|
||||
|
||||
/** Undo the octal escapes the kernel writes for space, tab, newline, `\`. */
|
||||
function unescapeMountPath(raw) {
|
||||
return raw.replace(/\\([0-7]{3})/g, (_, oct) =>
|
||||
String.fromCharCode(Number.parseInt(oct, 8)),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The mount points and file system types in `/proc/self/mountinfo` text.
|
||||
* Lines that do not have the documented shape are skipped.
|
||||
*
|
||||
* @param {string} text
|
||||
* @returns {MountEntry[]}
|
||||
*/
|
||||
export function parseMountInfo(text) {
|
||||
/** @type {MountEntry[]} */
|
||||
const mounts = [];
|
||||
for (const line of text.split("\n")) {
|
||||
const sep = line.indexOf(" - ");
|
||||
if (sep < 0) continue;
|
||||
const left = line.slice(0, sep).split(" ");
|
||||
const right = line.slice(sep + 3).split(" ");
|
||||
if (left.length < 6 || !right[0]) continue;
|
||||
mounts.push({ mountPoint: unescapeMountPath(left[4]), fsType: right[0] });
|
||||
}
|
||||
return mounts;
|
||||
}
|
||||
|
||||
/**
|
||||
* The mount `path` lives on: the longest mount point that is `path` or one of
|
||||
* its parents. A later entry for the same point wins, as it covers the earlier
|
||||
* one. Null when nothing matches (no `/` in the list).
|
||||
*
|
||||
* @param {string} path an absolute path, symlinks resolved
|
||||
* @param {MountEntry[]} mounts
|
||||
* @returns {MountEntry | null}
|
||||
*/
|
||||
export function mountContaining(path, mounts) {
|
||||
/** @type {MountEntry | null} */
|
||||
let best = null;
|
||||
for (const mount of mounts) {
|
||||
const point = mount.mountPoint;
|
||||
const covers =
|
||||
point === "/" ||
|
||||
path === point ||
|
||||
path.startsWith(point.endsWith("/") ? point : `${point}/`);
|
||||
if (covers && (best === null || point.length >= best.mountPoint.length)) {
|
||||
best = mount;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {{
|
||||
* writable: (dir: string) => boolean;
|
||||
* realpath: (path: string) => string;
|
||||
* readText: (path: string) => string | null;
|
||||
* exists: (path: string) => boolean;
|
||||
* device: (path: string) => number | null;
|
||||
* env: Env;
|
||||
* }} DirectoryProbe
|
||||
*/
|
||||
|
||||
/** @type {DirectoryProbe} */
|
||||
const SYSTEM_PROBE = {
|
||||
writable: isWritableDirectory,
|
||||
realpath: (path) => {
|
||||
try {
|
||||
return realpathSync(path);
|
||||
} catch {
|
||||
return path;
|
||||
}
|
||||
},
|
||||
readText: (path) => {
|
||||
try {
|
||||
return readFileSync(path, "utf8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
exists: (path) => existsSync(path),
|
||||
device: (path) => {
|
||||
try {
|
||||
return statSync(path).dev;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
env: process.env,
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether this process runs in a container whose root file system is thrown
|
||||
* away when the container is replaced: Docker (`/.dockerenv`), Podman
|
||||
* (`/run/.containerenv`), Kubernetes, a container cgroup, or a root file
|
||||
* system that is an overlay. A host, a VM or a system container with a disk
|
||||
* of its own answers false.
|
||||
*
|
||||
* @param {DirectoryProbe} probe
|
||||
* @param {MountEntry[] | null} mounts
|
||||
*/
|
||||
export function runsInContainer(probe, mounts) {
|
||||
if (probe.exists("/.dockerenv") || probe.exists("/run/.containerenv")) {
|
||||
return true;
|
||||
}
|
||||
if ((probe.env.KUBERNETES_SERVICE_HOST ?? "").trim()) return true;
|
||||
if (CONTAINER_CGROUP.test(probe.readText("/proc/self/cgroup") ?? "")) {
|
||||
return true;
|
||||
}
|
||||
const root = mounts ? mountContaining("/", mounts) : null;
|
||||
return root?.mountPoint === "/" && root.fsType === "overlay";
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {{
|
||||
* writable: boolean;
|
||||
* persistent: boolean;
|
||||
* reason: "volume" | "host" | "container_layer" | "memory" | "not_writable";
|
||||
* }} BackupDirectoryStatus
|
||||
*/
|
||||
|
||||
/**
|
||||
* Whether a dump written to `dir` is still there after the container is
|
||||
* replaced, which is what "kept on this server" promises.
|
||||
*
|
||||
* volume — in a container, on a mount of its own (the `backups`
|
||||
* volume, a bind mount): it survives.
|
||||
* host — not in a container: a writable directory is a disk.
|
||||
* container_layer — in a container, on its root file system: an update
|
||||
* replaces the container and the dump with it. The image
|
||||
* creates the default directory, so without the volume it
|
||||
* is writable and looks fine; this is the case to catch.
|
||||
* memory — on a tmpfs: gone at the next restart.
|
||||
* not_writable — nothing can be written there at all.
|
||||
*
|
||||
* Never throws. `probe` is the file system; tests hand in their own.
|
||||
*
|
||||
* @param {string} dir
|
||||
* @param {DirectoryProbe} [probe]
|
||||
* @returns {BackupDirectoryStatus}
|
||||
*/
|
||||
export function backupDirectoryStatus(dir, probe = SYSTEM_PROBE) {
|
||||
if (!probe.writable(dir)) {
|
||||
return { writable: false, persistent: false, reason: "not_writable" };
|
||||
}
|
||||
const text = probe.readText("/proc/self/mountinfo");
|
||||
const mounts = text === null ? null : parseMountInfo(text);
|
||||
if (!runsInContainer(probe, mounts)) {
|
||||
return { writable: true, persistent: true, reason: "host" };
|
||||
}
|
||||
const real = probe.realpath(dir);
|
||||
const mount = mounts ? mountContaining(real, mounts) : null;
|
||||
if (mount) {
|
||||
if (EPHEMERAL_FS_TYPES.has(mount.fsType)) {
|
||||
return { writable: true, persistent: false, reason: "memory" };
|
||||
}
|
||||
return mount.mountPoint === "/"
|
||||
? { writable: true, persistent: false, reason: "container_layer" }
|
||||
: { writable: true, persistent: true, reason: "volume" };
|
||||
}
|
||||
// No mount table to read: a directory on the root's own device is in the
|
||||
// container's layer.
|
||||
const own = probe.device(real);
|
||||
const root = probe.device("/");
|
||||
return own !== null && own === root
|
||||
? { writable: true, persistent: false, reason: "container_layer" }
|
||||
: { writable: true, persistent: true, reason: "volume" };
|
||||
}
|
||||
|
||||
// ── Keys ───────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
* alert can act on.
|
||||
*
|
||||
* Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of
|
||||
* this process: whether `DB_BACKUP_DIR` is writable here, and whether the
|
||||
* off-host bucket is configured. A ledger row that says a copy went off-host
|
||||
* this process: whether `DB_BACKUP_DIR` is writable here and survives the
|
||||
* container being replaced, and whether the off-host bucket is configured. A ledger row that says a copy went off-host
|
||||
* outranks this process's environment, for the reason the off-host card gives
|
||||
* (a split install can hand the bucket variables to the worker only).
|
||||
*/
|
||||
@@ -13,10 +13,11 @@ import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
|
||||
import {
|
||||
backupDirectoryStatus,
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
resolveBackupDir,
|
||||
resolveDailyKeep,
|
||||
type BackupDirectoryStatus,
|
||||
} from "./dump-core.mjs";
|
||||
|
||||
export interface DatabaseBackupHealth {
|
||||
@@ -30,8 +31,20 @@ export interface DatabaseBackupHealth {
|
||||
* page says.
|
||||
*/
|
||||
keepNightly: number;
|
||||
/** This process can write there. False means dumps only go off-host. */
|
||||
/** This process can write there. */
|
||||
localDirWritable: boolean;
|
||||
/**
|
||||
* A dump written there survives an update: the directory is on a volume or
|
||||
* a bind mount, or the app does not run in a container. False means no dump
|
||||
* is kept on this server, whatever an older ledger row says, and dumps only
|
||||
* go off-host. `localReason` says why.
|
||||
*/
|
||||
localPersistent: boolean;
|
||||
/**
|
||||
* `volume`, `host`, `container_layer` (the image's own directory, without
|
||||
* the volume mounted), `memory` (a tmpfs) or `not_writable`.
|
||||
*/
|
||||
localReason: BackupDirectoryStatus["reason"];
|
||||
/** The off-host bucket is configured, or a dump has been uploaded to it. */
|
||||
offhostConfigured: boolean;
|
||||
/** The newest dump that was kept on the server or reached the bucket. */
|
||||
@@ -55,15 +68,22 @@ const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000;
|
||||
export async function getDatabaseBackupHealth(
|
||||
now: Date = new Date(),
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
directoryStatus: (
|
||||
dir: string,
|
||||
) => BackupDirectoryStatus = backupDirectoryStatus,
|
||||
): Promise<DatabaseBackupHealth> {
|
||||
const directory = resolveBackupDir(env);
|
||||
const local = directoryStatus(directory);
|
||||
// A dump recorded as written into a directory that does not survive the
|
||||
// container was lost with the container that wrote it: only a copy in the
|
||||
// bucket still counts.
|
||||
const keptWhere = local.persistent
|
||||
? [{ localResult: "written" }, { offhostResult: "uploaded" }]
|
||||
: [{ offhostResult: "uploaded" }];
|
||||
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
|
||||
[
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: {
|
||||
fileName: { not: null },
|
||||
OR: [{ localResult: "written" }, { offhostResult: "uploaded" }],
|
||||
},
|
||||
where: { fileName: { not: null }, OR: keptWhere },
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: {
|
||||
startedAt: true,
|
||||
@@ -108,7 +128,9 @@ export async function getDatabaseBackupHealth(
|
||||
enabled: databaseBackupEnabled(env),
|
||||
directory,
|
||||
keepNightly: resolveDailyKeep(env),
|
||||
localDirWritable: isWritableDirectory(directory),
|
||||
localDirWritable: local.writable,
|
||||
localPersistent: local.persistent,
|
||||
localReason: local.reason,
|
||||
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
|
||||
lastGood: lastGood
|
||||
? {
|
||||
@@ -116,7 +138,7 @@ export async function getDatabaseBackupHealth(
|
||||
fileName: lastGood.fileName ?? "",
|
||||
sizeBytes:
|
||||
lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes),
|
||||
local: lastGood.localResult === "written",
|
||||
local: local.persistent && lastGood.localResult === "written",
|
||||
offhost: lastGood.offhostResult === "uploaded",
|
||||
}
|
||||
: null,
|
||||
|
||||
@@ -9,11 +9,14 @@
|
||||
* under `database/` in the same envelope the per-account copies use, sealed
|
||||
* with `BACKUP_ENCRYPTION_KEY`.
|
||||
*
|
||||
* Without a writable directory (an install that runs the image without the
|
||||
* volume) the dump is written to a temporary directory, uploaded and removed:
|
||||
* the off-host copy still happens, nothing is kept on the server, and the
|
||||
* ledger and the boot summary say so. With neither, the run is skipped and
|
||||
* recorded as such.
|
||||
* Without a directory that survives the container (an install that runs the
|
||||
* image without the volume: the image's own directory is writable but sits in
|
||||
* the container's layer, which the next update throws away) the dump is
|
||||
* written to a temporary directory, uploaded and removed: the off-host copy
|
||||
* still happens, nothing is kept on the server, and the ledger and the boot
|
||||
* summary say so. A dump left in the container's layer would be counted as
|
||||
* kept and lost with the container, so none is. With neither, the run is
|
||||
* skipped and recorded as such.
|
||||
*
|
||||
* Every run leaves one `DatabaseBackupRun` row: started, then finished with
|
||||
* what happened on each leg. `getDatabaseBackupHealth` reads them.
|
||||
@@ -35,19 +38,20 @@ import packageJson from "../../../package.json";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
backupDirectoryStatus,
|
||||
createDatabaseDump,
|
||||
databaseBackupEnabled,
|
||||
isWritableDirectory,
|
||||
offhostObjectKey,
|
||||
resolveBackupDir,
|
||||
resolveDailyKeep,
|
||||
type BackupDirectoryStatus,
|
||||
} from "./dump-core.mjs";
|
||||
|
||||
/** How long ledger rows are kept. A run a day; a year and a bit of history. */
|
||||
const LEDGER_RETENTION_DAYS = 400;
|
||||
|
||||
export type DatabaseBackupLocalResult =
|
||||
"written" | "no_directory" | "failed" | "skipped";
|
||||
"written" | "no_directory" | "not_persistent" | "failed" | "skipped";
|
||||
export type DatabaseBackupOffhostResult =
|
||||
"uploaded" | "not_configured" | "failed" | "skipped";
|
||||
|
||||
@@ -69,6 +73,8 @@ export interface RunDatabaseBackupOptions {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** A bucket double for tests; the configured bucket otherwise. */
|
||||
s3?: S3Like;
|
||||
/** Where `DB_BACKUP_DIR` stands; the file system's answer otherwise. */
|
||||
directoryStatus?: (dir: string) => BackupDirectoryStatus;
|
||||
}
|
||||
|
||||
/** The running build's version, as `/api/version` answers it. */
|
||||
@@ -111,7 +117,9 @@ export async function runDatabaseBackup(
|
||||
const env = options.env ?? process.env;
|
||||
const now = options.now ?? new Date();
|
||||
const dir = resolveBackupDir(env);
|
||||
const localWritable = isWritableDirectory(dir);
|
||||
const directory = (options.directoryStatus ?? backupDirectoryStatus)(dir);
|
||||
// Kept here only where it survives the container being replaced.
|
||||
const keepLocal = directory.persistent;
|
||||
const offhostConfig = (() => {
|
||||
try {
|
||||
return loadOffhostConfig();
|
||||
@@ -128,7 +136,11 @@ export async function runDatabaseBackup(
|
||||
status: "failed",
|
||||
fileName: null,
|
||||
dumpBytes: null,
|
||||
local: localWritable ? "failed" : "no_directory",
|
||||
local: keepLocal
|
||||
? "failed"
|
||||
: directory.writable
|
||||
? "not_persistent"
|
||||
: "no_directory",
|
||||
offhost: offhostConfig ? "failed" : "not_configured",
|
||||
removed: 0,
|
||||
error: null,
|
||||
@@ -150,11 +162,13 @@ export async function runDatabaseBackup(
|
||||
report.local = "skipped";
|
||||
report.offhost = "skipped";
|
||||
report.error = "DB_BACKUP_ENABLED is false";
|
||||
} else if (!localWritable && !offhostConfig) {
|
||||
} else if (!keepLocal && !offhostConfig) {
|
||||
report.status = "skipped";
|
||||
report.error = `no writable backup directory (${dir}) and no off-host bucket configured`;
|
||||
report.error = directory.writable
|
||||
? `the backup directory (${dir}) is not on persistent storage, so a dump there would be lost with the container, and no off-host bucket is configured`
|
||||
: `no writable backup directory (${dir}) and no off-host bucket configured`;
|
||||
} else {
|
||||
const target = localWritable
|
||||
const target = keepLocal
|
||||
? dir
|
||||
: (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-")));
|
||||
const dump = await createDatabaseDump({
|
||||
@@ -167,7 +181,7 @@ export async function runDatabaseBackup(
|
||||
buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null,
|
||||
deadline: options.deadline,
|
||||
});
|
||||
if (localWritable) report.local = "written";
|
||||
if (keepLocal) report.local = "written";
|
||||
report.fileName = basename(dump.dumpPath);
|
||||
report.dumpBytes = dump.manifest.dump.bytes;
|
||||
|
||||
@@ -188,7 +202,7 @@ export async function runDatabaseBackup(
|
||||
}
|
||||
}
|
||||
|
||||
if (localWritable) {
|
||||
if (keepLocal) {
|
||||
const removed = await applyRetention(dir, {
|
||||
dailyKeep: resolveDailyKeep(env),
|
||||
now: now.getTime(),
|
||||
|
||||
@@ -115,7 +115,7 @@ beforeEach(() => {
|
||||
ok: true,
|
||||
text: "glucose fasting creatinine values report",
|
||||
source: "local-pdf",
|
||||
} as never);
|
||||
});
|
||||
// Default: the auto-read toggle is ON (so the external-pick provider path is
|
||||
// exercised); the OFF-specific tests override this explicitly.
|
||||
vi.mocked(documentAutoReadEnabled).mockResolvedValue(true);
|
||||
@@ -284,12 +284,44 @@ describe("indexDocumentContent — local fallback path", () => {
|
||||
vi.mocked(localExtractText).mockResolvedValue({
|
||||
ok: false,
|
||||
reason: "empty",
|
||||
} as never);
|
||||
});
|
||||
|
||||
const outcome = await indexDocumentContent("user-1", "doc-1");
|
||||
expect(outcome).toEqual({ indexed: false, reason: "local-empty" });
|
||||
expect(upsertContentIndex).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports a PDF past the reader's limits as resource-limit, not as a file that would not open", async () => {
|
||||
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({
|
||||
chain: [],
|
||||
pick: null,
|
||||
localOcrEnabled: false,
|
||||
withheld: null,
|
||||
});
|
||||
vi.mocked(localExtractText).mockResolvedValue({
|
||||
ok: false,
|
||||
reason: "resource-limit",
|
||||
});
|
||||
|
||||
const outcome = await indexDocumentContent("user-1", "doc-1");
|
||||
expect(outcome).toEqual({ indexed: false, reason: "resource-limit" });
|
||||
expect(upsertContentIndex).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps resource-limit over a provider note: the provider never got further either", async () => {
|
||||
vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never);
|
||||
vi.mocked(prepareVisionInput).mockResolvedValue({
|
||||
ok: false,
|
||||
reason: "rasterFailed",
|
||||
});
|
||||
vi.mocked(localExtractText).mockResolvedValue({
|
||||
ok: false,
|
||||
reason: "resource-limit",
|
||||
});
|
||||
|
||||
const outcome = await indexDocumentContent("user-1", "doc-1");
|
||||
expect(outcome).toEqual({ indexed: false, reason: "resource-limit" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("indexDocumentContent — the documentAi capability", () => {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
/**
|
||||
* A PDF that crosses the reader's memory or time bound is reported as such
|
||||
* (`resource-limit`), not as a read error, which the index job records as a
|
||||
* stored file that would not open. The bound itself is the isolated reader's
|
||||
* (`../pdf-isolation.ts`); here it is the error it throws.
|
||||
*/
|
||||
|
||||
const { annotate, logCaught, readPdfText } = vi.hoisted(() => ({
|
||||
annotate: vi.fn(),
|
||||
logCaught: vi.fn(),
|
||||
readPdfText: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate }));
|
||||
vi.mock("@/lib/logging/signal", () => ({ logCaught }));
|
||||
vi.mock("../pdfjs", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../pdfjs")>()),
|
||||
readPdfText,
|
||||
}));
|
||||
|
||||
import { PdfResourceLimitError } from "../pdfjs";
|
||||
import { extractPdfText, localExtractText } from "../local-extract";
|
||||
|
||||
describe("a PDF past the reader's limits", () => {
|
||||
it.each(["memory", "time", "exit"] as const)(
|
||||
"answers resource-limit for the %s bound, without the error log a broken file gets",
|
||||
async (limit) => {
|
||||
readPdfText.mockRejectedValueOnce(new PdfResourceLimitError(limit));
|
||||
annotate.mockClear();
|
||||
logCaught.mockClear();
|
||||
const bytes = Buffer.from("%PDF-1.7 large");
|
||||
expect(await extractPdfText(bytes)).toEqual({
|
||||
ok: false,
|
||||
reason: "resource-limit",
|
||||
});
|
||||
expect(logCaught).not.toHaveBeenCalled();
|
||||
expect(annotate).toHaveBeenCalledWith({
|
||||
action: { name: "documents.localIndex.resourceLimit" },
|
||||
meta: { limit, bytes: bytes.length },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("still answers error for anything else the reader throws", async () => {
|
||||
const broken = new Error("Invalid PDF structure");
|
||||
broken.name = "InvalidPDFException";
|
||||
readPdfText.mockRejectedValueOnce(broken);
|
||||
expect(
|
||||
await localExtractText(Buffer.from("%PDF-1.7"), "application/pdf"),
|
||||
).toEqual({ ok: false, reason: "error" });
|
||||
});
|
||||
});
|
||||
@@ -78,6 +78,7 @@ export type IndexFailureReason =
|
||||
| "local-empty"
|
||||
| "local-unsupported"
|
||||
| "decrypt-error"
|
||||
| "resource-limit"
|
||||
| "raster-failed"
|
||||
| "provider-error"
|
||||
| "pdf-needs-anthropic"
|
||||
@@ -342,6 +343,12 @@ async function tryLocalIndex(
|
||||
if (result.reason === "unsupported") {
|
||||
return { indexed: false, reason: "local-unsupported" };
|
||||
}
|
||||
if (result.reason === "resource-limit") {
|
||||
// Too large or too complex for the reader's bounds: said as such, and
|
||||
// final like the thumbnail's refusal of the same bytes. Nothing retries
|
||||
// it on its own; a person can still ask for a read.
|
||||
return { indexed: false, reason: "resource-limit" };
|
||||
}
|
||||
if (result.reason === "error") {
|
||||
return { indexed: false, reason: "decrypt-error" };
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ import { Buffer } from "node:buffer";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { logCaught } from "@/lib/logging/signal";
|
||||
|
||||
import { readPdfText } from "./pdfjs";
|
||||
import { PdfResourceLimitError, readPdfText } from "./pdfjs";
|
||||
|
||||
/**
|
||||
* Minimum joined text length for a PDF to count as a real text-layer document.
|
||||
@@ -54,15 +54,23 @@ export type LocalExtractSource = "local-pdf" | "local-ocr";
|
||||
* a provider vision pass is the only way to read it.
|
||||
* - `unsupported` → a MIME the local path cannot read without OCR (images) or
|
||||
* at all; leave un-indexed locally.
|
||||
* - `resource-limit` → the PDF crossed the reader's memory or time bound
|
||||
* (`PdfResourceLimitError`): too large or too complex to
|
||||
* read here. The same bytes cross it again, so it is a
|
||||
* final answer, as it is for thumbnails.
|
||||
* - `error` → the file was malformed / unreadable; treat as un-indexed.
|
||||
*/
|
||||
export type LocalExtractResult =
|
||||
| { ok: true; text: string; source: LocalExtractSource }
|
||||
| { ok: false; reason: "empty" | "unsupported" | "error" };
|
||||
| {
|
||||
ok: false;
|
||||
reason: "empty" | "unsupported" | "resource-limit" | "error";
|
||||
};
|
||||
|
||||
/**
|
||||
* Extract a text-layer PDF's embedded text server-side. Returns `empty` for a
|
||||
* scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never
|
||||
* scanned/no-text-layer PDF, `resource-limit` for one past the reader's memory
|
||||
* or time bound, and `error` for a malformed/encrypted one — never
|
||||
* throws, so a bad document can never abort the caller (the upload/backfill job
|
||||
* just leaves it un-indexed). An `error` leaves a `warn` line naming what
|
||||
* pdf.js threw, because the outcome alone does not say why a document stayed
|
||||
@@ -76,6 +84,15 @@ export async function extractPdfText(
|
||||
// Bound the page walk so a pathological many-page PDF costs no more.
|
||||
text = (await readPdfText(buffer, LOCAL_MAX_PAGES)).trim();
|
||||
} catch (err) {
|
||||
if (err instanceof PdfResourceLimitError) {
|
||||
// Not a fault of the file's bytes being unreadable: they are readable,
|
||||
// only not within the bounds this server gives one document.
|
||||
annotate({
|
||||
action: { name: "documents.localIndex.resourceLimit" },
|
||||
meta: { limit: err.limit, bytes: buffer.length },
|
||||
});
|
||||
return { ok: false, reason: "resource-limit" };
|
||||
}
|
||||
const errorType = err instanceof Error ? err.name || "Error" : typeof err;
|
||||
annotate({
|
||||
action: { name: "documents.localIndex.extractFailed" },
|
||||
|
||||
@@ -300,6 +300,7 @@ const inboundDocument = z
|
||||
"local-empty",
|
||||
"local-unsupported",
|
||||
"decrypt-error",
|
||||
"resource-limit",
|
||||
"raster-failed",
|
||||
"provider-error",
|
||||
"pdf-needs-anthropic",
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
|
||||
import {
|
||||
DUMP_FAILING_AFTER_HOURS,
|
||||
evaluateDatabaseDump,
|
||||
evaluateDumpRetention,
|
||||
evaluateOffhostCopies,
|
||||
evaluateOffhostSetup,
|
||||
evaluateRecoveryKit,
|
||||
@@ -45,11 +46,15 @@ function dump(
|
||||
lastFailureReason,
|
||||
...rest
|
||||
} = over;
|
||||
// A directory that cannot be written keeps nothing either.
|
||||
const writable = rest.localDirWritable ?? true;
|
||||
return {
|
||||
enabled: true,
|
||||
directory: "/data/db-backups",
|
||||
keepNightly: 7,
|
||||
localDirWritable: true,
|
||||
localPersistent: writable,
|
||||
localReason: writable ? "volume" : "not_writable",
|
||||
offhostConfigured: false,
|
||||
lastGood: lastGoodAt
|
||||
? {
|
||||
@@ -79,6 +84,7 @@ function inputs(over: Partial<ReadinessInputs> = {}): ReadinessInputs {
|
||||
weekly: { accounts: [{ createdAt: ago(60 * DAY), lastCopyAt: ago(DAY) }] },
|
||||
offhost: {
|
||||
configured: true,
|
||||
databaseLifecycle: { state: "configured", expirationDays: 30 },
|
||||
accounts: [
|
||||
{ lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) },
|
||||
],
|
||||
@@ -172,6 +178,37 @@ describe("database dump", () => {
|
||||
).toBe("ok");
|
||||
});
|
||||
|
||||
it("does not count a writable directory inside the container as a place to keep dumps", () => {
|
||||
const layer = {
|
||||
localDirWritable: true,
|
||||
localPersistent: false,
|
||||
localReason: "container_layer" as const,
|
||||
};
|
||||
// No bucket: the dumps written there are lost at the next update.
|
||||
expect(evalDump(dump(layer))).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "not_persistent",
|
||||
params: { directory: "/data/db-backups" },
|
||||
});
|
||||
// A bucket: fine, and the tile says the bucket holds the only copy.
|
||||
expect(evalDump(dump({ ...layer, offhostConfigured: true }))).toMatchObject(
|
||||
{
|
||||
state: "ok",
|
||||
reason: "fresh_offhost_only",
|
||||
params: { directory: "/data/db-backups" },
|
||||
},
|
||||
);
|
||||
// The same directory on a volume reads as before.
|
||||
expect(evalDump(dump({ offhostConfigured: true }))).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "fresh",
|
||||
});
|
||||
// Switched off with nowhere lasting to write: not news.
|
||||
expect(
|
||||
evalDump(dump({ ...layer, enabled: false, lastGoodAt: null })),
|
||||
).toMatchObject({ state: "inactive", reason: "disabled" });
|
||||
});
|
||||
|
||||
it("gives a newly switched-on dump 36 hours before calling it failing", () => {
|
||||
const fresh = dump({ lastGoodAt: null });
|
||||
expect(evalDump(fresh, ago(HOUR))).toMatchObject({
|
||||
@@ -192,6 +229,76 @@ describe("database dump", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("dump retention in the bucket", () => {
|
||||
const goingOffhost = dump({ offhostConfigured: true });
|
||||
const evalRule = (
|
||||
databaseLifecycle: ReadinessInputs["offhost"]["databaseLifecycle"],
|
||||
d: ReadinessInputs["dump"] = goingOffhost,
|
||||
configured = true,
|
||||
) =>
|
||||
evaluateDumpRetention({
|
||||
dump: d,
|
||||
offhost: { configured, databaseLifecycle, accounts: [] },
|
||||
});
|
||||
|
||||
it("asks for a rule when dumps go to a bucket that never removes them", () => {
|
||||
expect(evalRule({ state: "missing", expirationDays: null })).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "missing",
|
||||
fixPath: "/admin/backups",
|
||||
});
|
||||
});
|
||||
|
||||
it("names the expiry when a rule covers database/, and stays quiet when the bucket will not say", () => {
|
||||
expect(evalRule({ state: "configured", expirationDays: 30 })).toMatchObject(
|
||||
{ state: "ok", reason: "configured", params: { days: 30 } },
|
||||
);
|
||||
expect(evalRule({ state: "unknown", expirationDays: null })).toMatchObject({
|
||||
state: "ok",
|
||||
reason: "unknown",
|
||||
});
|
||||
expect(evalRule(null)).toMatchObject({ state: "ok", reason: "unknown" });
|
||||
});
|
||||
|
||||
it("does not apply without a bucket, or while no dump goes to it", () => {
|
||||
const missing = { state: "missing", expirationDays: null } as const;
|
||||
expect(evalRule(missing, goingOffhost, false)).toMatchObject({
|
||||
state: "inactive",
|
||||
reason: "not_configured",
|
||||
});
|
||||
for (const d of [
|
||||
null,
|
||||
"unreadable" as const,
|
||||
dump({ offhostConfigured: false }),
|
||||
dump({ offhostConfigured: true, enabled: false }),
|
||||
]) {
|
||||
expect(evalRule(missing, d)).toMatchObject({
|
||||
state: "inactive",
|
||||
reason: "no_offhost_dumps",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("moves the overall verdict to attention on its own", () => {
|
||||
const r = evaluateRecoveryReadiness(
|
||||
inputs({
|
||||
dump: goingOffhost,
|
||||
offhost: {
|
||||
configured: true,
|
||||
databaseLifecycle: { state: "missing", expirationDays: null },
|
||||
accounts: [
|
||||
{ lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) },
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
r.signals.filter((x) => x.state !== "ok" && x.state !== "inactive"),
|
||||
).toEqual([expect.objectContaining({ id: "dump_retention" })]);
|
||||
expect(r.state).toBe("attention");
|
||||
});
|
||||
});
|
||||
|
||||
describe("weekly copy", () => {
|
||||
const weekly = (
|
||||
...accounts: Array<{ createdAt: Date; lastCopyAt: Date | null }>
|
||||
@@ -254,7 +361,11 @@ describe("off-host copies", () => {
|
||||
lastAttemptAt: Date | null;
|
||||
lastSuccessAt: Date | null;
|
||||
}>
|
||||
) => evaluateOffhostCopies({ now: NOW, offhost: { configured, accounts } });
|
||||
) =>
|
||||
evaluateOffhostCopies({
|
||||
now: NOW,
|
||||
offhost: { configured, databaseLifecycle: null, accounts },
|
||||
});
|
||||
|
||||
it("does not apply while no bucket is configured", () => {
|
||||
expect(offhost(false).state).toBe("inactive");
|
||||
@@ -302,7 +413,10 @@ describe("off-host copies", () => {
|
||||
|
||||
describe("restore drill", () => {
|
||||
const drill = (d: ReadinessInputs["drill"], configured = true) =>
|
||||
evaluateRestoreDrill({ offhost: { configured, accounts: [] }, drill: d });
|
||||
evaluateRestoreDrill({
|
||||
offhost: { configured, databaseLifecycle: null, accounts: [] },
|
||||
drill: d,
|
||||
});
|
||||
const run = {
|
||||
lastRunAt: ago(5 * DAY),
|
||||
lastSuccessAt: null as Date | null,
|
||||
@@ -351,7 +465,7 @@ describe("off-host setup", () => {
|
||||
evaluateOffhostSetup({
|
||||
now: NOW,
|
||||
firstAdminAt,
|
||||
offhost: { configured, accounts: [] },
|
||||
offhost: { configured, databaseLifecycle: null, accounts: [] },
|
||||
});
|
||||
|
||||
it("asks for a bucket 7 days after the first admin exists", () => {
|
||||
|
||||
@@ -7,8 +7,11 @@
|
||||
* ordinary row. A failed monthly drill was on the failing-jobs card three days
|
||||
* in thirty. An off-host bucket that was never configured showed only on the
|
||||
* backups page, and the key step confirmed one key of the several a restore
|
||||
* needs. This module turns them into six signals with one verdict each, which
|
||||
* the readiness tile shows and the hourly evaluation pushes to every admin.
|
||||
* needs. A bucket whose lifecycle rule left `database/` out kept every dump,
|
||||
* and every deleted account inside it, forever, and only the backups page
|
||||
* said so. This module turns them into seven signals with one verdict each,
|
||||
* which the readiness tile shows and the hourly evaluation pushes to every
|
||||
* admin.
|
||||
*
|
||||
* Pure: the inputs are read elsewhere (`readiness.ts`), so each threshold is a
|
||||
* unit test away and the tile and the push cannot disagree about a verdict.
|
||||
@@ -26,6 +29,7 @@ import {
|
||||
classifyOffhostBackup,
|
||||
type OffhostBackupFreshness,
|
||||
} from "@/lib/jobs/offhost-backup-freshness";
|
||||
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
|
||||
import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
|
||||
import type { KitEntry } from "@/lib/recovery/kit-rules";
|
||||
|
||||
@@ -50,6 +54,7 @@ export const READINESS_SIGNALS = [
|
||||
"offhost_copies",
|
||||
"restore_drill",
|
||||
"offhost_setup",
|
||||
"dump_retention",
|
||||
"recovery_kit",
|
||||
] as const;
|
||||
export type ReadinessSignalId = (typeof READINESS_SIGNALS)[number];
|
||||
@@ -69,6 +74,7 @@ export const READINESS_FIX_PATHS: Record<ReadinessSignalId, string> = {
|
||||
offhost_copies: "/admin/backups",
|
||||
restore_drill: "/admin/backups",
|
||||
offhost_setup: "/admin/backups",
|
||||
dump_retention: "/admin/backups",
|
||||
recovery_kit: "/admin/encryption",
|
||||
};
|
||||
|
||||
@@ -121,6 +127,12 @@ export interface ReadinessInputs {
|
||||
};
|
||||
offhost: {
|
||||
configured: boolean;
|
||||
/**
|
||||
* The bucket's lifecycle rules for keys under `database/`, as the backups
|
||||
* page reads them (`probeOffhostLifecycle`). Null when no bucket is
|
||||
* configured.
|
||||
*/
|
||||
databaseLifecycle: LifecycleVerdict | null;
|
||||
accounts: ReadonlyArray<{
|
||||
lastAttemptAt: Date | null;
|
||||
lastSuccessAt: Date | null;
|
||||
@@ -185,19 +197,24 @@ export function evaluateDatabaseDump(
|
||||
// directory, no bucket) switched off what it could not do, which is not
|
||||
// news.
|
||||
const neverUsed = dump.lastGood === null && dump.lastFailure === null;
|
||||
const cannotWrite = !dump.localDirWritable && !dump.offhostConfigured;
|
||||
const cannotWrite = !dump.localPersistent && !dump.offhostConfigured;
|
||||
return neverUsed && cannotWrite
|
||||
? signal(id, "inactive", "disabled", null)
|
||||
: signal(id, "attention", "disabled", lastGoodAt);
|
||||
}
|
||||
const lastFailureAt = dump.lastFailure ? new Date(dump.lastFailure.at) : null;
|
||||
const failureReason = dump.lastFailure?.error ?? null;
|
||||
// A directory this process cannot write is a valid setup when the dumps go
|
||||
// to the bucket; with no bucket either, no dump can be kept anywhere.
|
||||
if (!dump.localDirWritable && !dump.offhostConfigured) {
|
||||
return signal(id, "failing", "nowhere_to_write", lastGoodAt, {
|
||||
directory: dump.directory,
|
||||
});
|
||||
// A directory that keeps nothing (not writable, or writable inside the
|
||||
// container and lost at the next update) is a valid setup when the dumps
|
||||
// go to the bucket; with no bucket either, no dump is kept anywhere.
|
||||
if (!dump.localPersistent && !dump.offhostConfigured) {
|
||||
return signal(
|
||||
id,
|
||||
"failing",
|
||||
dump.localDirWritable ? "not_persistent" : "nowhere_to_write",
|
||||
lastGoodAt,
|
||||
{ directory: dump.directory },
|
||||
);
|
||||
}
|
||||
const limitMs = DUMP_FAILING_AFTER_HOURS * HOUR_MS;
|
||||
const failedSinceGood =
|
||||
@@ -242,7 +259,13 @@ export function evaluateDatabaseDump(
|
||||
failureReason,
|
||||
);
|
||||
}
|
||||
return signal(id, "ok", "fresh", lastGoodAt);
|
||||
// Fine, and worth saying: the bucket holds the only copy, and no dump is
|
||||
// taken before a migration.
|
||||
return dump.localPersistent
|
||||
? signal(id, "ok", "fresh", lastGoodAt)
|
||||
: signal(id, "ok", "fresh_offhost_only", lastGoodAt, {
|
||||
directory: dump.directory,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -374,6 +397,42 @@ export function evaluateOffhostSetup(
|
||||
: signal(id, "ok", "not_configured_grace", graceEnds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the bucket ever removes the database dumps. Each one holds every
|
||||
* account that existed when it was taken, so a bucket with no rule for
|
||||
* `database/` keeps a deleted account's data for as long as the bucket
|
||||
* exists. Attention, not failing: every restore still works. A bucket that
|
||||
* will not say (a credential without the permission, a target without the
|
||||
* call) is not news either way.
|
||||
*/
|
||||
export function evaluateDumpRetention(
|
||||
input: Pick<ReadinessInputs, "dump" | "offhost">,
|
||||
): ReadinessSignal {
|
||||
const id = "dump_retention";
|
||||
if (!input.offhost.configured) {
|
||||
return signal(id, "inactive", "not_configured", null);
|
||||
}
|
||||
const { dump } = input;
|
||||
const dumpsGoOffhost =
|
||||
dump !== null &&
|
||||
dump !== "unreadable" &&
|
||||
dump.enabled &&
|
||||
dump.offhostConfigured;
|
||||
if (!dumpsGoOffhost) {
|
||||
return signal(id, "inactive", "no_offhost_dumps", null);
|
||||
}
|
||||
const rule = input.offhost.databaseLifecycle;
|
||||
if (rule?.state === "missing") {
|
||||
return signal(id, "attention", "missing", null);
|
||||
}
|
||||
if (rule?.state === "configured" && rule.expirationDays !== null) {
|
||||
return signal(id, "ok", "configured", null, {
|
||||
days: rule.expirationDays,
|
||||
});
|
||||
}
|
||||
return signal(id, "ok", "unknown", null);
|
||||
}
|
||||
|
||||
export function evaluateRecoveryKit(
|
||||
input: Pick<ReadinessInputs, "now" | "firstAdminAt" | "kit">,
|
||||
): ReadinessSignal {
|
||||
@@ -420,6 +479,7 @@ export function evaluateRecoveryReadiness(
|
||||
evaluateOffhostCopies(inputs),
|
||||
evaluateRestoreDrill(inputs),
|
||||
evaluateOffhostSetup(inputs),
|
||||
evaluateDumpRetention(inputs),
|
||||
evaluateRecoveryKit(inputs),
|
||||
];
|
||||
const state = signals.reduce<Exclude<ReadinessState, "inactive">>(
|
||||
|
||||
@@ -11,12 +11,17 @@
|
||||
* - the off-host copies: the per-account ledger the nightly run writes, and
|
||||
* the same "configured" rule the backups page uses (the environment, or a
|
||||
* ledger row as proof that an upload happened);
|
||||
* - the bucket's lifecycle rule for `database/`: the same cached probe the
|
||||
* backups page shows, asked only when a bucket is configured;
|
||||
* - the drill: its persisted last outcome, with no time window;
|
||||
* - the kit: the same assessment the kit card shows;
|
||||
* - the dump: whatever the dump module reports, through a provider.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
offhostBackupConfigured,
|
||||
probeOffhostLifecycle,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
readDatabaseBackupHealth,
|
||||
type DatabaseBackupHealthProvider,
|
||||
@@ -74,6 +79,10 @@ export async function readReadinessInputs(
|
||||
const configured =
|
||||
offhostBackupConfigured() ||
|
||||
accounts.some((account) => account.offhostBackupState !== null);
|
||||
// Never throws: a bucket that cannot be asked answers `unknown`.
|
||||
const databaseLifecycle = configured
|
||||
? (await probeOffhostLifecycle()).database
|
||||
: null;
|
||||
|
||||
return {
|
||||
now,
|
||||
@@ -88,6 +97,7 @@ export async function readReadinessInputs(
|
||||
},
|
||||
offhost: {
|
||||
configured,
|
||||
databaseLifecycle,
|
||||
accounts: accounts.map((account) => ({
|
||||
lastAttemptAt: account.offhostBackupState?.lastAttemptAt ?? null,
|
||||
lastSuccessAt: account.offhostBackupState?.lastSuccessAt ?? null,
|
||||
|
||||
@@ -107,14 +107,14 @@ export type DocumentSummaryStateValue =
|
||||
/**
|
||||
* Refs #776 — why a document's most recent content-index attempt produced no
|
||||
* index. Mirrors the free-String `lastIndexOutcome` column (closed here, not
|
||||
* in the schema, so a new reason needs no migration); the detail view renders
|
||||
* a plain-words line per value instead of a bare "not indexed" that cannot
|
||||
* tell "never tried" from "tried and failed".
|
||||
* in the schema, so a new reason needs no migration); the detail view says
|
||||
* each in plain words, so "never tried" and "tried and failed" differ.
|
||||
*/
|
||||
export const DOCUMENT_INDEX_OUTCOMES = [
|
||||
"local-empty",
|
||||
"local-unsupported",
|
||||
"decrypt-error",
|
||||
"resource-limit",
|
||||
"raster-failed",
|
||||
"provider-error",
|
||||
"pdf-needs-anthropic",
|
||||
|
||||
Reference in new issue
Block a user