From d839fbdb11dc9cffe281219beba5b3d5456756ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:34:40 +0100 Subject: [PATCH 01/44] fix(demo): lift the busy-stretch blood pressure far enough to read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The blood-pressure chart scales one axis to fit systolic and diastolic, so the 12 mmHg the busy stretch added came out as a nearly flat line on a 60 to 140 axis, even with the 7-day trend on. The stretch now climbs over two days to about 144/91, holds for some ten days, and comes back down over the following days after ramipril goes from 5 to 10 mg. A dose change and a family-practice visit on the day it turns carry that step, and the medication now reads 10 mg. The day-to-day walk is quieter, with a little reading scatter on top, so the 7-day mean draws an arc rather than jitter. Two limits keep the rest of the page honest. The lift stops short of 149 mmHg: one more and the padded axis rounds out to 50 to 200 and flattens the rise again. And the stretch stays well under half of the last 30 days: the usual range is the median ± 3 MAD of that window, and a longer stretch moved the median onto it and widened the band to about 87 to 181 mmHg. The stored texts that describe the last weeks mention the stretch and the dose increase. The seed-shape test holds the larger swing, the ceiling, the smaller day-to-day steps, the median of the window and the dose change on the turn. --- scripts/seed-demo.ts | 141 +++++++++++++++------- tests/integration/demo-seed-shape.test.ts | 87 +++++++++++-- 2 files changed, 177 insertions(+), 51 deletions(-) diff --git a/scripts/seed-demo.ts b/scripts/seed-demo.ts index f81712b1d..44eddabb2 100644 --- a/scripts/seed-demo.ts +++ b/scripts/seed-demo.ts @@ -9,7 +9,9 @@ * - 1 admin user (demo/demo123demo123) * - 90 days of measurements (weight, BP, pulse, resting HR, body fat, * sleep, steps, fortnightly waist) — flat-or-improving trends, - * most-recent point from today + * most-recent point from today; blood pressure has one busy stretch two + * to three weeks back (about 144/91 for some ten days) that comes down + * after a ramipril dose increase * - A full week of per-stage sleep nights (awake/REM/light(core)/deep), * ~7.5 h each, including last night * - 90 days of body composition (fat/lean/muscle mass, total body water, @@ -42,8 +44,9 @@ * temperature, period flow, fertile-window mucus/OPK, per-day symptoms, * and a cached forward prediction * - Two practitioners, a knee arthroscopy with its operative report and - * discharge letter linked (small generated PDFs), its follow-up visit, and - * a check-up booked for tomorrow morning + * discharge letter linked (small generated PDFs), its follow-up visit, the + * blood-pressure visit that raised ramipril, and a check-up booked for + * tomorrow morning * - An AI-configured Coach with five sample conversations spread from * today back over a few weeks * - App settings (registration disabled, English locale, the key-backup @@ -403,19 +406,45 @@ async function bulkInsert( /** * A busy stretch inside the 90-day window, in days before "now": blood - * pressure and pulse rise and settle again, and the mood dips with them. - * Without it every short-window series was flat noise around its target and a - * 7-day trend line over 30 days drew a straight line; with it the line has a - * shape to follow. It ends well before today, so the latest readings, the - * tiles and the stored texts still describe a settled, optimal band. + * pressure climbs over two days to about 144/91 and stays there for some ten + * days, pulse rises with it and the mood dips. At the family practice on + * `doseUp` ramipril goes from 5 to 10 mg, and over the following days the + * readings come back down to the settled band. Without it every short-window + * series was flat noise around its target, and on a blood-pressure axis that + * has to fit both systolic and diastolic, even a 15 mmHg swing read as a flat + * line. It ends a few days before today, so the latest readings, the tiles and + * the stored texts describe a settled, optimal band again. + * + * It is also no longer than this on purpose. The personal usual range is the + * median ± 3 MAD of the last 30 days: once the stretch fills about half of that + * window, the median lands on the stretch and the band reads something like + * 87-181 mmHg. Kept to this length the band stays around 107-132. */ -const BUSY_STRETCH = { from: 27, to: 12 } as const; +const BUSY_STRETCH = { from: 21, peak: 19, doseUp: 11, settled: 6 } as const; -/** 0 outside the busy stretch, rising smoothly to 1 at its middle. */ +/** + * Systolic, diastolic and pulse on top of the settled band at the stretch's + * height. The chart pads its axis and rounds it to even ticks: with systolic + * readings up to about 148 the 30-day axis runs 60 to 160 in steps of 20, one + * mmHg more and it jumps to 50 to 200 in steps of 50, which flattens the very + * rise the stretch is there to show. So the lift stops short of that. + */ +const BUSY_STRETCH_LIFT = { sys: 25, dia: 15, pulse: 6 } as const; + +/** Ramipril dose after the stretch, in mg (from 5 mg before it). */ +const RAMIPRIL_STRETCH_DOSE = 10; + +/** + * 0 outside the busy stretch, 1 while it holds: a smooth climb from `from` to + * `peak`, level until the dose change on `doseUp`, a smooth fall to `settled`. + */ function busyStretch(daysBefore: number): number { - const { from, to } = BUSY_STRETCH; - if (daysBefore > from || daysBefore < to) return 0; - return Math.sin((Math.PI * (from - daysBefore)) / (from - to)) ** 2; + const { from, peak, doseUp, settled } = BUSY_STRETCH; + const smooth = (t: number) => t * t * (3 - 2 * t); + if (daysBefore >= from || daysBefore <= settled) return 0; + if (daysBefore > peak) return smooth((from - daysBefore) / (from - peak)); + if (daysBefore >= doseUp) return 1; + return smooth((daysBefore - settled) / (doseUp - settled)); } /** Pollen season bump: 0 outside `[from, to]`, peaking at `peak` mid-season. */ @@ -577,12 +606,14 @@ export async function seed() { // Weight: 86.5 → trending down to a stable ~82 (healthy BMI ~24.7 at 182cm) const weights = randomWalk(86.5, 82.0, span, 0.6); - // Systolic BP: 128 → settled in the optimal band ~118 (clamped) + // Systolic BP: 128 → settled in the optimal band ~118 (clamped). The day- + // to-day noise is kept small, so the 7-day trend over the busy stretch + // below draws an arc rather than jitter. const sysBP = randomWalk( 128, 118, span, - 3, + 1.2, VALUE_RANGES.BLOOD_PRESSURE_SYS, 0.1, ); @@ -591,7 +622,7 @@ export async function seed() { 82, 76, span, - 2.5, + 1, VALUE_RANGES.BLOOD_PRESSURE_DIA, 0.1, ); @@ -637,9 +668,21 @@ export async function seed() { Math.max(range.min, Math.round((series[i] + by) * 10) / 10), ); }; - lift(sysBP, busy * 12 + weekend * 2, VALUE_RANGES.BLOOD_PRESSURE_SYS); - lift(diaBP, busy * 7 + weekend * 1, VALUE_RANGES.BLOOD_PRESSURE_DIA); - lift(pulse, busy * 6, VALUE_RANGES.PULSE); + // A little reading-to-reading scatter on top of the slow walk, as a + // cuff has; the 7-day mean averages it out. + lift( + sysBP, + busy * BUSY_STRETCH_LIFT.sys + weekend * 1 + (Math.random() - 0.5) * 3, + VALUE_RANGES.BLOOD_PRESSURE_SYS, + ); + lift( + diaBP, + busy * BUSY_STRETCH_LIFT.dia + + weekend * 0.5 + + (Math.random() - 0.5) * 2, + VALUE_RANGES.BLOOD_PRESSURE_DIA, + ); + lift(pulse, busy * BUSY_STRETCH_LIFT.pulse, VALUE_RANGES.PULSE); } for (const series of [weights, restingHr]) { settleLatest(series); @@ -647,7 +690,7 @@ export async function seed() { // Today's reading settles on the days after the busy stretch only, so it // describes the band the series came back to. for (const series of [sysBP, diaBP, pulse]) { - settleLatest(series, BUSY_STRETCH.to - 1); + settleLatest(series, BUSY_STRETCH.settled); } // Each series carries the source that writes it on a real account, the @@ -1406,19 +1449,21 @@ export async function seed() { console.log("Creating medications..."); // Medication 1: Ramipril (blood pressure). Started about three years - // back at 2.5 mg and raised to 5 mg a year later; the multi-year history - // section below writes the dose changes, a short sick-day pause and the - // intakes before the 90-day window. `starts_on` is the stated start the - // timeline lane draws from. + // back at 2.5 mg, raised to 5 mg a year later, and to 10 mg during the + // busy stretch inside the 90-day window; the multi-year history section + // below writes the dose changes, a short sick-day pause and the intakes + // before the 90-day window. `starts_on` is the stated start the timeline + // lane draws from. const med1Id = cuid(); await client.query( `INSERT INTO medications (id, user_id, name, dose, active, notifications_enabled, starts_on, created_at, updated_at) - VALUES ($1, $2, 'Ramipril', '5mg', true, true, $3, $4, $4)`, + VALUES ($1, $2, 'Ramipril', $5, true, true, $3, $4, $4)`, [ med1Id, userId, formatDate(daysAgo(RAMIPRIL_START_DAYS_AGO)), daysAgo(RAMIPRIL_START_DAYS_AGO), + `${RAMIPRIL_STRETCH_DOSE}mg`, ], ); const sched1Id = cuid(); @@ -2311,6 +2356,16 @@ export async function seed() { reason: "Follow-up two weeks after the arthroscopy.", outcome: "Healing well, full range of motion. Physiotherapy continues.", }); + // The visit that turned the busy stretch: ten days of high home readings, + // ramipril raised that morning. + await insertEncounter({ + occurredAt: daysAgoAt(BUSY_STRETCH.doseUp, 8, 15), + status: "DONE", + kind: "ROUTINE", + practitionerId: familyPracticeId, + reason: "Home blood pressure high for about ten days.", + outcome: `Readings around 144/91 during a stressful stretch at work. Ramipril raised to ${RAMIPRIL_STRETCH_DOSE} mg, home readings again in two weeks.`, + }); await insertEncounter({ occurredAt: daysAgoAt(-1, 9, 30), status: "PLANNED", @@ -2633,10 +2688,12 @@ export async function seed() { intakeRows, ); - // The ramipril dose history and its sick-day pause. + // The ramipril dose history and its sick-day pause. The last step falls + // inside the 90-day window, on the day the busy stretch turns. for (const [n, value] of [ [RAMIPRIL_START_DAYS_AGO, 2.5], [RAMIPRIL_DOSE_UP_DAYS_AGO, 5], + [BUSY_STRETCH.doseUp, RAMIPRIL_STRETCH_DOSE], ] as const) { await client.query( `INSERT INTO medication_dose_changes (id, medication_id, effective_from, dose_value, dose_unit, created_at) @@ -3353,9 +3410,10 @@ export async function seed() { content: "How has my blood pressure been trending lately?", }, answer( - "Your blood pressure has been settling nicely. After a higher " + - "stretch about three weeks ago, the morning readings have come back " + - "down into the optimal band, and " + + "Your blood pressure has come back down nicely. During the stressful " + + "stretch two to three weeks ago your morning readings sat around " + + "144/91 for about ten days; since ramipril went up to 10 mg they " + + "have eased back into the optimal band within about a week, and " + "your resting heart rate is sitting in a comfortable range too. " + "Keeping up the steady sleep and daily movement looks like it is " + "paying off — worth staying consistent with the morning measurements " + @@ -3633,10 +3691,11 @@ export async function seed() { keyFindings: [ { tone: "good", - headline: "Blood pressure has settled into the optimal range.", + headline: "Blood pressure is back in the optimal range.", detail: - "Your 30-day average is well inside the optimal band and lower " + - "than where the quarter began.", + "After a higher stretch two to three weeks ago and a ramipril " + + "dose increase, your readings have come back down below where " + + "the quarter began.", delta: "↓ ~10 mmHg", sourceWindow: "30d", sourceMetric: "bp", @@ -3674,7 +3733,7 @@ export async function seed() { ], }, trendAnnotations: { - bp: "Your systolic is trending down into the optimal band — a pattern worth keeping.", + bp: "After a higher stretch and a ramipril dose increase, your systolic has come back down into the optimal band.", weight: "Weight has levelled off after an earlier decline — a stable, healthy place to hold.", mood: "Mood has been on a quiet upward run across the quarter.", @@ -3727,11 +3786,10 @@ export async function seed() { { scope: "blood-pressure", text: - "Your blood pressure is averaging about 118/76 over the last two " + - "weeks, back down after a busier stretch before that. It sits " + - "roughly 10 mmHg systolic below where this quarter started, " + - "comfortably inside the optimal band. That is a real, earned " + - "shift, not day-to-day noise. Keeping the morning " + + "Your blood pressure is back around 118/76 this week. Two to three " + + "weeks ago it climbed to about 144/91 and held there for about ten " + + "days; after ramipril went up to 10 mg it came down over the " + + "following days and now sits inside the optimal band again. Keeping the morning " + "readings on a steady schedule will make the trend even easier to " + "follow.", }, @@ -3846,8 +3904,9 @@ export async function seed() { period: "month", text: "Over the past month the gains from earlier in the quarter have " + - "become your new normal. Blood pressure has held in the optimal " + - "band rather than just dipping into it, weight has stabilised at a " + + "become your new normal. Blood pressure had one higher stretch, " + + "came back down after the ramipril dose went up and now sits in the " + + "optimal band again, weight has stabilised at a " + "healthy point, and resting heart rate has stayed low alongside " + "consistent daily activity. Sleep lengthened and steadied, and " + "your mood trend continued its quiet climb. The month reads less " + @@ -3962,7 +4021,7 @@ export async function seed() { ` Cycle: ${cycleLengths.length} cycles with BBT, flow, mucus/OPK, symptoms + a forecast`, ); console.log( - ` Visits: knee arthroscopy (2 linked PDFs) + follow-up, check-up booked for tomorrow`, + ` Visits: knee arthroscopy (2 linked PDFs) + follow-up, a blood-pressure visit, check-up booked for tomorrow`, ); console.log(` Coach: ${coachConversations.length} conversations`); console.log( diff --git a/tests/integration/demo-seed-shape.test.ts b/tests/integration/demo-seed-shape.test.ts index 849b1d9af..d35aa6742 100644 --- a/tests/integration/demo-seed-shape.test.ts +++ b/tests/integration/demo-seed-shape.test.ts @@ -3,14 +3,20 @@ * the seed is a mean-reverting walk, and around a settled target that is flat * noise: a 7-day trend line over the last 30 days of blood pressure drew a * straight line, which is exactly the view the demo exists to show off. The - * seed now carries a busy stretch a few weeks back that lifts blood pressure - * and lets it settle again. + * seed carries a busy stretch a few weeks back that lifts blood pressure to + * about 144/91 for some ten days, and ramipril goes up on the day it + * turns. * - * This runs the real `scripts/seed-demo.ts` and holds the last 30 days of - * systolic readings to two things at once: the 7-day mean moves visibly, and - * today's reading has come back down from the stretch. Before the stretch the - * swing measured 2 to 3 mmHg; with it, about 8 to 14. The seed is random, so the - * bounds sit between the two rather than at either. + * The blood-pressure chart scales one axis to fit systolic and diastolic, so a + * swing has to be large to read: a first version lifted systolic by 12 mmHg, + * its 7-day mean swung 8 to 14 mmHg, and on a 60 to 140 axis that still drew a + * nearly flat line. This runs the real `scripts/seed-demo.ts` and holds the + * last 30 days to a swing of the size that reads, to day-to-day noise small + * enough for the 7-day mean to draw an arc, to today's reading back in the + * settled band, and to the dose change landing on the turn. The seed is random, + * so the bounds leave room around the measured values (swing about 23 mmHg, + * day-to-day steps around 1 to 2 mmHg outside the climb and the fall). The + * peak also has a ceiling: past 148 the chart's padded axis jumps to 50-200. */ import { beforeAll, describe, expect, it, vi } from "vitest"; @@ -21,7 +27,9 @@ vi.mock("@/lib/db-compat", () => ({ })); let systolic: number[]; +let diastolic: number[]; let means: number[]; +let doseChanges: Array<{ effectiveFrom: Date; doseValue: number }>; beforeAll(async () => { const prisma = getPrismaClient(); @@ -45,6 +53,30 @@ beforeAll(async () => { select: { value: true }, }); systolic = rows.map((r) => Number(r.value)); + const diaRows = await prisma.measurement.findMany({ + where: { + userId: user.id, + type: "BLOOD_PRESSURE_DIA", + source: "WITHINGS", + deletedAt: null, + measuredAt: { gte: since }, + }, + orderBy: { measuredAt: "asc" }, + select: { value: true }, + }); + diastolic = diaRows.map((r) => Number(r.value)); + const ramipril = await prisma.medication.findFirstOrThrow({ + where: { userId: user.id, name: "Ramipril" }, + select: { id: true }, + }); + const changes = await prisma.medicationDoseChange.findMany({ + where: { medicationId: ramipril.id, effectiveFrom: { gte: since } }, + select: { effectiveFrom: true, doseValue: true }, + }); + doseChanges = changes.map((c) => ({ + effectiveFrom: c.effectiveFrom, + doseValue: Number(c.doseValue), + })); means = []; for (let i = 6; i < systolic.length; i++) { const week = systolic.slice(i - 6, i + 1); @@ -53,13 +85,48 @@ beforeAll(async () => { }, 180_000); describe("demo seed, short-window shape", () => { - it("gives the 7-day systolic mean over 30 days a visible rise and fall", () => { + it("gives the 7-day systolic mean over 30 days a rise and fall that reads", () => { expect(systolic.length).toBeGreaterThanOrEqual(28); const swing = Math.max(...means) - Math.min(...means); - expect(swing).toBeGreaterThanOrEqual(6); + expect(swing).toBeGreaterThanOrEqual(18); + expect(Math.max(...systolic)).toBeGreaterThanOrEqual(140); + // One mmHg past 148 and the padded axis rounds out to 50-200 in steps of + // 50, which flattens the rise again. + expect(Math.max(...systolic)).toBeLessThanOrEqual(148); + expect(Math.max(...diastolic)).toBeGreaterThanOrEqual(88); + }); + + it("keeps the day-to-day steps small enough for the 7-day mean to draw an arc", () => { + const steps = systolic.slice(1).map((v, i) => Math.abs(v - systolic[i])); + const sorted = [...steps].sort((a, b) => a - b); + expect(sorted[Math.floor(sorted.length / 2)]).toBeLessThanOrEqual(3); + }); + + it("leaves the median of the window in the settled band", () => { + // The usual range is the median ± 3 MAD of the last 30 days. A stretch + // that fills half the window moves that median onto the stretch and + // widens the band to something like 87-181 mmHg. + const sorted = [...systolic].sort((a, b) => a - b); + const median = sorted[Math.floor(sorted.length / 2)]; + expect(median).toBeLessThanOrEqual(124); + const deviations = systolic + .map((v) => Math.abs(v - median)) + .sort((a, b) => a - b); + const mad = deviations[Math.floor(deviations.length / 2)]; + expect(3 * 1.4826 * mad).toBeLessThanOrEqual(20); + }); + + it("raises ramipril on the day the stretch turns", () => { + expect(doseChanges).toHaveLength(1); + expect(doseChanges[0].doseValue).toBe(10); + const daysBefore = + (Date.now() - doseChanges[0].effectiveFrom.getTime()) / 86_400_000; + expect(daysBefore).toBeGreaterThan(8); + expect(daysBefore).toBeLessThan(14); }); it("brings today's systolic reading back down from the stretch", () => { - expect(systolic.at(-1)).toBeLessThanOrEqual(Math.max(...means) - 3); + expect(systolic.at(-1)).toBeLessThanOrEqual(Math.max(...means) - 15); + expect(systolic.at(-1)).toBeLessThanOrEqual(125); }); }); From 28340e7c66210f2a47b29f53238d472c83904152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:42:37 +0100 Subject: [PATCH 02/44] fix(coach): restore every step a turn records on reload The turn records up to 64 steps while the read path kept only the first 48, so the tail of a long answer's step list vanished on reload. Both ends now read one MAX_TURN_STEPS constant, and a unit plus an integration test round-trip a full 64-step answer through the stored provenance. --- .../baselines/version-stamp-comments.json | 4 +- .../ai/coach/__tests__/persistence.test.ts | 43 ++++++++- src/lib/ai/coach/activity/contract.ts | 7 ++ src/lib/ai/coach/persistence.ts | 15 ++- src/lib/ai/coach/turn/__tests__/steps.test.ts | 3 +- src/lib/ai/coach/turn/steps.ts | 8 +- .../coach-steps-round-trip.test.ts | 96 +++++++++++++++++++ 7 files changed, 157 insertions(+), 19 deletions(-) create mode 100644 tests/integration/coach-steps-round-trip.test.ts diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 81a01b793..5a3506c2e 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2897, - "total": 9249, + "total": 9247, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -1737,7 +1737,7 @@ "lib/ai/coach/turn/reply-guards.ts": 18, "lib/ai/coach/turn/reuse-turn.ts": 1, "lib/ai/coach/turn/sse.ts": 10, - "lib/ai/coach/turn/steps.ts": 3, + "lib/ai/coach/turn/steps.ts": 1, "lib/ai/coach/turn/types.ts": 17, "lib/ai/coach/types.ts": 58, "lib/ai/coach/visits-snapshot.ts": 1, diff --git a/src/lib/ai/coach/__tests__/persistence.test.ts b/src/lib/ai/coach/__tests__/persistence.test.ts index 5e555e918..0075ef193 100644 --- a/src/lib/ai/coach/__tests__/persistence.test.ts +++ b/src/lib/ai/coach/__tests__/persistence.test.ts @@ -27,7 +27,8 @@ import { summariseTitle, } from "../persistence"; import { encryptToBytes } from "../bytes-codec"; -import type { CoachProvenance } from "../types"; +import { MAX_TURN_STEPS } from "../activity/contract"; +import type { CoachProvenance, CoachStep } from "../types"; describe("summariseTitle", () => { it("returns the input unchanged when below 80 chars", () => { @@ -212,6 +213,46 @@ describe("appendMessage — provenance round-trip", () => { expect(out.metricSource).toEqual(envelope); }); + it("restores every step the turn may record, and none past the cap", async () => { + const step = (n: number): CoachStep => ({ + id: `s${n}`, + tool: "get_metric_series", + labelKey: "coach.step.readWindow", + label: "Read the series", + domain: "bp", + window: "last30days", + status: "done", + count: n, + }); + const recorded = Array.from({ length: MAX_TURN_STEPS }, (_, i) => + step(i + 1), + ); + stubEchoCreate(); + const full = await appendMessage({ + conversationId: "conv_1", + role: "assistant", + content: "A long answer.", + metricSource: { windows: [], metrics: [], steps: recorded }, + }); + expect(full.metricSource?.steps).toEqual(recorded); + + // A stored blob longer than the cap is cut at the cap, not before it. + stubEchoCreate( + JSON.stringify({ + windows: [], + metrics: [], + steps: [...recorded, step(MAX_TURN_STEPS + 1)], + }), + ); + const over = await appendMessage({ + conversationId: "conv_1", + role: "assistant", + content: "An over-long blob.", + metricSource: { windows: [], metrics: [] }, + }); + expect(over.metricSource?.steps).toEqual(recorded); + }); + it("tolerates a legacy blob without the new fields (no throw, fields absent)", async () => { // A pre-feature row: only the original windows/metrics envelope on disk. stubEchoCreate( diff --git a/src/lib/ai/coach/activity/contract.ts b/src/lib/ai/coach/activity/contract.ts index cca931799..d4b4d65e2 100644 --- a/src/lib/ai/coach/activity/contract.ts +++ b/src/lib/ai/coach/activity/contract.ts @@ -41,6 +41,13 @@ export const ACTIVITY_TEXT_MAX_CHARS = 400; /** `a1`..`a99`: the 100th entry of a turn is not recorded. */ export const ACTIVITY_MAX_ENTRIES = 99; +/** + * `s1`..`s64`: the live steps of one turn (four per round of the longest + * turn). Later tool calls still run but show no step. The turn records up to + * this many and a reload restores up to this many, from this one constant. + */ +export const MAX_TURN_STEPS = 64; + export interface ActivityStart { phase: CoachActivityPhase; round: number; diff --git a/src/lib/ai/coach/persistence.ts b/src/lib/ai/coach/persistence.ts index 787d4e09f..a40ffd939 100644 --- a/src/lib/ai/coach/persistence.ts +++ b/src/lib/ai/coach/persistence.ts @@ -28,7 +28,11 @@ import { coachStopSchema, coachTrailSchema, } from "./stream-events"; -import { ACTIVITY_MAX_ENTRIES, TRAIL_MAX_BYTES } from "./activity/contract"; +import { + ACTIVITY_MAX_ENTRIES, + MAX_TURN_STEPS, + TRAIL_MAX_BYTES, +} from "./activity/contract"; import { commitTurnWrites } from "./memory/turn-writes"; import { COACH_CONVERSATION_TITLE_MAX } from "./types"; import { isRedundantViewChip } from "./follow-ups/view-chip"; @@ -56,7 +60,6 @@ import type { CoachResultEntry, CoachResultMeta, CoachResultTable, - CoachStep, CoachStepDomain, CoachSuggestion, CoachTrail, @@ -225,11 +228,6 @@ function restoreOne(raw: unknown, schema: z.ZodType): T | undefined { return parsed.success ? parsed.data : undefined; } -/** The live steps a turn persisted (at most 48 per turn since v1.41). */ -function restoreSteps(raw: unknown): CoachStep[] | undefined { - return restoreEach(raw, coachStepSchema, 48); -} - /** The method line; dropped whole when it does not parse. */ function restoreMethod(raw: unknown): CoachMethod | undefined { const parsed = coachMethodSchema.safeParse(raw); @@ -341,7 +339,8 @@ function provenanceFromJson(raw: string | null): CoachProvenance | null { const toolCalls = restoreToolCalls(parsed.toolCalls); // v1.39.4 — the dialog fields: steps, method, table metadata, chips, the // clarification choices, and the forced-answer marker. - const steps = restoreSteps(parsed.steps); + // Up to the same cap the turn records steps under. + const steps = restoreEach(parsed.steps, coachStepSchema, MAX_TURN_STEPS); const method = restoreMethod(parsed.method); const results = restoreResultMetas(parsed.results); // A reply stored before view chips were withheld for a table the answer diff --git a/src/lib/ai/coach/turn/__tests__/steps.test.ts b/src/lib/ai/coach/turn/__tests__/steps.test.ts index b5453ed75..06a297fbf 100644 --- a/src/lib/ai/coach/turn/__tests__/steps.test.ts +++ b/src/lib/ai/coach/turn/__tests__/steps.test.ts @@ -34,7 +34,8 @@ import { } from "@/lib/ai/coach/tools/definitions"; import type { CoachToolResult } from "@/lib/ai/coach/tools/run/types"; -import { MAX_TURN_STEPS, snapshotStep, toStep } from "../steps"; +import { MAX_TURN_STEPS } from "@/lib/ai/coach/activity/contract"; +import { snapshotStep, toStep } from "../steps"; const SOURCES = coachScopeSourceSchema.options; const WINDOWS = coachScopeWindowSchema.options; diff --git a/src/lib/ai/coach/turn/steps.ts b/src/lib/ai/coach/turn/steps.ts index 87aa40787..09b6153df 100644 --- a/src/lib/ai/coach/turn/steps.ts +++ b/src/lib/ai/coach/turn/steps.ts @@ -38,13 +38,7 @@ import { } from "@/lib/ai/coach/tools/definitions"; import { isTurnResultRef } from "@/lib/ai/coach/results/refs"; import { COMPARE_SERIES_TOOL_NAME } from "@/lib/ai/coach/tools/compare-series"; - -/** - * At most this many steps per turn; later calls run but show no step. - * v1.41 — four per round of the longest turn: 48 for twelve rounds, 64 since - * v1.41.2 raised the cap to sixteen. - */ -export const MAX_TURN_STEPS = 64; +import { MAX_TURN_STEPS } from "@/lib/ai/coach/activity/contract"; const SCOPE_SOURCES: ReadonlySet = new Set( coachScopeSourceSchema.options, diff --git a/tests/integration/coach-steps-round-trip.test.ts b/tests/integration/coach-steps-round-trip.test.ts new file mode 100644 index 000000000..0abdf9906 --- /dev/null +++ b/tests/integration/coach-steps-round-trip.test.ts @@ -0,0 +1,96 @@ +/** + * The steps of a long Coach answer survive a reload against a real Postgres. + * + * The turn records up to `MAX_TURN_STEPS` steps on `metricSource.steps`; the + * read path restores them from `coach_messages.metric_source_json`. Both ends + * read the one constant, so a full turn's worth comes back whole and an + * over-long stored blob is cut at that cap, not before it. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { MAX_TURN_STEPS } from "@/lib/ai/coach/activity/contract"; +import { + appendMessage, + createConversation, + fetchConversationWithMessages, +} from "@/lib/ai/coach/persistence"; +import type { CoachStep } from "@/lib/ai/coach/types"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); +}); + +function step(n: number): CoachStep { + return { + id: `s${n}`, + tool: "get_metric_series", + labelKey: "coach.step.readWindow", + label: "Read the last 30 days", + domain: "bp", + window: "last30days", + status: "done", + count: n, + }; +} + +const RECORDED = Array.from({ length: MAX_TURN_STEPS }, (_, i) => step(i + 1)); + +async function seedConversation(): Promise<{ + userId: string; + conversationId: string; +}> { + const user = await getPrismaClient().user.create({ + data: { username: "steps-owner", role: "USER" }, + }); + const convo = await createConversation({ userId: user.id, title: "long" }); + return { userId: user.id, conversationId: convo.id }; +} + +describe("Coach steps across a reload", () => { + it("restores every step of a full turn", async () => { + const { userId, conversationId } = await seedConversation(); + const reply = await appendMessage({ + conversationId, + role: "assistant", + content: "A long answer.", + metricSource: { windows: [], metrics: [], steps: RECORDED }, + }); + + const reloaded = await fetchConversationWithMessages( + userId, + conversationId, + ); + const message = reloaded?.messages.find((m) => m.id === reply.id); + expect(message?.metricSource?.steps).toHaveLength(MAX_TURN_STEPS); + expect(message?.metricSource?.steps).toEqual(RECORDED); + }); + + it("cuts an over-long stored blob at the cap, not before it", async () => { + const { userId, conversationId } = await seedConversation(); + const reply = await appendMessage({ + conversationId, + role: "assistant", + content: "An over-long blob.", + metricSource: { windows: [], metrics: [] }, + }); + await getPrismaClient().coachMessage.update({ + where: { id: reply.id }, + data: { + metricSourceJson: JSON.stringify({ + windows: [], + metrics: [], + steps: [...RECORDED, step(MAX_TURN_STEPS + 1)], + }), + }, + }); + + const reloaded = await fetchConversationWithMessages( + userId, + conversationId, + ); + const message = reloaded?.messages.find((m) => m.id === reply.id); + expect(message?.metricSource?.steps).toEqual(RECORDED); + }); +}); From 856eb036f83a284e515aafbc23e7bf40de95bb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:44:42 +0100 Subject: [PATCH 03/44] fix(documents): point the on-device scan hint at the Labs switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hint shown when a text-only provider needs on-device text recognition sent people to the AI settings, where no such switch exists. It now names the switch by its label under Settings → Modules → Labs in every locale and links straight to that card. Inside somebody else's record the link is withheld, since Settings there is the caller's own and closed; the sentence stays. --- messages/de.json | 5 +- messages/en.json | 5 +- messages/es.json | 5 +- messages/fr.json | 5 +- messages/it.json | 5 +- messages/ko.json | 5 +- messages/nl.json | 5 +- messages/pl.json | 5 +- messages/pt-BR.json | 5 +- .../__tests__/document-ai-panels.test.tsx | 29 ++++++++++-- .../__tests__/document-ai-section.test.tsx | 7 ++- .../documents/document-ai-panels.tsx | 47 +++++++++++++------ 12 files changed, 90 insertions(+), 38 deletions(-) diff --git a/messages/de.json b/messages/de.json index cfe7abcc4..60821114e 100644 --- a/messages/de.json +++ b/messages/de.json @@ -11769,8 +11769,9 @@ "dismiss": "Verwerfen", "empty": "Aus diesem Dokument konnte nichts gelesen werden.", "unavailableBody": "Richte einen KI-Anbieter ein, um Titel, Typ und Datum vorzuschlagen.", - "unavailableLocalOcr": "Aktiviere lokale Texterkennung in den KI-Einstellungen, um Bilder auf diesem Gerät zu lesen.", + "unavailableLocalOcr": "Schalte „Scans auf diesem Gerät lesen“ unter Einstellungen → Module → Labor ein, um Bilder auf diesem Gerät zu lesen.", "unavailableAction": "KI-Einstellungen öffnen", + "unavailableLocalOcrAction": "Labor-Einstellungen öffnen", "errorGeneric": "Dokument konnte nicht gelesen werden. Bitte erneut versuchen.", "errorOcr": "Aus diesem Bild konnte kein Text gelesen werden.", "errorRateLimited": "Das stündliche KI-Leselimit ist erreicht. Bitte später erneut versuchen.", @@ -11782,7 +11783,7 @@ "errorFileType": "Dieser Dateityp kann nicht gelesen werden.", "errorNotIndexed": "Lies das Dokument zuerst aus, dann extrahiere.", "errorOutputLimit": "Der KI-Anbieter hat sein Ausgabelimit erreicht, bevor er einen einzigen Wert geliefert hat. Versuche ein Modell, das längere Antworten erlaubt.", - "errorLocalOcrDisabled": "Aktiviere zuerst die lokale Texterkennung in den KI-Einstellungen.", + "errorLocalOcrDisabled": "Schalte zuerst „Scans auf diesem Gerät lesen“ unter Einstellungen → Module → Labor ein.", "errorProvider": "Dafür ist kein KI-Anbieter eingerichtet.", "errorTextImageOnly": "Lokale Texterkennung liest nur Bilder, nicht diese Datei.", "errorConsent": "Für das Lesen von Dokumenten mit KI brauchst du zuerst deine Einwilligung.", diff --git a/messages/en.json b/messages/en.json index 9e7fa4317..53b89951f 100644 --- a/messages/en.json +++ b/messages/en.json @@ -11769,8 +11769,9 @@ "dismiss": "Dismiss", "empty": "Nothing could be read from this document.", "unavailableBody": "Set up an AI provider to suggest a title, type, and date.", - "unavailableLocalOcr": "Turn on local OCR in the AI settings to read images on this device.", + "unavailableLocalOcr": "Turn on “Read scans on this device” under Settings → Modules → Labs to read images on this device.", "unavailableAction": "Open AI settings", + "unavailableLocalOcrAction": "Open lab settings", "errorGeneric": "Couldn't read the document. Try again.", "errorOcr": "Couldn't read text from this image.", "errorRateLimited": "The hourly AI reading limit is reached. Try again later.", @@ -11782,7 +11783,7 @@ "errorFileType": "This file type can't be read.", "errorNotIndexed": "Read the document first, then extract.", "errorOutputLimit": "The AI provider reached its output limit before it returned a single value. Try a model that allows longer answers.", - "errorLocalOcrDisabled": "Turn on local OCR in the AI settings first.", + "errorLocalOcrDisabled": "Turn on “Read scans on this device” under Settings → Modules → Labs first.", "errorProvider": "No AI provider is set up for this.", "errorTextImageOnly": "Local OCR reads images only, not this file.", "errorConsent": "Reading documents with AI needs your consent first.", diff --git a/messages/es.json b/messages/es.json index 55dd09b69..be842de69 100644 --- a/messages/es.json +++ b/messages/es.json @@ -11769,8 +11769,9 @@ "dismiss": "Descartar", "empty": "No se pudo leer nada de este documento.", "unavailableBody": "Configura un proveedor de IA para sugerir título, tipo y fecha.", - "unavailableLocalOcr": "Activa el OCR local en los ajustes de IA para leer imágenes en este dispositivo.", + "unavailableLocalOcr": "Activa «Leer escaneos en este dispositivo» en Ajustes → Módulos → Laboratorio para leer imágenes en este dispositivo.", "unavailableAction": "Abrir ajustes de IA", + "unavailableLocalOcrAction": "Abrir ajustes de laboratorio", "errorGeneric": "No se pudo leer el documento. Inténtalo de nuevo.", "errorOcr": "No se pudo leer texto de esta imagen.", "errorRateLimited": "Se alcanzó el límite de lecturas de IA por hora. Inténtalo más tarde.", @@ -11782,7 +11783,7 @@ "errorFileType": "Este tipo de archivo no se puede leer.", "errorNotIndexed": "Primero lee el documento y luego extrae.", "errorOutputLimit": "El proveedor de IA alcanzó su límite de salida antes de devolver un solo valor. Prueba con un modelo que permita respuestas más largas.", - "errorLocalOcrDisabled": "Activa primero el OCR local en los ajustes de IA.", + "errorLocalOcrDisabled": "Activa primero «Leer escaneos en este dispositivo» en Ajustes → Módulos → Laboratorio.", "errorProvider": "No hay ningún proveedor de IA configurado para esto.", "errorTextImageOnly": "El OCR local solo lee imágenes, no este archivo.", "errorConsent": "Leer documentos con IA necesita primero tu consentimiento.", diff --git a/messages/fr.json b/messages/fr.json index d74a6043b..e009d81f5 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -11769,8 +11769,9 @@ "dismiss": "Ignorer", "empty": "Impossible de lire ce document.", "unavailableBody": "Configurez un fournisseur d'IA pour suggérer un titre, un type et une date.", - "unavailableLocalOcr": "Activez l'OCR local dans les paramètres d'IA pour lire les images sur cet appareil.", + "unavailableLocalOcr": "Activez « Lire les scans sur cet appareil » dans Paramètres → Modules → Analyses pour lire les images sur cet appareil.", "unavailableAction": "Ouvrir les paramètres d'IA", + "unavailableLocalOcrAction": "Ouvrir les paramètres des analyses", "errorGeneric": "Impossible de lire le document. Réessayez.", "errorOcr": "Impossible de lire le texte de cette image.", "errorRateLimited": "La limite horaire de lectures IA est atteinte. Réessayez plus tard.", @@ -11782,7 +11783,7 @@ "errorFileType": "Ce type de fichier ne peut pas être lu.", "errorNotIndexed": "Lisez d’abord le document, puis extrayez.", "errorOutputLimit": "Le fournisseur d’IA a atteint sa limite de sortie avant de renvoyer la moindre valeur. Essayez un modèle qui accepte des réponses plus longues.", - "errorLocalOcrDisabled": "Activez d'abord l'OCR local dans les paramètres d'IA.", + "errorLocalOcrDisabled": "Activez d'abord « Lire les scans sur cet appareil » dans Paramètres → Modules → Analyses.", "errorProvider": "Aucun fournisseur d'IA n'est configuré pour cela.", "errorTextImageOnly": "L'OCR local ne lit que les images, pas ce fichier.", "errorConsent": "La lecture de documents par IA nécessite d'abord votre consentement.", diff --git a/messages/it.json b/messages/it.json index b1de9b105..47004b65b 100644 --- a/messages/it.json +++ b/messages/it.json @@ -11769,8 +11769,9 @@ "dismiss": "Ignora", "empty": "Impossibile leggere questo documento.", "unavailableBody": "Configura un provider IA per suggerire titolo, tipo e data.", - "unavailableLocalOcr": "Attiva l'OCR locale nelle impostazioni IA per leggere le immagini su questo dispositivo.", + "unavailableLocalOcr": "Attiva «Leggi le scansioni su questo dispositivo» in Impostazioni → Moduli → Esami per leggere le immagini su questo dispositivo.", "unavailableAction": "Apri impostazioni IA", + "unavailableLocalOcrAction": "Apri le impostazioni degli esami", "errorGeneric": "Impossibile leggere il documento. Riprova.", "errorOcr": "Impossibile leggere il testo da questa immagine.", "errorRateLimited": "Il limite orario di letture IA è stato raggiunto. Riprova più tardi.", @@ -11782,7 +11783,7 @@ "errorFileType": "Questo tipo di file non può essere letto.", "errorNotIndexed": "Leggi prima il documento, poi estrai.", "errorOutputLimit": "Il fornitore di IA ha raggiunto il suo limite di output prima di restituire un solo valore. Prova con un modello che consente risposte più lunghe.", - "errorLocalOcrDisabled": "Attiva prima l'OCR locale nelle impostazioni IA.", + "errorLocalOcrDisabled": "Attiva prima «Leggi le scansioni su questo dispositivo» in Impostazioni → Moduli → Esami.", "errorProvider": "Nessun provider IA è configurato per questo.", "errorTextImageOnly": "L'OCR locale legge solo immagini, non questo file.", "errorConsent": "La lettura dei documenti con IA richiede prima il tuo consenso.", diff --git a/messages/ko.json b/messages/ko.json index 3fbdc541b..7e743c844 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -11769,8 +11769,9 @@ "dismiss": "닫기", "empty": "이 문서에서 읽어낼 내용이 없었어요.", "unavailableBody": "AI 제공자를 설정하면 제목·종류·날짜를 추천해요.", - "unavailableLocalOcr": "AI 설정에서 로컬 OCR을 켜면 이 기기에서 이미지를 읽어요.", + "unavailableLocalOcr": "설정 → 모듈 → 검사에서 “이 기기에서 스캔 읽기”를 켜면 이 기기에서 이미지를 읽어요.", "unavailableAction": "AI 설정 열기", + "unavailableLocalOcrAction": "검사 설정 열기", "errorGeneric": "문서를 읽지 못했어요. 다시 시도해 주세요.", "errorOcr": "이 이미지에서 텍스트를 읽지 못했어요.", "errorRateLimited": "시간당 AI 읽기 한도에 도달했어요. 나중에 다시 시도해 주세요.", @@ -11782,7 +11783,7 @@ "errorFileType": "이 파일 형식은 읽을 수 없어요.", "errorNotIndexed": "문서를 먼저 읽은 뒤에 추출해 주세요.", "errorOutputLimit": "AI 제공자가 값을 하나도 돌려주기 전에 출력 한도에 도달했어요. 더 긴 답변을 허용하는 모델로 다시 시도해 주세요.", - "errorLocalOcrDisabled": "AI 설정에서 로컬 OCR을 먼저 켜 주세요.", + "errorLocalOcrDisabled": "설정 → 모듈 → 검사에서 “이 기기에서 스캔 읽기”를 먼저 켜 주세요.", "errorProvider": "이 작업에 쓸 AI 제공자가 설정돼 있지 않아요.", "errorTextImageOnly": "로컬 OCR은 이미지만 읽어요. 이 파일은 읽을 수 없어요.", "errorConsent": "AI로 문서를 읽으려면 먼저 동의가 필요합니다.", diff --git a/messages/nl.json b/messages/nl.json index af2efc0a4..6a390f9eb 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -11769,8 +11769,9 @@ "dismiss": "Negeren", "empty": "Er kon niets uit dit document worden gelezen.", "unavailableBody": "Stel een AI-provider in om een titel, soort en datum voor te stellen.", - "unavailableLocalOcr": "Zet lokale OCR aan in de AI-instellingen om afbeeldingen op dit apparaat te lezen.", + "unavailableLocalOcr": "Zet “Scans op dit apparaat lezen” aan onder Instellingen → Modules → Labwaarden om afbeeldingen op dit apparaat te lezen.", "unavailableAction": "AI-instellingen openen", + "unavailableLocalOcrAction": "Instellingen voor labwaarden openen", "errorGeneric": "Het document kon niet worden gelezen. Probeer het opnieuw.", "errorOcr": "Er kon geen tekst uit deze afbeelding worden gelezen.", "errorRateLimited": "De limiet voor lezen met AI per uur is bereikt. Probeer het later opnieuw.", @@ -11782,7 +11783,7 @@ "errorFileType": "Dit bestandstype kan niet worden gelezen.", "errorNotIndexed": "Laat het document eerst lezen en extraheer daarna.", "errorOutputLimit": "De AI-provider bereikte zijn uitvoerlimiet voordat hij één waarde teruggaf. Probeer een model dat langere antwoorden toestaat.", - "errorLocalOcrDisabled": "Zet eerst lokale OCR aan in de AI-instellingen.", + "errorLocalOcrDisabled": "Zet eerst “Scans op dit apparaat lezen” aan onder Instellingen → Modules → Labwaarden.", "errorProvider": "Hiervoor is geen AI-provider ingesteld.", "errorTextImageOnly": "Lokale OCR leest alleen afbeeldingen, niet dit bestand.", "errorConsent": "Voor het lezen van documenten met AI is eerst je toestemming nodig.", diff --git a/messages/pl.json b/messages/pl.json index b0dec38cc..0d86a2800 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -11769,8 +11769,9 @@ "dismiss": "Odrzuć", "empty": "Nie udało się nic odczytać z tego dokumentu.", "unavailableBody": "Skonfiguruj dostawcę AI, aby zaproponować tytuł, typ i datę.", - "unavailableLocalOcr": "Włącz lokalne OCR w ustawieniach AI, aby odczytywać obrazy na tym urządzeniu.", + "unavailableLocalOcr": "Włącz „Odczytuj skany na tym urządzeniu” w Ustawienia → Moduły → Badania, aby odczytywać obrazy na tym urządzeniu.", "unavailableAction": "Otwórz ustawienia AI", + "unavailableLocalOcrAction": "Otwórz ustawienia badań", "errorGeneric": "Nie udało się odczytać dokumentu. Spróbuj ponownie.", "errorOcr": "Nie udało się odczytać tekstu z tego obrazu.", "errorRateLimited": "Godzinowy limit odczytów AI został osiągnięty. Spróbuj ponownie później.", @@ -11782,7 +11783,7 @@ "errorFileType": "Tego typu pliku nie można odczytać.", "errorNotIndexed": "Najpierw odczytaj dokument, potem wyodrębnij.", "errorOutputLimit": "Dostawca AI osiągnął limit odpowiedzi, zanim zwrócił choć jedną wartość. Spróbuj modelu, który pozwala na dłuższe odpowiedzi.", - "errorLocalOcrDisabled": "Najpierw włącz lokalne OCR w ustawieniach AI.", + "errorLocalOcrDisabled": "Najpierw włącz „Odczytuj skany na tym urządzeniu” w Ustawienia → Moduły → Badania.", "errorProvider": "Nie skonfigurowano dostawcy AI do tego celu.", "errorTextImageOnly": "Lokalne OCR odczytuje tylko obrazy, nie ten plik.", "errorConsent": "Odczyt dokumentów przez AI wymaga najpierw Twojej zgody.", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index 3137e637c..33a57ff8b 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -11769,8 +11769,9 @@ "dismiss": "Dispensar", "empty": "Nada pôde ser lido deste documento.", "unavailableBody": "Configure um provedor de IA para sugerir um título, tipo e data.", - "unavailableLocalOcr": "Ative o OCR local nas configurações de IA para ler imagens neste dispositivo.", + "unavailableLocalOcr": "Ative “Ler digitalizações neste dispositivo” em Configurações → Módulos → Exames para ler imagens neste dispositivo.", "unavailableAction": "Abrir configurações de IA", + "unavailableLocalOcrAction": "Abrir configurações de exames", "errorGeneric": "Não foi possível ler o documento. Tente novamente.", "errorOcr": "Não foi possível ler texto desta imagem.", "errorRateLimited": "O limite horário de leitura por IA foi atingido. Tente novamente mais tarde.", @@ -11782,7 +11783,7 @@ "errorFileType": "Este tipo de arquivo não pode ser lido.", "errorNotIndexed": "Leia o documento primeiro, depois extraia.", "errorOutputLimit": "O provedor de IA atingiu o limite de saída antes de devolver um único valor. Tente um modelo que permita respostas mais longas.", - "errorLocalOcrDisabled": "Ative o OCR local nas configurações de IA primeiro.", + "errorLocalOcrDisabled": "Ative primeiro “Ler digitalizações neste dispositivo” em Configurações → Módulos → Exames.", "errorProvider": "Nenhum provedor de IA está configurado para isso.", "errorTextImageOnly": "O OCR local lê apenas imagens, não este arquivo.", "errorConsent": "Ler documentos com IA precisa do seu consentimento primeiro.", diff --git a/src/components/documents/__tests__/document-ai-panels.test.tsx b/src/components/documents/__tests__/document-ai-panels.test.tsx index 53748f937..d6e2c20dc 100644 --- a/src/components/documents/__tests__/document-ai-panels.test.tsx +++ b/src/components/documents/__tests__/document-ai-panels.test.tsx @@ -1,7 +1,12 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { renderToStaticMarkup } from "react-dom/server"; import { I18nProvider } from "@/lib/i18n/context"; + +const record = vi.hoisted(() => ({ inSharedRecord: false })); +vi.mock("@/hooks/use-record-capabilities", () => ({ + useRecordCapabilities: () => ({ inSharedRecord: record.inSharedRecord }), +})); import type { DocumentSuggestionDto } from "@/lib/validations/inbound-documents"; import { AiUnavailableHint, @@ -57,10 +62,26 @@ describe("", () => { expect(html).not.toContain('role="alert"'); }); - it("nudges toward local OCR when a text-only provider needs it", () => { + it("points at the on-device scan switch when a text-only provider needs it", () => { const html = render(); - expect(html).toContain("local OCR"); - expect(html).toContain('href="/settings/ai"'); + // The switch lives on the Labs module page, under its own label. + expect(html).toContain("Read scans on this device"); + expect(html).toContain("Settings → Modules → Labs"); + expect(html).toContain('href="/settings/modules/labs#labs-local-ocr"'); + expect(html).toContain("Open lab settings"); + expect(html).not.toContain('href="/settings/ai"'); + }); + + it("keeps the sentence but withholds the Settings link inside somebody else's record", () => { + record.inSharedRecord = true; + try { + const html = render(); + expect(html).toContain("Read scans on this device"); + expect(html).not.toContain('data-slot="assist-settings-link"'); + expect(html).not.toContain('href="/settings'); + } finally { + record.inSharedRecord = false; + } }); }); diff --git a/src/components/documents/__tests__/document-ai-section.test.tsx b/src/components/documents/__tests__/document-ai-section.test.tsx index 742859335..c6b85142d 100644 --- a/src/components/documents/__tests__/document-ai-section.test.tsx +++ b/src/components/documents/__tests__/document-ai-section.test.tsx @@ -1,7 +1,12 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { renderToStaticMarkup } from "react-dom/server"; import { I18nProvider } from "@/lib/i18n/context"; + +const record = vi.hoisted(() => ({ inSharedRecord: false })); +vi.mock("@/hooks/use-record-capabilities", () => ({ + useRecordCapabilities: () => ({ inSharedRecord: record.inSharedRecord }), +})); import type { DocumentSuggestionDto } from "@/lib/validations/inbound-documents"; import { DocumentAiSection } from "../document-ai-section"; diff --git a/src/components/documents/document-ai-panels.tsx b/src/components/documents/document-ai-panels.tsx index b2c5e8d66..114f3d987 100644 --- a/src/components/documents/document-ai-panels.tsx +++ b/src/components/documents/document-ai-panels.tsx @@ -22,6 +22,7 @@ import Link from "next/link"; import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; import type { AiRunPhase } from "@/hooks/use-ai-run"; +import { useRecordCapabilities } from "@/hooks/use-record-capabilities"; import { useTranslations } from "@/lib/i18n/context"; import { cn } from "@/lib/utils"; import { @@ -64,9 +65,13 @@ export function AiRunPhaseNote({ } /** - * Calm pointer to the AI settings, shown when assist is unavailable. When the - * document is already searchable (auto-indexed locally) the copy stays honest — - * it is searchable, an AI provider only adds a richer read. + * Calm pointer to the setting that would make assist available: the AI + * settings when no provider is configured, the on-device scan switch on the + * Labs module page when the provider cannot read images. Both live under + * `/settings`, which is the caller's own and closed inside somebody else's + * record, so the link is withheld there and the sentence stands alone. When + * the document is already searchable (auto-indexed locally) the copy stays + * honest — it is searchable, an AI provider only adds a richer read. */ export function AiUnavailableHint({ reason, @@ -74,10 +79,11 @@ export function AiUnavailableHint({ reason: "no-provider" | "enable-local-ocr" | null; }) { const { t } = useTranslations(); - const body = - reason === "enable-local-ocr" - ? t("documents.assist.unavailableLocalOcr") - : t("documents.assist.unavailableBody"); + const { inSharedRecord } = useRecordCapabilities(); + const localOcr = reason === "enable-local-ocr"; + const body = localOcr + ? t("documents.assist.unavailableLocalOcr") + : t("documents.assist.unavailableBody"); return (

- {body}{" "} - - {t("documents.assist.unavailableAction")} - + {body} + {!inSharedRecord && ( + <> + {" "} + + {localOcr + ? t("documents.assist.unavailableLocalOcrAction") + : t("documents.assist.unavailableAction")} + + + )}

); From d6bb7bc905946f8e9ae7ed82d694bc35f2136059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:46:11 +0100 Subject: [PATCH 04/44] docs(api): describe the legacy integration-status fields as still served The published contract said the nine health-shaped fields that predate `health` were removed in v1.44, but every status read still sends them: older native builds decode them, the current one falls back to them against a server without `health`, and the web card's copied diagnostics read the failure streak. They stay deprecated, without a removal release. --- docs/api/openapi.yaml | 69 +++++++++---------- .../baselines/version-stamp-comments.json | 4 +- src/app/api/integrations/status/route.ts | 9 +-- src/lib/openapi/routes/integrations.ts | 12 ++-- 4 files changed, 48 insertions(+), 46 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 9255e81e3..aa48c04d7 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -82136,7 +82136,7 @@ components: - type: "null" syncHealth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Present on the GET read; omitted from the PATCH echo." + description: "Deprecated, still served: read `health`. Present on the GET read; omitted from the PATCH echo." $ref: "#/components/schemas/SyncHealth" metricFreshness: description: Per-metric-type freshness for the `APPLE_HEALTH` source, with the per-type arrival facts. Present on the @@ -82814,7 +82814,7 @@ components: type: boolean state: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." type: string enum: - connected @@ -82837,14 +82837,14 @@ components: - type: "null" lastError: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Last recorded failure message, decrypted from the ledger. - Null when the last attempt succeeded or none has run." + description: "Deprecated, still served: read `health`. Last recorded failure message, decrypted from the ledger. Null + when the last attempt succeeded or none has run." type: - string - "null" syncHealth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." $ref: "#/components/schemas/SyncHealth" health: description: The unified connection state. Present whenever the connection exists; null in the rare case it resolved to @@ -83033,9 +83033,9 @@ components: pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. The stored access token's expiry is in the past. Read - straight off the row — unlike the Withings status read, this endpoint does not refresh; the sync path - refreshes lazily instead." + description: "Deprecated, still served: read `health`. The stored access token's expiry is in the past. Read straight + off the row — unlike the Withings status read, this endpoint does not refresh; the sync path refreshes + lazily instead." type: boolean tokenExpiresAt: type: string @@ -83124,8 +83124,7 @@ components: pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. True after a refresh attempt that did not produce a live - token." + description: "Deprecated, still served: read `health`. True after a refresh attempt that did not produce a live token." type: boolean tokenRefreshFailed: description: The read tried to refresh an expired token and could not — either the per-user credentials are gone or @@ -83145,8 +83144,8 @@ components: - "null" hasActivityScope: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Pre-computed convenience flag the reconnect banner reads. A - null `scope` counts as missing activity." + description: "Deprecated, still served: read `health`. Pre-computed convenience flag the reconnect banner reads. A null + `scope` counts as missing activity." type: boolean health: description: The unified connection state. Present whenever the connection exists; null in the rare case it resolved to @@ -83455,8 +83454,8 @@ components: pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Read straight off the row — this endpoint does not refresh; - the sync path refreshes lazily." + description: "Deprecated, still served: read `health`. Read straight off the row — this endpoint does not refresh; the + sync path refreshes lazily." type: boolean tokenExpiresAt: type: string @@ -83626,8 +83625,8 @@ components: pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Read straight off the row — this endpoint does not refresh; - the sync path refreshes lazily." + description: "Deprecated, still served: read `health`. Read straight off the row — this endpoint does not refresh; the + sync path refreshes lazily." type: boolean tokenExpiresAt: type: string @@ -83677,7 +83676,7 @@ components: when every type was read to its start. Kept after the backfill completes; cleared on reconnect. needsReauth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. The stored refresh token was rejected with `invalid_grant` — + description: "Deprecated, still served: read `health`. The stored refresh token was rejected with `invalid_grant` — Google expires it after 7 days while the OAuth client is in Testing publishing mode, and a user-revoked grant lands here too. Distinct from `parked` and from `disconnected`: the connection still exists, it just cannot mint a token until the user reconnects. Only Google Health has this field." @@ -83881,7 +83880,7 @@ components: description: The user has stored their own BYO client id and secret. state: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." type: string enum: - connected @@ -83904,13 +83903,13 @@ components: - type: "null" lastError: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." type: - string - "null" syncHealth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." $ref: "#/components/schemas/SyncHealth" health: description: The unified connection state. Present whenever the connection exists; null in the rare case it resolved to @@ -84305,7 +84304,7 @@ components: - disconnected - parked - unknown - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." deprecated: true lastSuccessAt: anyOf: @@ -84321,7 +84320,7 @@ components: - type: "null" lastError: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." type: - string - "null" @@ -84348,11 +84347,11 @@ components: additionalProperties: false - type: "null" deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Per-kind consecutive-failure counters, or null when the row - has never written a bucket payload. Compare against the top-level `threshold` to know how close the provider - is to the operator alert." + description: "Deprecated, still served: read `health`. Per-kind consecutive-failure counters, or null when the row has + never written a bucket payload. Compare against the top-level `threshold` to know how close the provider is + to the operator alert." syncHealth: - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." deprecated: true $ref: "#/components/schemas/SyncHealth" metricFreshness: @@ -84389,7 +84388,7 @@ components: - type: "null" tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Null — not false — when there is no connection to judge." + description: "Deprecated, still served: read `health`. Null — not false — when there is no connection to judge." type: - boolean - "null" @@ -84432,14 +84431,14 @@ components: - "null" hasActivityScope: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Withings only." + description: "Deprecated, still served: read `health`. Withings only." type: boolean planUnavailable: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Withings only. The readings this connection's Withings API - plan withholds (status 277 since the 2026-10-12 plan split), deduplicated and in display order. Empty when - the plan refuses nothing. The sync skips these and keeps importing everything else; the list refreshes when - the weekly re-probe asks again." + description: "Deprecated, still served: read `health`. Withings only. The readings this connection's Withings API plan + withholds (status 277 since the 2026-10-12 plan split), deduplicated and in display order. Empty when the + plan refuses nothing. The sync skips these and keeps importing everything else; the list refreshes when the + weekly re-probe asks again." type: array items: type: string @@ -84468,7 +84467,7 @@ components: - weight needsReauth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Google Health only." + description: "Deprecated, still served: read `health`. Google Health only." type: boolean available: description: Oura / Polar / Strava only. @@ -84478,7 +84477,7 @@ components: type: boolean scopeReconnect: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Oura only. Collections the stored grant does not cover: Oura + description: "Deprecated, still served: read `health`. Oura only. Collections the stored grant does not cover: Oura refused them for a missing scope (named in its 401, or inferred when sibling collections succeeded with the same token), or the scopes Oura reported at connect leave theirs out. Not a failure and not counted toward the reconnect threshold; one reconnect asks for the missing scopes. Empty when the grant covers everything @@ -84534,7 +84533,7 @@ components: description: The unified connection state, null when the provider is not connected (and for Fitbit once `retirement.ended`). It supersedes `state`, `lastError`, `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, `hasActivityScope`, `planUnavailable` and `scopeReconnect` on this entry; - those nine are deprecated and removed in v1.44. + those nine are deprecated and still served for clients that predate `health`. required: - integration - state diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 5a3506c2e..95b97061a 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2897, - "total": 9247, + "total": 9246, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -360,7 +360,7 @@ "app/api/integrations/healthkit/__tests__/route.test.ts": 4, "app/api/integrations/healthkit/route.ts": 3, "app/api/integrations/status/__tests__/route.test.ts": 2, - "app/api/integrations/status/route.ts": 7, + "app/api/integrations/status/route.ts": 6, "app/api/integrations/withings/resume/__tests__/route.test.ts": 1, "app/api/integrations/withings/resume/route.ts": 1, "app/api/internal/deploy-webhook/__tests__/route.test.ts": 2, diff --git a/src/app/api/integrations/status/route.ts b/src/app/api/integrations/status/route.ts index 417037307..56539ee6e 100644 --- a/src/app/api/integrations/status/route.ts +++ b/src/app/api/integrations/status/route.ts @@ -502,8 +502,8 @@ export const GET = apiHandler(async () => { } satisfies IntegrationViewModel & NightscoutExtras, { ...publish(dawarichStatus), - // v1.43 — Dawarich, read by the card through `health`. The ledger core - // rides along like on every entry, and goes with the rest in v1.44. + // Dawarich, read by the card through `health`. The ledger core rides + // along like on every entry. connected: dawarichConfigured, configured: dawarichConfigured, health: health["dawarich"], @@ -524,8 +524,9 @@ interface IntegrationViewModel { * health-shaped field on this entry (`state`, `lastError`, * `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, * `hasActivityScope`, `planUnavailable`, `scopeReconnect`) is folded into - * it and goes once the settings cards render `health`; no other client - * reads this envelope. + * it. Those stay on the wire, deprecated in the published document; the + * cards render `health`, and only their copied diagnostics still read + * `consecutiveFailuresByKind`. No native client reads this envelope. */ health: ConnectionHealth | null; state: string; diff --git a/src/lib/openapi/routes/integrations.ts b/src/lib/openapi/routes/integrations.ts index c2acc5677..a50b645d7 100644 --- a/src/lib/openapi/routes/integrations.ts +++ b/src/lib/openapi/routes/integrations.ts @@ -75,10 +75,12 @@ const healthKitEntry = z "One resolved HealthKit metric mapping (defaults merged with the user's stored overrides).", }); -// The health-shaped fields that predate `health` (v1.42.4). They stay on the -// wire through v1.43 so App 1.2.0 keeps decoding, are marked deprecated in the -// document, and go in v1.44. -const LEGACY_HEALTH_FIELD = "Deprecated: removed in v1.44, read `health`."; +// The health-shaped fields that predate `health`. They are marked deprecated +// in the document and stay on the wire: native builds from before `health` +// still decode them, the current one falls back to them only against a server +// that sends no `health`, and the web card's copied diagnostics read the +// failure streak. No removal release is set. +const LEGACY_HEALTH_FIELD = "Deprecated, still served: read `health`."; function legacy(text?: string): { deprecated: true; description: string } { return { @@ -1184,7 +1186,7 @@ const integrationEnvelopeEntry = z health: connectionHealth .nullable() .describe( - "The unified connection state, null when the provider is not connected (and for Fitbit once `retirement.ended`). It supersedes `state`, `lastError`, `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, `hasActivityScope`, `planUnavailable` and `scopeReconnect` on this entry; those nine are deprecated and removed in v1.44.", + "The unified connection state, null when the provider is not connected (and for Fitbit once `retirement.ended`). It supersedes `state`, `lastError`, `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, `hasActivityScope`, `planUnavailable` and `scopeReconnect` on this entry; those nine are deprecated and still served for clients that predate `health`.", ), }) .meta({ From a57d2ffe83c992f9e9825df418a7ec612f8ebfae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:46:51 +0100 Subject: [PATCH 05/44] test(fhir): pin the search parameters the CapabilityStatement declares The statement already declares `date` and `code` on Observation beside paging on every search; the route test now asserts that list per resource, so a parameter the search honours cannot drop out of it unnoticed. --- tests/integration/routes/fhir.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/routes/fhir.test.ts b/tests/integration/routes/fhir.test.ts index a094784b8..8ff64fff2 100644 --- a/tests/integration/routes/fhir.test.ts +++ b/tests/integration/routes/fhir.test.ts @@ -313,6 +313,9 @@ describe("GET /api/fhir/metadata", () => { z.looseObject({ type: z.string(), interaction: z.array(z.object({ code: z.string() })), + searchParam: z.array( + z.object({ name: z.string(), type: z.string() }), + ), operation: z.array(z.object({ name: z.string() })).optional(), }), ), @@ -330,6 +333,21 @@ describe("GET /api/fhir/metadata", () => { expect(patient).toHaveLength(1); expect(patient[0].operation?.map((o) => o.name)).toEqual(["everything"]); + // Paging on every search; the Observation search alone honours the + // time-series range and the code filter, and says so. + for (const resource of resources) { + expect(resource.searchParam, resource.type).toEqual([ + { name: "_count", type: "number" }, + { name: "_offset", type: "number" }, + ...(resource.type === "Observation" + ? [ + { name: "date", type: "date" }, + { name: "code", type: "token" }, + ] + : []), + ]); + } + const root = path.join(process.cwd(), "src/app/api/fhir"); for (const resource of resources) { for (const { code } of resource.interaction) { From 72d90282d5cfc4efb824745765942e376b40ca50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:48:40 +0100 Subject: [PATCH 06/44] fix(settings): open the docs pages themselves, not their old addresses The integration runbooks, the Garmin note, the guardian note on a managed record's connections, the backups admin link and the README's self-hosting link all pointed at retired docs paths that only resolve through redirects (or not at all, on a host without them). They now go to the final guide pages, and the settings tests assert the new addresses. --- README.md | 2 +- .../baselines/version-stamp-comments.json | 4 ++-- src/components/admin/backups-section.tsx | 2 +- .../__tests__/integrations-section.test.tsx | 7 ++++++- .../managed-integration-status.test.tsx | 2 +- .../fitbit-retirement-notice.test.tsx | 2 +- .../settings/integrations/garmin-info-note.tsx | 4 ++-- .../settings/integrations/setup-guide-link.tsx | 18 ++++++++++-------- .../settings/managed-integration-status.tsx | 2 +- .../__tests__/setup-docs-parity.test.ts | 8 ++++---- 10 files changed, 29 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 854eb8cc3..65c2c13f3 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Verify the running version: curl -s http://localhost:3000/api/version # returns version + buildSha + builtAt ``` -`SESSION_COOKIE_SECURE=false` is the one setting plain-HTTP self-hosts must set: without it the session cookie carries the `Secure` flag, the browser drops it over HTTP, and login silently fails. Leave it unset (or `true`) only when a TLS reverse proxy serves HTTPS — and then set `NEXT_PUBLIC_APP_URL` and `APP_URL` to the public URL first (see the [self-hosting guide](https://docs.healthlog.dev/self-hosting/)). +`SESSION_COOKIE_SECURE=false` is the one setting plain-HTTP self-hosts must set: without it the session cookie carries the `Secure` flag, the browser drops it over HTTP, and login silently fails. Leave it unset (or `true`) only when a TLS reverse proxy serves HTTPS — and then set `NEXT_PUBLIC_APP_URL` and `APP_URL` to the public URL first (see the [self-hosting guide](https://docs.healthlog.dev/getting-started/self-host-in-10-minutes/)). Persistent state lives in exactly one named volume (`postgres-data`, the Postgres data); the app container is stateless, so backing up the database backs up everything. An install from v1.45 or earlier moves its Postgres 16 database into it on the first start ([details](docs/ops/postgres-major-upgrade.md)). diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 95b97061a..6902bda8e 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2897, - "total": 9246, + "total": 9243, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -1381,7 +1381,7 @@ "components/settings/integrations/connections-panel.tsx": 5, "components/settings/integrations/garmin-info-note.tsx": 1, "components/settings/integrations/notification-channels-panel.tsx": 1, - "components/settings/integrations/setup-guide-link.tsx": 5, + "components/settings/integrations/setup-guide-link.tsx": 2, "components/settings/integrations/shared.tsx": 3, "components/settings/labs-section.tsx": 1, "components/settings/layout-groups.ts": 2, diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index d8dcb62c2..0eb220c00 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -98,7 +98,7 @@ export function BackupsSection() { {/* External docs link — `noopener noreferrer` because this leaves the admin shell. */} { for (const provider of PROVIDERS) { expect(html).toContain(`data-testid="${provider}-setup-guide"`); expect(html).toContain( - `href="https://docs.healthlog.dev/integrations/${provider}"`, + `href="https://docs.healthlog.dev/guides/connect-${provider}/"`, ); } + // Garmin has no connector; its note goes straight to the section of the + // Google Health guide that explains the route, not through a redirect. + expect(html).toContain( + 'href="https://docs.healthlog.dev/guides/connect-google-health/#garmin-owners"', + ); }); }); diff --git a/src/components/settings/__tests__/managed-integration-status.test.tsx b/src/components/settings/__tests__/managed-integration-status.test.tsx index d7126b5dd..f805c891a 100644 --- a/src/components/settings/__tests__/managed-integration-status.test.tsx +++ b/src/components/settings/__tests__/managed-integration-status.test.tsx @@ -139,7 +139,7 @@ describe("", () => { "Services you connect here send their data to Managed record's record, not to yours.", ); expect(html).toContain( - 'href="https://docs.healthlog.dev/self-hosting/account-sharing#records-with-no-login"', + 'href="https://docs.healthlog.dev/guides/managed-profiles/#connected-services"', ); }); diff --git a/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx b/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx index 21fd77d23..753be99d2 100644 --- a/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx +++ b/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx @@ -31,7 +31,7 @@ describe("", () => { expect(html).toContain("Google Health reads the same Fitbit and Pixel"); expect(html).toContain("Everything already imported stays."); expect(html).toContain('href="#google-health"'); - expect(html).toContain("/google-health"); + expect(html).toContain("/guides/connect-google-health/"); }); it("says the connection has stopped once the day has come", () => { diff --git a/src/components/settings/integrations/garmin-info-note.tsx b/src/components/settings/integrations/garmin-info-note.tsx index 4d2986acc..10c3fd645 100644 --- a/src/components/settings/integrations/garmin-info-note.tsx +++ b/src/components/settings/integrations/garmin-info-note.tsx @@ -11,7 +11,7 @@ import { ExternalLink, Watch } from "lucide-react"; import { SettingsCard } from "@/components/settings/settings-card"; import { SettingsCardHeader } from "@/components/settings/_card-header"; -import { INTEGRATION_DOCS_BASE } from "@/components/settings/integrations/setup-guide-link"; +import { GARMIN_DOCS_HREF } from "@/components/settings/integrations/setup-guide-link"; import { useTranslations } from "@/lib/i18n/context"; export function GarminInfoNote() { @@ -25,7 +25,7 @@ export function GarminInfoNote() {

{t("settings.garminInfo.body")}{" "} /`, e.g. `https://docs.healthlog.dev/guides/connect-whoop/`. + * These are the pages themselves: the older `/integrations/` paths + * only resolve through redirects. Kept as a single constant so the host never + * drifts across cards. */ -export const INTEGRATION_DOCS_BASE = "https://docs.healthlog.dev/integrations"; +export const DOCS_GUIDES_BASE = "https://docs.healthlog.dev/guides"; + +/** Garmin has no connector; the Google Health guide explains the route. */ +export const GARMIN_DOCS_HREF = `${DOCS_GUIDES_BASE}/connect-google-health/#garmin-owners`; export type IntegrationDocsProvider = | "whoop" @@ -30,15 +35,12 @@ export type IntegrationDocsProvider = | "polar" | "oura" | "nightscout" - // v1.27.0 — Google Health runbook (docs.healthlog.dev/integrations/google-health). | "google-health" - // v1.28.x — Strava runbook (docs.healthlog.dev/integrations/strava). | "strava" - // v1.43 — Dawarich runbook (docs.healthlog.dev/integrations/dawarich). | "dawarich"; export function integrationDocsHref(provider: IntegrationDocsProvider): string { - return `${INTEGRATION_DOCS_BASE}/${provider}`; + return `${DOCS_GUIDES_BASE}/connect-${provider}/`; } /** diff --git a/src/components/settings/managed-integration-status.tsx b/src/components/settings/managed-integration-status.tsx index 6f84aa551..1498add4a 100644 --- a/src/components/settings/managed-integration-status.tsx +++ b/src/components/settings/managed-integration-status.tsx @@ -54,7 +54,7 @@ const INTEGRATION_DISPLAY_NAMES = { /** Where the guardian boundary is explained, connected services included. */ const SHARING_DOCS_HREF = - "https://docs.healthlog.dev/self-hosting/account-sharing#records-with-no-login"; + "https://docs.healthlog.dev/guides/managed-profiles/#connected-services"; function iconFor(key: ManagedIntegrationKey) { if (key === "nightscout") return Droplet; diff --git a/src/lib/google-health/__tests__/setup-docs-parity.test.ts b/src/lib/google-health/__tests__/setup-docs-parity.test.ts index 9e8a3a094..653b72cfe 100644 --- a/src/lib/google-health/__tests__/setup-docs-parity.test.ts +++ b/src/lib/google-health/__tests__/setup-docs-parity.test.ts @@ -2,8 +2,8 @@ * Doc ↔ code parity guard for the Google Health operator runbook. * * The setup-guide link on the Settings card sends a mid-setup user to - * `docs.healthlog.dev/integrations/google-health`, rendered from - * `docs/integrations/google-health.md`. If the client ever requests a scope the + * `docs.healthlog.dev/guides/connect-google-health/`; the operator runbook in + * this repository is `docs/integrations/google-health.md`. If the client ever requests a scope the * runbook doesn't list, or the walkthrough regresses back to a wrong provider * (the file was once a Fitbit-Web-API walkthrough), a self-hoster follows a * broken recipe and the OAuth consent screen rejects the connection. @@ -20,7 +20,7 @@ import { describe, expect, it } from "vitest"; import { GOOGLE_HEALTH_CORE_SCOPES } from "@/lib/google-health/oauth"; import { - INTEGRATION_DOCS_BASE, + DOCS_GUIDES_BASE, integrationDocsHref, } from "@/components/settings/integrations/setup-guide-link"; @@ -70,7 +70,7 @@ describe("google-health setup runbook parity", () => { // The literal argument also compile-checks that "google-health" is a member // of the IntegrationDocsProvider union. expect(integrationDocsHref("google-health")).toBe( - `${INTEGRATION_DOCS_BASE}/google-health`, + `${DOCS_GUIDES_BASE}/connect-google-health/`, ); }); }); From f85035a8ecd82ab9ec0d2b75bc2307c31ec2a106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:49:46 +0100 Subject: [PATCH 07/44] docs(coach): say what the labs block reads since it covers the whole record The labs block reads the whole record, or up to the lookback limit, not just recent readings, and a named read also comes back empty when no marker matches. The header of the labs snapshot and the note at its call site still described the earlier twelve-month window. --- src/lib/ai/coach/labs-snapshot.ts | 5 +++-- src/lib/ai/coach/snapshot.ts | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/ai/coach/labs-snapshot.ts b/src/lib/ai/coach/labs-snapshot.ts index bb030f516..5d606dd35 100644 --- a/src/lib/ai/coach/labs-snapshot.ts +++ b/src/lib/ai/coach/labs-snapshot.ts @@ -48,8 +48,9 @@ * `userId` is narrowed from the authenticated session by the caller and feeds * the Prisma `where` field-by-field; it is never an input. Unlike the illness * block, Labs is intentionally NOT module-gated (mirrors `/api/labs` — the data - * is always owner-scoped and safe to read), so the block is `null` only when the - * account has no recent readings. + * is always owner-scoped and safe to read). The block is `null` only when no + * reading falls inside the lookback limit (the whole record when there is + * none), or, for a named read, when no marker matches the analyte. */ import { sameLabUnit } from "@/lib/labs/unit-normalise"; import { diff --git a/src/lib/ai/coach/snapshot.ts b/src/lib/ai/coach/snapshot.ts index b572218a7..7e884a756 100644 --- a/src/lib/ai/coach/snapshot.ts +++ b/src/lib/ai/coach/snapshot.ts @@ -668,8 +668,8 @@ async function buildCoachSnapshotImpl( // scope-gated metric: attached without a `registerBlock` so the budget // degrader never sheds it, and Labs is intentionally not module-gated (the // helper reads owner-scoped rows directly, mirroring `/api/labs`). The block - // carries the most-recent resolved reading per biomarker (last 12 months, - // capped) so the Coach can answer "what was my LDL" without re-deriving. + // carries the latest resolved reading per biomarker (whole record or the + // lookback limit, capped) so the Coach can answer "what was my LDL". const labsBlockPromise = buildLabsSnapshotBlock(userId, now, reach); // v1.38 — doctor-visit context. Like illness/labs it is attempted always (a From 12f8265fdba7c275a07b434e5b9486f9b952ce97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:53:19 +0100 Subject: [PATCH 08/44] docs: correct claims and anchors the code no longer backs Fix two dead anchors (the MCP trust model, the README quick start) and the statements that no longer match the app: a guardian can connect a managed profile's services, the environment module is on by default and reaches the air-quality host too, its readable location columns were dropped in 0409, settings sections carry their current names, the Withings card lives on /settings/integrations, the TrueNAS start period matches the image, the deploy check names HEALTHLOG_PROD_URL, and the docs index describes what self-hosting/ and integrations/ hold. --- README.md | 2 +- docs/README.md | 20 +++++++++------ docs/integrations/apple-health.md | 2 +- docs/integrations/dawarich.md | 2 +- docs/integrations/fitbit.md | 2 +- docs/integrations/google-health.md | 2 +- docs/integrations/strava.md | 2 +- docs/integrations/withings.md | 8 +++--- docs/ops/deploy.md | 7 ++++-- docs/ops/measurement-maintenance.md | 5 ++-- docs/self-hosting/account-sharing.md | 14 +++++++---- docs/self-hosting/environment.md | 9 ++++--- docs/self-hosting/getting-started.md | 14 +++++------ docs/self-hosting/mcp.md | 8 +++--- docs/self-hosting/reproductive-health-data.md | 2 +- docs/self-hosting/truenas.md | 25 ++++++++++--------- docs/self-hosting/unraid.md | 4 +-- docs/ui-guidelines.md | 2 +- 18 files changed, 73 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 65c2c13f3..edf748bbb 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Try the [live demo](https://demo.healthlog.dev) to see a working install — sig - **Clinical signals, opt-in and additive.** Validated self-checks for mental wellbeing and sleep (WHO-5, PHQ-9, GAD-7, Sleep Condition Indicator — official questionnaire wording, calm crisis-resource handling), grip strength, a 0–10 pain score, waist and body measurements, and a longevity lab panel — each added through one signal registry, so a new signal wires into charts, correlations, the coach, the FHIR map, and the API at once. - **Records, structured.** Allergies and family history as structured records that flow into the doctor report (reactions and notes are encrypted, the substance and condition names are not), plus lab values you key in or scan from a printed report — OCR extracts the stated values, units, and dates for you to review and confirm before anything is saved. HealthLog transcribes what a report says; it never interprets or diagnoses from it. A vaccination record keeps a lifetime immunization log, doses grouped under the disease they protect against, and prints in the doctor report. - **A document vault for the paper trail.** An opt-in module stores every letter, report, and scan encrypted at rest — PDFs and photos preview inline, Office and text files stay download-only, and nothing user-uploaded can ever execute in the app. One searchable timeline with type, condition, and year filters, drag-and-drop and paste-to-upload, bulk filing, undo-able delete with a 30-day grace, and per-file/per-account limits the operator tunes. -- **Context that explains the numbers.** An opt-in environmental module folds weather, daylight, and temperature in as influence signals (winter darkness against mood, heat against blood pressure) using a home location you set, so a pattern has somewhere to point. +- **Context that explains the numbers.** An environmental module folds weather, daylight, temperature, air quality, and pollen in as influence signals (winter darkness against mood, heat against blood pressure) using a home location you set, so a pattern has somewhere to point. It is on by default, and nothing leaves the host until someone sets that location. - **Your devices, one server.** Withings, WHOOP, Oura and Polar sync over OAuth2, Google Health connects Fitbit and Pixel Watch through your own Google Cloud client, Nightscout streams CGM glucose, an Apple Health `export.zip` folds your full history in, and the native iOS app streams HealthKit live. A per-metric source priority decides which reading is canonical when wearables overlap. - **Medication tracking that tells the truth.** Flexible schedules (weekly injections, weekday-only, intervals, PRN, cyclic), a configurable intake window per dose, and a traceable dose history where every slot reads taken, late, skipped, or missed — the same ledger the compliance rate is computed from, so the percentage can never disagree with the timeline. - **AI insights you own.** Daily briefing, health scores, correlations, and a conversational coach grounded in your own measurements and interpreted against cited guideline reference bands — where a value sits and what that means in plain words, never a diagnosis. The coach proposes concrete if-then plans you accept or decline, and suppresses proactive check-ins when there's nothing genuine to say. Bring your own OpenAI or Anthropic key, or point at a local endpoint (Ollama, LM Studio, vLLM) so AI requests stay on your network. diff --git a/docs/README.md b/docs/README.md index 7b99ebc09..225e5a1f8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,9 +17,13 @@ notes, and machine-readable specs that ship alongside the source: encryption-key rotation, password reset and account recovery, env checks, TLS pinning, data repair. These are what an operator reaches for during an incident. -- [`self-hosting/`](./self-hosting/) — horizontal-scaling notes - (`HEALTHLOG_PROCESS_TYPE=web|worker|all`) and the deploy-pipeline - recipe. Routine install steps stay on `docs.healthlog.dev`. Includes +- [`self-hosting/`](./self-hosting/) — operator notes next to the code: + [`getting-started.md`](./self-hosting/getting-started.md) (install and + updates, the page the boot log points at), the TrueNAS, Unraid and + Portainer installs with their catalog templates, Tailscale, the reverse + proxy, horizontal scaling (`HEALTHLOG_PROCESS_TYPE=web|worker|all`), + notification channels, the environment module, document sources and + reproductive-health data. Includes [`self-hosting/mcp.md`](./self-hosting/mcp.md) — enabling the off-by-default MCP connector and pointing Claude.ai / ChatGPT / Claude Desktop at it, and [`self-hosting/sso.md`](./self-hosting/sso.md) @@ -44,9 +48,11 @@ notes, and machine-readable specs that ship alongside the source: - [`migration/`](./migration/) — migration notes. Currently only v1.3 to v1.4; later releases are covered by `CHANGELOG.md` and . -- [`integrations/`](./integrations/) — per-integration reference - (Withings, Fitbit, Google Health, WHOOP, Apple Health, AI providers, - data import). +- [`integrations/`](./integrations/) — per-integration reference: + Withings, WHOOP, Fitbit, Google Health, Strava, Nightscout, Dawarich and + the Garmin route; the Apple Health, Health Connect, document and data + imports; AI providers; and Home Assistant with its blueprint. Oura and + Polar are covered on the docs site only. - [`adr/`](./adr/) — architecture decision records. - [`security/`](./security/) — threat-model notes. - [`diagrams/`](./diagrams/) — architecture diagrams rendered through @@ -70,4 +76,4 @@ notes, and machine-readable specs that ship alongside the source: If you're a self-hoster looking to spin up an instance, start at or jump to the [Quick Start in the project -README](../README.md#quick-start). +README](../README.md#quick-start-docker-compose). diff --git a/docs/integrations/apple-health.md b/docs/integrations/apple-health.md index dc7a3d615..33f72b545 100644 --- a/docs/integrations/apple-health.md +++ b/docs/integrations/apple-health.md @@ -295,7 +295,7 @@ Concrete consequences: RMSSD HRV has its own ladder, **WHOOP ≻ OURA ≻ POLAR ≻ APPLE_HEALTH**, separate from the SDNN one. -Override per-user via Settings → Sources (`/settings/sources`). +Override per-user via Settings → Source priority (`/settings/sources`). ## Failure modes diff --git a/docs/integrations/dawarich.md b/docs/integrations/dawarich.md index 7e19680f9..20540d2e7 100644 --- a/docs/integrations/dawarich.md +++ b/docs/integrations/dawarich.md @@ -70,7 +70,7 @@ The variable is on the `environment:` whitelist in `docker-compose.yml`. - Time away from home and the distance travelled join the correlation scan as two more factors. - Runs of at least two travel days that no trip covers yet are offered as - trips under **Settings → Environment → Location periods**. Using one fills + trips under **Settings → Location & weather → Location periods**. Using one fills in the dates; you pick the place. - The Coach and the MCP `get_day` tool see the figures and whether it was a travel day, never the country. diff --git a/docs/integrations/fitbit.md b/docs/integrations/fitbit.md index 4c5684c21..c9eb2eae5 100644 --- a/docs/integrations/fitbit.md +++ b/docs/integrations/fitbit.md @@ -103,7 +103,7 @@ The Fitbit client id/secret are per-user (Settings), not env vars. If Fitbit and another source (Withings, WHOOP, Apple Health) both supply the same vital — resting heart rate, blood oxygen, body temperature or respiratory rate — -the source priority ladder in **Settings → Sources** decides which one wins +the source priority ladder in **Settings → Source priority** decides which one wins per metric. FITBIT is ranked in every default metric ladder; override per metric on that page. diff --git a/docs/integrations/google-health.md b/docs/integrations/google-health.md index 069d09e83..3176a827d 100644 --- a/docs/integrations/google-health.md +++ b/docs/integrations/google-health.md @@ -305,7 +305,7 @@ triggers the verification + CASA requirements above.) If Google Health and another source (Withings, WHOOP, Apple Health) both supply the same vital — resting heart rate, blood oxygen, respiratory rate — the source -priority ladder in **Settings → Sources** decides which one wins +priority ladder in **Settings → Source priority** decides which one wins per metric. ## Disconnect diff --git a/docs/integrations/strava.md b/docs/integrations/strava.md index da6a7ed3a..5607894fb 100644 --- a/docs/integrations/strava.md +++ b/docs/integrations/strava.md @@ -34,7 +34,7 @@ default the device-native capture (Apple Watch, WHOOP, Withings) wins over the Strava copy, because a Strava activity is often a re-upload with lower-fidelity or missing heart-rate data. A run you record **only** on Strava (phone GPS) survives untouched. You can promote Strava above the wearables in -**Settings → Sources** if you prefer the Strava copy. +**Settings → Source priority** if you prefer the Strava copy. ## 1. Register a Strava API application diff --git a/docs/integrations/withings.md b/docs/integrations/withings.md index 99ddc858a..a211978c9 100644 --- a/docs/integrations/withings.md +++ b/docs/integrations/withings.md @@ -88,7 +88,7 @@ or right away after reconnecting Withings once. needed — `user.metrics` covers weight / BP / heart-list / sleep / SpO₂ / body comp / temperature / VO₂ max; `user.activity` is the workout-equivalent stream for steps / energy / distance / floors. - See `WITHINGS_OAUTH_SCOPE` in `src/lib/withings/client.ts`. + See `WITHINGS_OAUTH_SCOPE` in `src/lib/withings/oauth.ts`. 4. Save the application. Withings will issue a **Client ID** and a **Client Secret** — keep the secret out of any chat log; it grants read access to every linked account's full health history. @@ -145,7 +145,7 @@ callback. The portal only owns the OAuth callback URL. ## 4. Link from the Settings page With the credentials saved in Settings, open -`/settings/integrations/withings` and click **Connect Withings**. The flow: +`/settings/integrations#withings` and click **Connect Withings**. The flow: 1. The Connect button redirects to Withings' `oauth2/authorize`. 2. The user signs in on Withings and grants the requested scopes. @@ -181,7 +181,7 @@ import: when Apple Health is also linked, so the Apple Health stream wins for the cumulative axis. The Withings rows stay in the DB. -Override per-user via Settings → Sources (`/settings/sources`). +Override per-user via Settings → Source priority (`/settings/sources`). ## Troubleshooting @@ -199,7 +199,7 @@ Override per-user via Settings → Sources (`/settings/sources`). ### The webhook stops firing Webhook subscriptions can lapse after a token revoke or a portal- -side cleanup. Re-subscribe by opening `/settings/integrations/withings` +side cleanup. Re-subscribe by opening `/settings/integrations#withings` and clicking **Disconnect** then **Connect Withings** again — the callback exchange re-runs `Notify.subscribe()` with a fresh callback URL. diff --git a/docs/ops/deploy.md b/docs/ops/deploy.md index 0a1f8ff7c..b6cfee395 100644 --- a/docs/ops/deploy.md +++ b/docs/ops/deploy.md @@ -270,10 +270,13 @@ always confirm the served build after a deploy: ```sh # Asserts prod + demo serve 1.9.0, prints each target's buildSha, # exits non-zero on a mismatch or timeout after a bounded retry loop. -pnpm dlx tsx scripts/assert-deploy.ts 1.9.0 +HEALTHLOG_PROD_URL=https://healthlog.example.com \ + pnpm dlx tsx scripts/assert-deploy.ts 1.9.0 ``` -Pass `--only=prod` / `--only=demo` to check a single target, or +`HEALTHLOG_PROD_URL` is the base URL of your instance; it has no default and +is required unless you pass `--only=demo`. Pass `--only=prod` / +`--only=demo` to check a single target, or `--attempts=N --interval=ms` to widen the retry window while a slow pull settles. diff --git a/docs/ops/measurement-maintenance.md b/docs/ops/measurement-maintenance.md index f7ca054ab..c43caa52d 100644 --- a/docs/ops/measurement-maintenance.md +++ b/docs/ops/measurement-maintenance.md @@ -96,8 +96,9 @@ maintenance refuses to start while one is. Have at least twice the largest size free on the database volume. -- **Take a backup** (Settings, Data, Backups), as before any operation on the - whole table. +- **Take a backup**: a database dump, as in + [Updates](../self-hosting/getting-started.md#updates), as before any + operation on the whole table. ## Starting it diff --git a/docs/self-hosting/account-sharing.md b/docs/self-hosting/account-sharing.md index 7cc76a1de..9e75646fa 100644 --- a/docs/self-hosting/account-sharing.md +++ b/docs/self-hosting/account-sharing.md @@ -214,11 +214,15 @@ as something to log in as. or delete the profile under a fresh second factor. Deleting the profile deletes everything recorded for it. - A guardian may keep that record's settings: modules, units, thresholds, - its language and timezone, and its notification preferences. Connected - services are visible as a status only. Connect, callback, sync, - credential, disconnect, resume and test are not guardian surfaces in - this release, and their deep links land on the shared-record unavailable - page rather than on a raw refusal. + its language and timezone, and its notification preferences. A guardian + can also connect and disconnect the record's Withings, WHOOP, Oura, Polar, + Strava and Google Health from its integrations page; the data lands in the + profile's record, and a profile without its own provider app uses the + guardian's. Fitbit (no new connections), Nightscout and Dawarich cannot be + connected for a profile. Sync, the connection test and provider + credentials are not guardian surfaces: once connected, the record syncs on + its own schedule, and their deep links land on the shared-record + unavailable page rather than on a raw refusal. - **Its reminders go to its guardians.** A record with no login has no channels of its own, so medication reminders, measurement reminders, safety-floor alerts and low-stock alerts fan out to every active diff --git a/docs/self-hosting/environment.md b/docs/self-hosting/environment.md index 51dbb6d3d..095b63fd7 100644 --- a/docs/self-hosting/environment.md +++ b/docs/self-hosting/environment.md @@ -86,10 +86,11 @@ backfill button in settings fetches both for the range it is given. Since v1.42 the home, each dated location period and each stored day keep their coarse location sealed with the instance's encryption key (one value per row, `{ lat, lon, label }`, under the label -`healthlog/environment-location/v1`). The boot-time encryption backfill seals -rows written by older releases and clears their readable columns; the -readable columns are dropped in a later release once the backfill reports -zero rows everywhere. Key rotation (`scripts/rotate-encryption-key.ts`, or the +`healthlog/environment-location/v1`). The boot-time encryption backfill sealed +rows written by older releases and cleared their readable columns, and +migration `0409` (v1.43) dropped those columns; it refuses to run while a row +still holds a readable location +([migration-0409-plaintext-location.md](../ops/migration-0409-plaintext-location.md)). Key rotation (`scripts/rotate-encryption-key.ts`, or the in-app rotation) covers all three. What the person sees is unchanged: the settings page and diff --git a/docs/self-hosting/getting-started.md b/docs/self-hosting/getting-started.md index f4fb5d208..c791df566 100644 --- a/docs/self-hosting/getting-started.md +++ b/docs/self-hosting/getting-started.md @@ -9,8 +9,7 @@ already installed, longer if a TLS-fronted public hostname is in scope - **Docker 24+ with Compose v2.** The bundled `docker-compose.yml` pulls a multi-arch image from GitHub Container Registry (linux/amd64 - - linux/arm64), so a Raspberry Pi 5 or an x86 VPS both work - unchanged. + and linux/arm64), so a Raspberry Pi 5 or an x86 VPS both work unchanged. - **3 GB RAM (4 GB recommended), 10 GB free disk.** The Postgres data volume grows with measurement history; the Apple Health import worker can briefly hold a 1.5 GB upload in `/tmp` while it parses. The app @@ -23,11 +22,12 @@ already installed, longer if a TLS-fronted public hostname is in scope below the supported floor. On a busy instance, raise it. - **Outbound HTTPS** to `ghcr.io` (image pulls) plus whichever integration endpoints you plan to enable (Withings, OpenAI, etc.). - The opt-in environmental-context module additionally reaches - `archive-api.open-meteo.com` and `geocoding-api.open-meteo.com` for - weather and daylight; it stays off until you - enable it and set a home location, and every outbound call goes - through the SSRF-guarded egress wrapper. + The environmental-context module additionally reaches + `archive-api.open-meteo.com`, `air-quality-api.open-meteo.com` and + `geocoding-api.open-meteo.com` for weather, air quality and the city + search. It is on by default, but nothing leaves the host until someone + sets a home location, and every outbound call goes through the + SSRF-guarded egress wrapper. Three requests happen by default without anything connected: the HaveIBeenPwned range query when a password is set (only the first five characters of the password's SHA-1 are sent), a daily check of the diff --git a/docs/self-hosting/mcp.md b/docs/self-hosting/mcp.md index a54b2d1c5..9f4334175 100644 --- a/docs/self-hosting/mcp.md +++ b/docs/self-hosting/mcp.md @@ -19,7 +19,7 @@ on top of it, see [building skills](../api/mcp-skills.md). > **Read this before enabling writes.** A remote, write-capable connector > reached over OAuth is a powerful combination. Read the -> [trust model](#trust-model-read-this-before-enabling-writes) at the end +> [trust model](#trust-model--read-this-before-enabling-writes) at the end > of this page first. ## Prerequisites @@ -43,7 +43,7 @@ LAN-only origin works for Claude Desktop over stdio). The switch lives with every other module: **Settings → Modules**, the **MCP** row. Turn it on and the `/mcp` endpoint goes live for your account. -**Settings → MCP** is where you use it. The page has three parts: +**Settings → MCP connector** is where you use it. The page has three parts: - **State** — whether the module is on, with a link to its switch in Modules, and your endpoint URL (`/mcp`). Turning the module off @@ -55,7 +55,7 @@ The switch lives with every other module: **Settings → Modules**, the ## 2. Mint a connector token -Still in **Settings → MCP → Tokens**: +Still in **Settings → MCP connector → Tokens**: 1. Choose the scope with the **write** toggle: - **off** → a read-only token (`health:read`). This is the default and @@ -106,7 +106,7 @@ refused (`403`) as a DNS-rebinding defence. client-id-metadata-document clients with dynamic registration as a fallback. 3. Approve the consent screen. The connection then appears under - **Settings → MCP → Connections**, where you can revoke it. + **Settings → MCP connector → Connections**, where you can revoke it. No token to paste — OAuth mints and rotates a short-lived `health:read` access token (60 minutes, with a refresh token) behind the scenes. diff --git a/docs/self-hosting/reproductive-health-data.md b/docs/self-hosting/reproductive-health-data.md index 128dbc6d3..314b65aa2 100644 --- a/docs/self-hosting/reproductive-health-data.md +++ b/docs/self-hosting/reproductive-health-data.md @@ -95,7 +95,7 @@ different things depending on which button you press. two actions below, the row stays. A start that was folded into a later cycle stays for as long as that cycle does, because deleting the later cycle gives it back. -- **Settings → Delete All Data** removes the rows outright, tombstones +- **Settings → Advanced → Delete all data** removes the rows outright, tombstones included. Everything the account owns goes except what you sign in with; the confirmation dialog lists it. - **Deleting the account** removes the account row, and the database diff --git a/docs/self-hosting/truenas.md b/docs/self-hosting/truenas.md index d312e4fb7..f8a1b6328 100644 --- a/docs/self-hosting/truenas.md +++ b/docs/self-hosting/truenas.md @@ -49,9 +49,10 @@ HealthLog behind a reverse proxy with its own domain, change App URL to that Settings the form does not show go under Additional Environment Variables, for example `UPDATE_CHECK_DISABLED=1`. The full list is in -[environment.md](environment.md). To use your own GeoLite2 databases, add the -directory under Additional Storage and point `GEOLITE2_DIR` at its mount path -([geolite2.md](geolite2.md)). +[`.env.production.example`](../../.env.production.example) and on the +[environment variables reference](https://docs.healthlog.dev/reference/environment-variables/). +To use your own GeoLite2 databases, add the directory under Additional Storage +and point `GEOLITE2_DIR` at its mount path ([geolite2.md](geolite2.md)). ## Plain HTTP on the LAN @@ -206,15 +207,15 @@ before you touch it. Until the catalog app is listed, install HealthLog with Apps, Discover Apps, the menu next to Custom App, Install via YAML, using these values: -| Setting | Value | -| ------------ | ---------------------------------------------------------------------------------------- | -| Image | `ghcr.io/mbombeck/healthlog`, tag `X.Y.Z` (amd64 and arm64) | -| Port | `3000` (HTTP) | -| User | runs as uid `1001` by default; no extra capabilities, `no-new-privileges` is fine | -| Health check | `GET /api/health`, `200` when ready | -| Start period | at least 60 s, with more retries than the image's 5 × 30 s: migrations run at every boot | -| Database | PostgreSQL 18 (16 works too), no extensions, in its own container with its own dataset | -| Volumes | none for the app; the database gets its own dataset | +| Setting | Value | +| ------------ | -------------------------------------------------------------------------------------- | +| Image | `ghcr.io/mbombeck/healthlog`, tag `X.Y.Z` (amd64 and arm64) | +| Port | `3000` (HTTP) | +| User | runs as uid `1001` by default; no extra capabilities, `no-new-privileges` is fine | +| Health check | `GET /api/health`, `200` when ready | +| Start period | 10 min, like the image: migrations run at every boot and wait out a held lock | +| Database | PostgreSQL 18 (16 works too), no extensions, in its own container with its own dataset | +| Volumes | none for the app; the database gets its own dataset | | Variable | Required | Notes | | ----------------------- | -------- | ---------------------------------------------------------------------------------------- | diff --git a/docs/self-hosting/unraid.md b/docs/self-hosting/unraid.md index 87c0e91df..e0d9564b6 100644 --- a/docs/self-hosting/unraid.md +++ b/docs/self-hosting/unraid.md @@ -116,8 +116,8 @@ PostgreSQL container you installed in Step 1. Back that one up. `SESSION_COOKIE_SECURE=false` is required for plain-HTTP LAN installs (`http://tower:3000`). Left at its default, the session cookie carries the -`Secure` flag, the browser refuses to send it over HTTP, and login appears to -"do nothing" — the page reloads to the login screen with no error. +`Secure` flag, which the browser would drop over HTTP. The login page says so +before anything is sent: "Signing in cannot complete on this page". Set `SESSION_COOKIE_SECURE=true` (or leave it unset) only when a TLS reverse proxy serves HealthLog over HTTPS. In that case also point `NEXT_PUBLIC_APP_URL` diff --git a/docs/ui-guidelines.md b/docs/ui-guidelines.md index 093a0ef35..5b00604e2 100644 --- a/docs/ui-guidelines.md +++ b/docs/ui-guidelines.md @@ -311,7 +311,7 @@ Tiles at the top of the dashboard live in a **single row**. Width caps at the chart's width directly below. **Never** wrap to a second row. If the user enables more tiles than fit, the dashboard surfaces a banner -inviting them to deactivate or reorder via Settings → Dashboard. The +inviting them to deactivate or reorder via Settings → Arrangement. The layout editor in Settings shows a live preview of the constraint and flags overflow before save. From a641a615c31931e5f7da3fe802c507a2c81f9abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:56:28 +0100 Subject: [PATCH 09/44] docs: point the Strava and Dawarich notes at their site guides Both files had drifted copies of the docs-site guides, which carry the same sections and more, and nothing in the app, the scripts, the other docs or the site links to them. Each is now a short pointer to its guide. --- docs/integrations/dawarich.md | 85 +++-------------------------------- docs/integrations/strava.md | 76 ++----------------------------- 2 files changed, 9 insertions(+), 152 deletions(-) diff --git a/docs/integrations/dawarich.md b/docs/integrations/dawarich.md index 20540d2e7..3f16b3f08 100644 --- a/docs/integrations/dawarich.md +++ b/docs/integrations/dawarich.md @@ -1,82 +1,7 @@ # Dawarich integration -HealthLog can read daily movement context from a [Dawarich](https://dawarich.app) -instance: how far you moved, how much of it by car or by train, how many places -you visited, how long you were away from home, and whether a day was spent in -another country. It is part of the Environment module and runs only while that -module is on. - -HealthLog never stores a coordinate, a track, a place name or a visit. It reads -the day summaries and keeps one row of numbers per day. The place list is read -live when you pick your home and is not saved. - -## What is read - -Once a night (03:20 Europe/Berlin) HealthLog fetches the last seven days, ending -yesterday. Connecting fetches the last 30 days. Each request carries your API -key as `Authorization: Bearer `. - -| Dawarich endpoint | What HealthLog keeps | -| ----------------------- | -------------------------------------------------------------------------------------- | -| `GET /api/v1/timeline` | total distance, distance by car (`driving`), distance by train, places visited | -| `GET /api/v1/visits` | minutes spent at your home place, to work out time away from home | -| `GET /api/v1/residency` | the country of each day, sealed at rest, and whether it differs from your home country | -| `GET /api/v1/places` | nothing; the list is shown once so you can pick your home | -| `GET /api/v1/users/me` | nothing; used to check the key | - -Time away from home is the day's length minus the time a visit at your home -place covers. Without a home place, or on a day without a single visit, it is -left empty rather than shown as zero. Dawarich leaves out a mode that moved -less than half a kilometre; HealthLog then reads that mode as zero for the day. - -Residency needs API access on a hosted Dawarich plan; a self-hosted instance -has it. Without it there are no travel days, and everything else still works. -Dawarich keys residency by UTC date, so on a day that crosses a border late in -the evening the country can be the next morning's. - -## Connect - -1. In Dawarich, open your account settings and copy the API key. -2. In HealthLog, turn on the Environment module, then open - **Settings → Integrations → Dawarich**. -3. Enter the address of the instance and the key, and connect. -4. Pick your home place from the list, and set your home country if you like. - Without one, the country you spent most days in over the last year counts. - -Run Dawarich and HealthLog in the same time zone. Dawarich groups the -timeline by its own zone, HealthLog by yours; if they differ, a day can come -back without its distances. - -## A Dawarich instance on your own network - -A public instance works without any server setting. An instance on a private -network (a LAN address, a `.lan` name, a container on the same host) is refused -unless the operator lists its exact origin: - -```env -DAWARICH_PRIVATE_ORIGINS="http://dawarich.lan:3000" -``` - -The grammar is the one `NIGHTSCOUT_PRIVATE_ORIGINS` uses -([Nightscout](./nightscout.md)): one `scheme://host[:port]` per entry, comma -separated, no path, no wildcard and no range. Listing a host does not open its -subdomains or another port. The unspecified address, link-local and the -cloud-metadata range can never be listed. Restart HealthLog after changing it. -The variable is on the `environment:` whitelist in `docker-compose.yml`. - -## Where the values show up - -- The day view has an **On the move** section with the day's figures. -- Time away from home and the distance travelled join the correlation scan as - two more factors. -- Runs of at least two travel days that no trip covers yet are offered as - trips under **Settings → Location & weather → Location periods**. Using one fills - in the dates; you pick the place. -- The Coach and the MCP `get_day` tool see the figures and whether it was a - travel day, never the country. - -## Disconnect - -Disconnecting stops the nightly fetch and keeps the stored days. **Disconnect -and delete days** removes them as well. The days are part of the backup; the -address and key are not, so a restored account connects again. +The Dawarich guide lives on the docs site: +. It covers what is read +and what is never stored, connecting an instance, allowing one on your own +network (`DAWARICH_PRIVATE_ORIGINS`), where the values show up, and +disconnecting. diff --git a/docs/integrations/strava.md b/docs/integrations/strava.md index 5607894fb..3e9bb2804 100644 --- a/docs/integrations/strava.md +++ b/docs/integrations/strava.md @@ -1,74 +1,6 @@ # Strava integration -HealthLog reads your Strava activities and folds them into the same workout -history as every other source. Strava is a **workout source only** — its API is -activity-centric and exposes no sleep, recovery, body-composition or glucose -data, so connecting it adds runs, rides, swims and the rest to your workout -surface and nothing else. - -The OAuth app is brought **per user**: each self-hoster registers their own -Strava API application and pastes the client id/secret into Settings. Strava -caps every newly-created app at "single-player mode" (an athlete capacity of 1) -until you request a capacity increase — perfectly fine for a personal -self-host, and the reason a single shared app cannot serve many operators. - -## What you get — and what you don't - -**Covered** (one `Workout` row per activity): sport type, start and end time, -moving duration, distance, elevation gain, average and maximum heart rate (when -the activity recorded it), and calories. Power, cadence, the "trainer"/"commute" -flags and the activity name are kept on the row's metadata for provenance but -are **not** surfaced as training analytics — HealthLog treats Strava as another -workout source, not a training-load platform. - -**Not covered**: Strava exposes no daily metrics (steps, sleep, resting heart -rate, weight, …). Those come from your wearable's own integration or from Apple -Health / Google Health Connect. - -## Duplicate activities are collapsed automatically - -If you record a run on an Apple Watch **and** it also lands in Strava, the same -run arrives twice — once via Apple Health, once via Strava. HealthLog keeps one -canonical row per logical workout using your **source-priority ladder**: by -default the device-native capture (Apple Watch, WHOOP, Withings) wins over the -Strava copy, because a Strava activity is often a re-upload with lower-fidelity -or missing heart-rate data. A run you record **only** on Strava (phone GPS) -survives untouched. You can promote Strava above the wearables in -**Settings → Source priority** if you prefer the Strava copy. - -## 1. Register a Strava API application - -1. Sign in at . -2. Create an application. For **Authorization Callback Domain** enter the host - of your HealthLog instance (the domain only, no scheme or path). -3. Copy the **Client ID** and **Client Secret**. - -## 2. Paste the credentials into HealthLog - -1. Open **Settings → Integrations** and find the **Strava** card. -2. Paste the Client ID and Client Secret into the **API Credentials** fields and - save. (An operator can instead configure a shared app via the - `STRAVA_CLIENT_ID` / `STRAVA_CLIENT_SECRET` environment variables; per-user - credentials always take precedence.) -3. Click **Connect Strava** and approve the access request. HealthLog requests - the `activity:read_all` scope so it can read every activity, including those - you marked "only me". - -## 3. First sync - -The first history import runs in the background shortly after you connect; -afterwards an hourly poll pulls new activities. Strava rotates its refresh token -on every refresh, and HealthLog persists the rotated token each time, so the -connection stays healthy without you re-authorising. - -## Disconnecting - -Disconnecting revokes the grant at Strava and clears the stored tokens. -Activities already imported stay in your history; reconnecting resumes the sync. - -## Related - -- Wearable that syncs to Strava but not directly to HealthLog? See the device's - own page, or ingest via [Apple Health](./apple-health.md) / - [Google Health Connect](./google-health.md). -- **Garmin owners:** Garmin has no direct connector — see [garmin.md](./garmin.md). +The Strava guide lives on the docs site: +. It covers what Strava +adds (workouts only), how duplicate activities from a wearable are collapsed, +registering the Strava API application, the first sync and disconnecting. From 93fb0f68d89651d7e8ec2d2a36db1c0a15f9c183 Mon Sep 17 00:00:00 2001 From: Usama Muhammad <46465291+muhdusama@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:17:59 +1100 Subject: [PATCH 10/44] fix(mcp): declare labs availability metadata (cherry picked from commit c1a5861fdcbfe374fcbb5362eed3717a5bd9b15d) --- src/lib/mcp/__tests__/tools.test.ts | 46 +++++++++++++++++++++++++++++ src/lib/mcp/tools.ts | 7 ++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/lib/mcp/__tests__/tools.test.ts b/src/lib/mcp/__tests__/tools.test.ts index 5b9d9a8c1..3de4475d4 100644 --- a/src/lib/mcp/__tests__/tools.test.ts +++ b/src/lib/mcp/__tests__/tools.test.ts @@ -510,6 +510,52 @@ describe("get_illness_recovery — condition body site", () => { }); describe("get_labs", () => { + it.each([{}, { analyte: "LDL" }])( + "declares empty-read availability for ordinary and filtered reads: %j", + async (args) => { + const { z } = await import("zod/v4"); + const empty = { + present: false, + reason: "out_of_window", + searchedWindow: "lastYear", + available: { + count: 2, + firstDate: "2020-01-01", + lastDate: "2020-01-02", + reachableWithWindow: null, + }, + }; + vi.mocked(executeCoachTool).mockResolvedValue(empty); + const def = tool("get_labs"); + const result = await def.run(CTX, args); + expect(executeCoachTool).toHaveBeenCalledWith({ + userId: CTX.userId, + name: "get_labs", + rawArguments: JSON.stringify(args), + }); + expect(z.object(def.outputShape!).strict().parse(result)).toEqual(empty); + }, + ); + + it("keeps output validation closed and availability typed", async () => { + const { z } = await import("zod/v4"); + const schema = z.object(tool("get_labs").outputShape!).strict(); + expect(z.toJSONSchema(schema).additionalProperties).toBe(false); + expect(schema.safeParse({ present: false, unexpected: true }).success).toBe( + false, + ); + expect( + schema.safeParse({ present: false, searchedWindow: 7 }).success, + ).toBe(false); + expect( + schema.safeParse({ present: false, available: { count: "2" } }).success, + ).toBe(false); + expect(schema.parse({ present: false, reason: "no_data" })).toEqual({ + present: false, + reason: "no_data", + }); + }); + it("forwards the optional analyte filter and returns readings with units + bands", async () => { vi.mocked(executeCoachTool).mockResolvedValue({ present: true, diff --git a/src/lib/mcp/tools.ts b/src/lib/mcp/tools.ts index 2c9813c30..c26fc1daf 100644 --- a/src/lib/mcp/tools.ts +++ b/src/lib/mcp/tools.ts @@ -1201,11 +1201,8 @@ const listMetricsOutput: z.ZodRawShape = { /** Output schema for `get_labs` — latest-per-biomarker OR a paginated history. */ const getLabsOutput: z.ZodRawShape = { - present: z.boolean(), - reason: z.string().optional(), - // Latest-mode payload (the Coach labs section, carried opaque). - data: z.unknown().optional(), - grounding: z.string().optional(), + // Latest-mode payload, including typed empty-read availability metadata. + ...coachReadOutput, // History-mode payload (per-analyte trajectory + pagination). analyte: z.string().optional(), readings: z From 6d3a1582fa3b75224d75254a950c4fb53b50ca19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 12:51:39 +0100 Subject: [PATCH 11/44] feat(backup): take a nightly whole-database dump with a manifest The per-account copies roll one account back on the same server; nothing the app made could rebuild a lost one. A scheduled job now runs pg_dump over the whole database (every schema, pg-boss included) into DB_BACKUP_DIR as healthlog--daily.dump, with a manifest beside it: app version and build, newest migration, Postgres version, size and SHA-256, exact row counts of every table, and the fingerprints of the keys the database needs. The counts read the same exported snapshot pg_dump reads, so a dump taken while the app writes still matches its manifest. The archive is written as .partial, listed back with pg_restore --list, and renamed after its manifest; a failed run leaves nothing named like a good dump. Retention removes only files this job wrote, oldest first, and never the newest good one (DB_BACKUP_KEEP, default 7; two pre-migration dumps). With the off-host bucket configured, each dump and its manifest go up under database/ in the envelope the account copies use, streamed. Every run lands in the new database_backup_runs ledger (migration 0486), and getDatabaseBackupHealth() answers the last good dump, the last failure, whether the directory is writable and whether off-host is configured. The boot readiness summary says where dumps go, or that none can be kept. The dump, naming, retention and restore rules live in one plain ESM module so the image's own scripts can share them without tsx. --- .../0486_database_backup_runs/migration.sql | 30 + prisma/schema.prisma | 29 + .../boot/__tests__/readiness-summary.test.ts | 53 + src/lib/boot/readiness-summary.ts | 78 +- src/lib/data-wipe/wipe-plan.ts | 2 + .../__tests__/dump-core.test.ts | 630 +++++++++ src/lib/database-backup/dump-core.mjs | 1183 +++++++++++++++++ src/lib/database-backup/health.ts | 117 ++ src/lib/database-backup/run.ts | 243 ++++ src/lib/jobs/database-backup-queue.ts | 40 + src/lib/jobs/job-outcome.ts | 6 + src/lib/jobs/offhost-backup.ts | 8 +- src/lib/jobs/queue-runtime.ts | 12 + src/lib/jobs/registry/maintenance.ts | 18 +- src/lib/jobs/reminder/backup-handlers.ts | 33 +- src/lib/jobs/subsystem-surface.ts | 1 + .../__golden__/queue-registration.json | 40 +- tests/integration/database-backup.test.ts | 511 +++++++ 18 files changed, 3025 insertions(+), 9 deletions(-) create mode 100644 prisma/migrations/0486_database_backup_runs/migration.sql create mode 100644 src/lib/database-backup/__tests__/dump-core.test.ts create mode 100644 src/lib/database-backup/dump-core.mjs create mode 100644 src/lib/database-backup/health.ts create mode 100644 src/lib/database-backup/run.ts create mode 100644 src/lib/jobs/database-backup-queue.ts create mode 100644 tests/integration/database-backup.test.ts diff --git a/prisma/migrations/0486_database_backup_runs/migration.sql b/prisma/migrations/0486_database_backup_runs/migration.sql new file mode 100644 index 000000000..37bd430af --- /dev/null +++ b/prisma/migrations/0486_database_backup_runs/migration.sql @@ -0,0 +1,30 @@ +-- The nightly whole-database dump records each run: whether a dump was +-- written in DB_BACKUP_DIR and copied to the off-host bucket, and why not. +-- Instance metadata with no account; the admin backups card and the backup +-- alerts read the newest rows. +-- +-- The table is new, so nothing waits on its lock in practice; the wait is +-- bounded in a DO block anyway, and every statement is safe to run again, so +-- the boot retries the file on a lock timeout. +-- +-- @retry-on-lock-timeout + +DO $migration$ +BEGIN + PERFORM set_config('lock_timeout', '5s', true); + CREATE TABLE IF NOT EXISTS "database_backup_runs" ( + "id" TEXT NOT NULL, + "started_at" TIMESTAMP(3) NOT NULL, + "finished_at" TIMESTAMP(3), + "status" TEXT NOT NULL, + "file_name" TEXT, + "dump_bytes" BIGINT, + "local_result" TEXT NOT NULL, + "offhost_result" TEXT NOT NULL, + "error" TEXT, + CONSTRAINT "database_backup_runs_pkey" PRIMARY KEY ("id") + ); + CREATE INDEX IF NOT EXISTS "database_backup_runs_started_at_idx" + ON "database_backup_runs"("started_at"); +END +$migration$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8397e0cb5..ae2d0c292 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5929,6 +5929,35 @@ model OffhostPurgeRequest { @@map("offhost_purge_requests") } +/// One run of the nightly whole-database dump (the `database-backup` job): +/// whether a dump was written on this server and copied off-host, and why not +/// when it was not. Instance metadata, not an account's data. Written by +/// `runDatabaseBackup`; read by `getDatabaseBackupHealth`, which feeds the +/// database row of the admin backups card and the backup alerts. A row left +/// `running` by a process that died is read as a failed run once the job's +/// expiry has passed. +model DatabaseBackupRun { + id String @id @default(cuid()) + startedAt DateTime @map("started_at") + finishedAt DateTime? @map("finished_at") + /// `running`, `succeeded`, `failed` or `skipped`. + status String + /// The dump's file name, once one was written and checked. + fileName String? @map("file_name") + /// Size of the dump in bytes. + dumpBytes BigInt? @map("dump_bytes") + /// `written` (kept in DB_BACKUP_DIR), `no_directory` (no writable + /// directory, the dump went off-host only), or `failed`. + localResult String @map("local_result") + /// `uploaded`, `not_configured` or `failed`. + offhostResult String @map("offhost_result") + /// Why the run failed or was skipped, trimmed. Never key material. + error String? + + @@index([startedAt]) + @@map("database_backup_runs") +} + // ─── Apple Health export.zip import (v1.4.34) ─────────────── // // Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the diff --git a/src/lib/boot/__tests__/readiness-summary.test.ts b/src/lib/boot/__tests__/readiness-summary.test.ts index fc23b92ee..003ade157 100644 --- a/src/lib/boot/__tests__/readiness-summary.test.ts +++ b/src/lib/boot/__tests__/readiness-summary.test.ts @@ -149,3 +149,56 @@ describe("collectReadiness — retired AI grant", () => { expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined(); }); }); + +describe("collectReadiness — the database dump", () => { + const offhost = { + BACKUP_S3_ENDPOINT: "https://s3.example.test", + BACKUP_S3_BUCKET: "b", + BACKUP_S3_ACCESS_KEY: "a", + BACKUP_S3_SECRET_KEY: "s", + BACKUP_ENCRYPTION_KEY: HEX32, + }; + const dumpLine = ( + env: Record, + writable: boolean, + ) => + collectReadiness(env, () => writable).lines.find( + (l) => l.label === "Database dump", + ); + + it("names the schedule, the directory and the copies kept", () => { + const line = dumpLine(baseEnv(), true); + expect(line?.status).toBe("ok"); + expect(line?.detail).toBe( + "nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7", + ); + expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain( + "copied off-host", + ); + }); + + it("warns when nothing can be kept on the server", () => { + expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({ + status: "warn", + detail: expect.stringContaining("goes off-host only"), + }); + expect(dumpLine(baseEnv(), false)).toMatchObject({ + status: "warn", + detail: expect.stringContaining("no database dump can be taken"), + }); + }); + + it("warns when it is switched off, and names a schedule it ignored", () => { + expect( + dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true), + ).toMatchObject({ status: "warn" }); + const report = collectReadiness( + { ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" }, + () => true, + ); + expect( + report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail, + ).toBe("not a five-field cron; using 30 1 * * *"); + expect(report.hasBlocker).toBe(false); + }); +}); diff --git a/src/lib/boot/readiness-summary.ts b/src/lib/boot/readiness-summary.ts index 14e707157..e8907cb29 100644 --- a/src/lib/boot/readiness-summary.ts +++ b/src/lib/boot/readiness-summary.ts @@ -19,6 +19,13 @@ import { LEGACY_ANY_HOST_WARNING, legacyAnyHostConfigured, } from "@/lib/ai/local-host-allowlist"; +import { + databaseBackupEnabled, + isWritableDirectory, + resolveBackupDir, + resolveBackupSchedule, + resolveDailyKeep, +} from "@/lib/database-backup/dump-core.mjs"; export interface ReadinessLine { /** Short label, e.g. "ENCRYPTION_KEY". */ @@ -60,8 +67,73 @@ function anySet(env: Env, keys: string[]): boolean { return keys.some((k) => (env[k] ?? "").trim().length > 0); } -/** Inspect the environment and build the readiness report. Pure + side-effect free. */ -export function collectReadiness(env: Env = process.env): ReadinessReport { +/** + * The nightly database dump's line. The dump is what rebuilds a lost server, + * so a host where it cannot be kept anywhere is said out loud at every boot, + * and so is one where it only goes off-host. + */ +export function databaseBackupReadiness( + env: Env, + writable: (dir: string) => boolean, +): ReadinessLine[] { + const label = "Database dump"; + if (!databaseBackupEnabled(env)) { + return [ + { + label, + status: "warn", + detail: + "DB_BACKUP_ENABLED=false: no nightly database dump is taken; back the database up another way", + }, + ]; + } + const lines: ReadinessLine[] = []; + const schedule = resolveBackupSchedule(env); + if (!schedule.valid) { + lines.push({ + label: "DB_BACKUP_SCHEDULE", + status: "warn", + detail: `not a five-field cron; using ${schedule.cron}`, + }); + } + const dir = resolveBackupDir(env); + const offhost = allSet(env, [ + "BACKUP_S3_ENDPOINT", + "BACKUP_S3_BUCKET", + "BACKUP_S3_ACCESS_KEY", + "BACKUP_S3_SECRET_KEY", + "BACKUP_ENCRYPTION_KEY", + ]); + if (writable(dir)) { + lines.push({ + label, + status: "ok", + detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`, + }); + } else if (offhost) { + lines.push({ + label, + status: "warn", + detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`, + }); + } else { + lines.push({ + label, + status: "warn", + detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`, + }); + } + return lines; +} + +/** + * Inspect the environment and build the readiness report. Pure apart from + * `writable`, the probe of the dump directory, which tests replace. + */ +export function collectReadiness( + env: Env = process.env, + writable: (dir: string) => boolean = isWritableDirectory, +): ReadinessReport { const lines: ReadinessLine[] = []; // --- Core secrets (boot blockers) --------------------------------------- @@ -139,6 +211,8 @@ export function collectReadiness(env: Env = process.env): ReadinessReport { }); } + lines.push(...databaseBackupReadiness(env, writable)); + // --- Optional subsystems (informational) -------------------------------- const optional: Array<[string, boolean]> = [ ["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])], diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 92cead915..211e00af6 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -353,6 +353,8 @@ export const INSTANCE_SCOPED: Readonly> = { HostMetric: "instance-wide host telemetry, not attributable to an account", OffhostBackupKeyUse: "per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account", + DatabaseBackupRun: + "the instance's ledger of nightly whole-database dumps; describes the server, not any account", OffhostPurgeRequest: "the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account", CycleSymptomCategory: diff --git a/src/lib/database-backup/__tests__/dump-core.test.ts b/src/lib/database-backup/__tests__/dump-core.test.ts new file mode 100644 index 000000000..fad98e4a3 --- /dev/null +++ b/src/lib/database-backup/__tests__/dump-core.test.ts @@ -0,0 +1,630 @@ +/** + * The rules of the whole-database dump that need no database: names, + * retention, the manifest, key fingerprints, row-count comparison, the + * connection the client tools get, and the off-host envelope read back by the + * restore's own reader. + * + * The fingerprints and the envelope are checked against the app's own code + * (`src/lib/crypto.ts`, `src/lib/jobs/offhost-backup.ts`), not against a copy + * of the rule: the restore script cannot import TypeScript, so these are the + * tests that keep its plain-JS reading of both from drifting. + */ +import { + mkdtempSync, + createReadStream, + rmSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + applyRetention, + buildManifest, + compareKeyFingerprints, + compareRowCounts, + connectionFromUrl, + decodeKeyMaterial, + decryptOffhostFile, + dumpBaseName, + formatDumpTimestamp, + keyFingerprints, + listDumpSets, + offhostObjectKey, + parseDumpFileName, + parseManifest, + resolveBackupSchedule, + resolveDailyKeep, + selectForDeletion, + type DumpSet, +} from "../dump-core.mjs"; +import type { S3Like } from "@/lib/jobs/offhost-backup"; + +/** Every byte of a stream, in order. */ +async function collect(stream: AsyncIterable): Promise { + const chunks: Buffer[] = []; + for await (const c of stream) chunks.push(Buffer.from(c as Uint8Array)); + return Buffer.concat(chunks); +} + +const HEX_A = "a".repeat(64); +const HEX_B = "b".repeat(64); +const HEX_C = "c".repeat(64); +const HEX_D = "d".repeat(64); + +const dirs: string[] = []; +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), "hl-dump-core-")); + dirs.push(dir); + return dir; +} +afterEach(() => { + for (const dir of dirs.splice(0)) + rmSync(dir, { recursive: true, force: true }); + vi.unstubAllEnvs(); +}); + +describe("dump names", () => { + it("names a dump by its UTC instant and kind", () => { + const at = new Date("2026-10-25T01:30:07.123Z"); + expect(formatDumpTimestamp(at)).toBe("20261025T013007Z"); + expect(dumpBaseName("daily", at)).toBe("healthlog-20261025T013007Z-daily"); + expect(dumpBaseName("pre-migration", at)).toBe( + "healthlog-20261025T013007Z-pre-migration", + ); + }); + + it("parses every file it writes, and back to the same instant", () => { + const base = "healthlog-20261025T013007Z-daily"; + const cases: Array<[string, string, string]> = [ + [`${base}.dump`, "dump", "final"], + [`${base}.manifest.json`, "manifest", "final"], + [`${base}.dump.partial`, "dump", "partial"], + [`${base}.manifest.json.partial`, "manifest", "partial"], + [`${base}.dump.enc`, "dump", "encrypted"], + [`${base}.manifest.json.enc`, "manifest", "encrypted"], + ]; + for (const [name, role, variant] of cases) { + const parsed = parseDumpFileName(name); + expect(parsed, name).not.toBeNull(); + expect(parsed?.base).toBe(base); + expect(parsed?.kind).toBe("daily"); + expect(parsed?.role).toBe(role); + expect(parsed?.variant).toBe(variant); + expect(parsed?.timestamp.toISOString()).toBe("2026-10-25T01:30:07.000Z"); + } + expect( + parseDumpFileName("healthlog-20261025T013007Z-pre-migration.dump")?.kind, + ).toBe("pre-migration"); + }); + + it("refuses every name it did not write", () => { + for (const name of [ + "healthlog.dump", + "backup-20261025T013007Z-daily.dump", + "healthlog-20261025T013007Z-weekly.dump", + "healthlog-20261025-daily.dump", + "healthlog-20261025T013007Z-daily.sql", + "healthlog-20261025T013007Z-daily.dump.bak", + "healthlog-20261399T013007Z-daily.dump", + "notes.txt", + ]) { + expect(parseDumpFileName(name), name).toBeNull(); + } + }); +}); + +describe("configuration", () => { + it("keeps at least one daily dump and ignores what is not a number", () => { + expect(resolveDailyKeep({})).toBe(7); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "14" })).toBe(14); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "0" })).toBe(1); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "-3" })).toBe(7); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "seven" })).toBe(7); + }); + + it("takes a five-field cron and says when it ignored one", () => { + expect(resolveBackupSchedule({})).toEqual({ + cron: "30 1 * * *", + valid: true, + configured: false, + }); + expect( + resolveBackupSchedule({ DB_BACKUP_SCHEDULE: " 0 2 * * 1-5 " }), + ).toEqual({ + cron: "0 2 * * 1-5", + valid: true, + configured: true, + }); + for (const bad of ["daily", "0 2 * *", "0 2 * * * *", "0 2 * * mon; rm"]) { + expect(resolveBackupSchedule({ DB_BACKUP_SCHEDULE: bad })).toEqual({ + cron: "30 1 * * *", + valid: false, + configured: true, + }); + } + }); +}); + +/** A dump set as `listDumpSets` reports it, for the pure retention rule. */ +function set( + kind: "daily" | "pre-migration", + iso: string, + parts: { dump?: boolean; manifest?: boolean; partialAgeMs?: number } = {}, + now = Date.parse("2026-10-25T12:00:00Z"), +): DumpSet { + const base = dumpBaseName(kind, new Date(iso)); + return { + base, + kind, + timestamp: new Date(iso), + dump: parts.dump === false ? null : { path: `/b/${base}.dump`, bytes: 1 }, + manifest: parts.manifest === false ? null : `/b/${base}.manifest.json`, + encryptedDump: null, + encryptedManifest: null, + leftovers: + parts.partialAgeMs === undefined + ? [] + : [ + { + path: `/b/${base}.dump.partial`, + mtimeMs: now - parts.partialAgeMs, + }, + ], + }; +} + +describe("retention", () => { + const now = Date.parse("2026-10-25T12:00:00Z"); + + it("keeps the newest N daily dumps and removes the rest, both files each", () => { + const sets = [1, 2, 3, 4, 5].map((d) => + set("daily", `2026-10-2${d}T01:30:00Z`), + ); + const doomed = selectForDeletion(sets, { + keep: { daily: 3, "pre-migration": 2 }, + now, + }); + expect(doomed.sort()).toEqual( + [ + "/b/healthlog-20261021T013000Z-daily.dump", + "/b/healthlog-20261021T013000Z-daily.manifest.json", + "/b/healthlog-20261022T013000Z-daily.dump", + "/b/healthlog-20261022T013000Z-daily.manifest.json", + ].sort(), + ); + }); + + it("never removes the newest good dump, whatever keep says", () => { + const sets = [ + set("daily", "2026-10-24T01:30:00Z"), + set("daily", "2026-10-23T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 0, "pre-migration": 0 }, + now, + }); + expect(doomed).not.toContain("/b/healthlog-20261024T013000Z-daily.dump"); + expect(doomed).toContain("/b/healthlog-20261023T013000Z-daily.dump"); + }); + + it("counts only good dumps: an unfinished newer one does not push out the newest good one", () => { + const sets = [ + set("daily", "2026-10-25T01:30:00Z", { + dump: false, + partialAgeMs: 60_000, + }), + set("daily", "2026-10-24T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 1, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([]); + }); + + it("keeps two pre-migration dumps apart from the daily count", () => { + const sets = [ + set("pre-migration", "2026-10-20T08:00:00Z"), + set("pre-migration", "2026-10-10T08:00:00Z"), + set("pre-migration", "2026-10-01T08:00:00Z"), + set("daily", "2026-10-24T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 1, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([ + "/b/healthlog-20261001T080000Z-pre-migration.dump", + "/b/healthlog-20261001T080000Z-pre-migration.manifest.json", + ]); + }); + + it("removes a .partial only once it is older than twelve hours", () => { + const fresh = set("daily", "2026-10-25T01:30:00Z", { + dump: false, + manifest: false, + partialAgeMs: 60 * 60_000, + }); + const stale = set("daily", "2026-10-23T01:30:00Z", { + dump: false, + manifest: false, + partialAgeMs: 13 * 60 * 60_000, + }); + const doomed = selectForDeletion([fresh, stale], { + keep: { daily: 7, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([ + "/b/healthlog-20261023T013000Z-daily.dump.partial", + ]); + }); + + it("works on a real directory and leaves files it did not write alone", async () => { + const dir = tempDir(); + const names: string[] = []; + for (let day = 15; day <= 24; day++) { + const base = `healthlog-202610${day}T013000Z-daily`; + names.push(`${base}.dump`, `${base}.manifest.json`); + } + // Two that are not ours, and an off-host copy an operator put here. + names.push( + "my-own-dump.dump", + "healthlog-20261001T013000Z-daily.dump.bak", + "healthlog-20261002T013000Z-daily.dump.enc", + ); + for (const name of names) writeFileSync(join(dir, name), "x"); + // A crash's leftovers: one old, one from a run that may still be going. + const oldPartial = join( + dir, + "healthlog-20261003T013000Z-daily.dump.partial", + ); + writeFileSync(oldPartial, "x"); + const day = 24 * 60 * 60; + utimesSync(oldPartial, Date.now() / 1000 - day, Date.now() / 1000 - day); + writeFileSync( + join(dir, "healthlog-20261025T013000Z-daily.dump.partial"), + "x", + ); + + const removed = await applyRetention(dir, { dailyKeep: 7 }); + expect(removed).toHaveLength(7); + const left = (await readdir(dir)).sort(); + expect(left).toContain("my-own-dump.dump"); + expect(left).toContain("healthlog-20261001T013000Z-daily.dump.bak"); + expect(left).toContain("healthlog-20261002T013000Z-daily.dump.enc"); + expect(left).toContain("healthlog-20261025T013000Z-daily.dump.partial"); + expect(left).not.toContain("healthlog-20261003T013000Z-daily.dump.partial"); + expect(left.filter((n) => n.endsWith("-daily.dump"))).toEqual( + [18, 19, 20, 21, 22, 23, 24].map( + (d) => `healthlog-202610${d}T013000Z-daily.dump`, + ), + ); + expect(listDumpSets(dir)[1].base).toBe("healthlog-20261024T013000Z-daily"); + }); +}); + +describe("key fingerprints", () => { + it("are the fingerprints the app's own key code gives, for the map and the single key", async () => { + vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ v1: HEX_A, v2: HEX_B })); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "v2"); + vi.stubEnv("ENCRYPTION_KEY", ""); + const crypto = await import("@/lib/crypto"); + crypto._resetCryptoCacheForTests(); + const prints = keyFingerprints(process.env); + expect(prints.encryption.activeKeyId).toBe("v2"); + expect(prints.encryption.keys).toEqual([ + { id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }, + { id: "v2", fingerprint: crypto.getKeyFingerprint("v2") }, + ]); + + vi.stubEnv("ENCRYPTION_KEYS", ""); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", ""); + const base64 = Buffer.from(HEX_C, "hex").toString("base64"); + vi.stubEnv("ENCRYPTION_KEY", base64); + crypto._resetCryptoCacheForTests(); + const single = keyFingerprints(process.env); + expect(single.encryption).toEqual({ + activeKeyId: "v1", + keys: [{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }], + }); + crypto._resetCryptoCacheForTests(); + }); + + it("names the off-host key the way an off-host object records it", async () => { + const { offhostKeyId } = await import("@/lib/jobs/offhost-backup"); + const prints = keyFingerprints({ + ENCRYPTION_KEY: HEX_A, + BACKUP_ENCRYPTION_KEY: HEX_D, + BACKUP_ENCRYPTION_PREVIOUS_KEYS: ` ${HEX_C} ,`, + API_TOKEN_HMAC_KEY: "hmac-secret-of-some-length-0123456789", + }); + expect(prints.backupEncryptionKey).toBe( + offhostKeyId(Buffer.from(HEX_D, "hex")), + ); + expect(prints.backupEncryptionPreviousKeys).toEqual([ + offhostKeyId(Buffer.from(HEX_C, "hex")), + ]); + expect(prints.apiTokenHmacKey).toMatch(/^[0-9a-f]{12}$/); + }); + + it("never carries key material", () => { + const env = { + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A }), + BACKUP_ENCRYPTION_KEY: HEX_B, + API_TOKEN_HMAC_KEY: HEX_C, + }; + const text = JSON.stringify(keyFingerprints(env)); + for (const secret of [HEX_A, HEX_B, HEX_C]) { + expect(text).not.toContain(secret.slice(0, 16)); + } + }); + + it("decodes keys the way the app does, and nothing that is not a key", () => { + expect(decodeKeyMaterial(HEX_A)?.length).toBe(32); + expect( + decodeKeyMaterial(Buffer.alloc(32, 7).toString("base64"))?.length, + ).toBe(32); + expect(decodeKeyMaterial("")).toBeNull(); + expect(decodeKeyMaterial("not a key!")).toBeNull(); + }); +}); + +describe("comparing a dump's keys with the environment's", () => { + const written = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_B }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + + it("finds nothing when every key is there", () => { + const same = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B, v1: HEX_A }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + expect(compareKeyFingerprints(written, same)).toEqual({ + findings: [], + refuse: false, + }); + }); + + it("refuses when the active key is different, and says which", () => { + const wrong = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_D }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + const result = compareKeyFingerprints(written, wrong); + expect(result.refuse).toBe(true); + expect(result.findings).toHaveLength(1); + expect(result.findings[0]).toMatchObject({ + key: "ENCRYPTION_KEYS[v2]", + state: "different", + severity: "refuse", + }); + }); + + it("refuses when the active key is missing, warns for a retired one", () => { + const onlyOld = keyFingerprints({ ENCRYPTION_KEY: HEX_A }); + const result = compareKeyFingerprints(written, onlyOld); + expect(result.refuse).toBe(true); + const byKey = Object.fromEntries(result.findings.map((f) => [f.key, f])); + expect(byKey["ENCRYPTION_KEYS[v2]"]).toMatchObject({ + state: "missing", + severity: "refuse", + }); + expect(byKey["ENCRYPTION_KEYS[v1]"]).toBeUndefined(); + expect(byKey.BACKUP_ENCRYPTION_KEY).toMatchObject({ + state: "missing", + severity: "warn", + }); + expect(byKey.API_TOKEN_HMAC_KEY).toMatchObject({ + state: "missing", + severity: "warn", + }); + + const noRetired = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B }), + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-two", + }); + const second = compareKeyFingerprints(written, noRetired); + expect(second.refuse).toBe(false); + expect(second.findings.map((f) => [f.key, f.severity])).toEqual([ + ["ENCRYPTION_KEYS[v1]", "warn"], + ["API_TOKEN_HMAC_KEY", "warn"], + ]); + }); +}); + +describe("the manifest", () => { + const manifest = buildManifest({ + kind: "daily", + file: "healthlog-20261025T013000Z-daily.dump", + createdAt: new Date("2026-10-25T01:30:00Z"), + appVersion: "1.46.0", + buildSha: "abc1234", + latestMigration: "0486_database_backup_runs", + server: { version: "18.6", versionNum: 180006 }, + pgDump: "pg_dump (PostgreSQL) 18.6", + archive: { bytes: 1234, sha256: "f".repeat(64), entries: 900 }, + rowCounts: { "public.users": 3, "pgboss.job": 12, "public.a": 0 }, + env: { ENCRYPTION_KEY: HEX_A, API_TOKEN_HMAC_KEY: "h" }, + }); + + it("carries what a restore and an operator need", () => { + expect(manifest).toMatchObject({ + format: "healthlog-database-dump", + formatVersion: 1, + kind: "daily", + file: "healthlog-20261025T013000Z-daily.dump", + createdAt: "2026-10-25T01:30:00.000Z", + appVersion: "1.46.0", + buildSha: "abc1234", + latestMigration: "0486_database_backup_runs", + postgres: { serverVersion: "18.6", serverVersionNum: 180006 }, + dump: { bytes: 1234, sha256: "f".repeat(64), entries: 900 }, + }); + expect(Object.keys(manifest.rowCounts)).toEqual([ + "pgboss.job", + "public.a", + "public.users", + ]); + expect(manifest.keys.encryption.keys[0].id).toBe("v1"); + }); + + it("reads back, and refuses anything else", () => { + expect(parseManifest(JSON.parse(JSON.stringify(manifest)))).toEqual( + manifest, + ); + expect(() => parseManifest({ format: "something-else" })).toThrow( + /not a HealthLog/, + ); + expect(() => parseManifest({ ...manifest, formatVersion: 2 })).toThrow(); + expect(() => parseManifest(null)).toThrow(); + }); +}); + +describe("row counts after a restore", () => { + it("names every table that is missing, extra or different", () => { + expect( + compareRowCounts( + { "public.a": 1, "public.b": 2 }, + { "public.a": 1, "public.b": 2 }, + ), + ).toEqual([]); + expect( + compareRowCounts( + { "public.a": 1, "public.b": 2, "public.c": 5 }, + { "public.a": 1, "public.b": 0, "public.d": 1 }, + ), + ).toEqual([ + "public.b: 2 rows in the dump, 0 restored", + "public.c: in the dump with 5 rows, missing now", + "public.d: restored but not in the dump's manifest", + ]); + }); +}); + +describe("the connection the client tools get", () => { + it("drops Prisma's parameters and keeps the password out of the arguments", () => { + const c = connectionFromUrl( + "postgresql://health%40log:p%40ss%2Fw@db:6543/hl?schema=app&connection_limit=20&pool_timeout=20&sslmode=require", + ); + expect(c.pgEnv).toEqual({ + PGHOST: "db", + PGPORT: "6543", + PGUSER: "health@log", + PGPASSWORD: "p@ss/w", + PGDATABASE: "hl", + PGSSLMODE: "require", + }); + expect(c.schema).toBe("app"); + const cs = String(c.clientConfig.connectionString); + expect(cs).not.toContain("connection_limit"); + expect(cs).not.toContain("schema="); + expect(cs).toContain("sslmode=require"); + }); +}); + +describe("an off-host copy read back by the restore", () => { + /** The uploader the job uses, into memory. */ + async function upload(objectKey: string, key: Buffer, body: Buffer) { + const { uploadEncryptedBackup } = await import("@/lib/jobs/offhost-backup"); + const stored = new Map(); + const s3: S3Like = { + putStream: async (k: string, stream: Readable) => { + stored.set(k, await collect(stream)); + }, + putObject: async () => {}, + getObject: async () => Buffer.alloc(0), + headObject: async () => false, + listObjects: async () => [], + deleteObject: async () => {}, + }; + await uploadEncryptedBackup(s3, objectKey, key, async (write) => { + // Several pieces, as the job streams the file. + for (let i = 0; i < body.length; i += 1000) { + await write(body.subarray(i, i + 1000)); + } + }); + return stored.get(objectKey) as Buffer; + } + + it("opens what the job's uploader sealed, byte for byte", async () => { + const dir = tempDir(); + const key = Buffer.from(HEX_D, "hex"); + const body = Buffer.from(Array.from({ length: 50_000 }, (_, i) => i % 251)); + const name = "healthlog-20261025T013000Z-daily.dump.enc"; + const sealed = await upload(offhostObjectKey(name), key, body); + writeFileSync(join(dir, name), sealed); + + await decryptOffhostFile({ + inPath: join(dir, name), + outPath: join(dir, "opened.dump"), + objectKey: offhostObjectKey(name), + keys: [Buffer.from(HEX_A, "hex"), key], + }); + expect( + (await collect(createReadStream(join(dir, "opened.dump")))).equals(body), + ).toBe(true); + }); + + it("refuses a copy renamed, altered or under a key it does not hold, and leaves nothing", async () => { + const dir = tempDir(); + const key = Buffer.from(HEX_D, "hex"); + const body = Buffer.from("PGDMP fake archive bytes".repeat(100)); + const name = "healthlog-20261025T013000Z-daily.dump.enc"; + const sealed = await upload(offhostObjectKey(name), key, body); + writeFileSync(join(dir, name), sealed); + const out = join(dir, "opened.dump"); + + // Renamed: the object key is bound as associated data. + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey( + "healthlog-20261024T013000Z-daily.dump.enc", + ), + keys: [key], + }), + ).rejects.toThrow(/does not open/); + expect(await readdir(dir)).toEqual([name]); + + // Altered. + const altered = Buffer.from(sealed); + altered[altered.length - 40] ^= 1; + writeFileSync(join(dir, name), altered); + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey(name), + keys: [key], + }), + ).rejects.toThrow(); + expect(await readdir(dir)).toEqual([name]); + + // A key it was not written under. + writeFileSync(join(dir, name), sealed); + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey(name), + keys: [Buffer.from(HEX_A, "hex")], + }), + ).rejects.toThrow(/neither BACKUP_ENCRYPTION_KEY nor/); + expect(await readdir(dir)).toEqual([name]); + }); +}); diff --git a/src/lib/database-backup/dump-core.mjs b/src/lib/database-backup/dump-core.mjs new file mode 100644 index 000000000..14f13d2ca --- /dev/null +++ b/src/lib/database-backup/dump-core.mjs @@ -0,0 +1,1183 @@ +/** + * The whole-database dump: naming, writing, checking, keeping and reading it + * back. One module for three callers that cannot share TypeScript: + * + * - the nightly job in the app (`src/lib/database-backup/run.ts`), bundled; + * - the pre-migration dump the entrypoint takes (`scripts/db-dump.mjs`); + * - the restore (`scripts/db-restore.mjs`), which runs in the image with + * plain `node` on a server that has nothing but the dump and the keys. + * + * It is plain ESM with no imports beyond Node and `pg`, for the same reason + * `src/lib/auth/argon2-params.mjs` is: the production image strips `tsx`, and + * a restore that needs a source checkout is not a restore. Every rule that + * decides whether a dump is good, which files may be deleted and whether the + * keys fit lives here once, so the writer and the reader cannot drift. + * + * What a dump is: `pg_dump --format=custom` of the whole database (every + * schema, pg-boss included), named `healthlog--.dump`, + * with `healthlog--.manifest.json` beside it. A dump is + * good only when both exist under their final names: the archive is written + * as `.partial`, read back with `pg_restore --list`, the manifest is written + * and renamed first, and the archive is renamed last. A run that dies at any + * point leaves nothing that looks good. + * + * The row counts in the manifest come from the same snapshot `pg_dump` reads + * (`pg_export_snapshot()` + `--snapshot`), so a dump taken while the app is + * writing still restores to exactly the counts it claims. + */ +import { spawn } from "node:child_process"; +import { createDecipheriv, createHash } from "node:crypto"; +import { + accessSync, + closeSync, + constants as fsConstants, + createReadStream, + createWriteStream, + mkdirSync, + openSync, + readSync, + readdirSync, + statSync, +} from "node:fs"; +import { readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; + +/** Where dumps are written when `DB_BACKUP_DIR` is unset. */ +export const DEFAULT_BACKUP_DIR = "/var/lib/healthlog/backups"; + +/** Daily dumps kept when `DB_BACKUP_KEEP` is unset. */ +export const DEFAULT_DAILY_KEEP = 7; + +/** Pre-migration dumps kept: the one before this update and the one before. */ +export const PRE_MIGRATION_KEEP = 2; + +/** The cron the nightly dump runs on when `DB_BACKUP_SCHEDULE` is unset. */ +export const DEFAULT_BACKUP_SCHEDULE = "30 1 * * *"; + +/** The kinds of dump this module writes. */ +export const DUMP_KINDS = /** @type {const} */ (["daily", "pre-migration"]); + +/** The manifest's own format marker. */ +export const MANIFEST_FORMAT = "healthlog-database-dump"; +export const MANIFEST_FORMAT_VERSION = 1; + +/** The bucket prefix off-host database dumps live under. */ +export const OFFHOST_DATABASE_PREFIX = "database/"; + +/** + * A leftover `.partial` or an orphaned manifest younger than this may belong + * to a run that is still going (another container, the entrypoint), so it is + * left alone. + */ +const STALE_LEFTOVER_MS = 12 * 60 * 60 * 1000; + +const NAME_PATTERN = + /^healthlog-(\d{8}T\d{6}Z)-(daily|pre-migration)\.(dump|manifest\.json)(\.partial|\.enc)?$/; + +/** + * @typedef {"daily" | "pre-migration"} DumpKind + * @typedef {{ + * base: string; + * kind: DumpKind; + * timestamp: Date; + * role: "dump" | "manifest"; + * variant: "final" | "partial" | "encrypted"; + * }} ParsedDumpName + */ + +/** `20261025T013000Z` for an instant, in UTC. */ +export function formatDumpTimestamp(date) { + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[-:]/g, ""); +} + +/** + * The shared stem of a dump and its manifest. + * + * @param {DumpKind} kind + * @param {Date} date + */ +export function dumpBaseName(kind, date) { + return `healthlog-${formatDumpTimestamp(date)}-${kind}`; +} + +/** @param {string} base */ +export function dumpFileName(base) { + return `${base}.dump`; +} + +/** @param {string} base */ +export function manifestFileName(base) { + return `${base}.manifest.json`; +} + +/** + * Parse a file name this module writes. Anything else answers null, which is + * what keeps retention away from files it did not write. + * + * @param {string} name + * @returns {ParsedDumpName | null} + */ +export function parseDumpFileName(name) { + const m = NAME_PATTERN.exec(name); + if (!m) return null; + const [, stamp, kind, role, suffix] = m; + const iso = `${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)}T${stamp.slice(9, 11)}:${stamp.slice(11, 13)}:${stamp.slice(13, 15)}Z`; + const timestamp = new Date(iso); + if (Number.isNaN(timestamp.getTime())) return null; + return { + base: `healthlog-${stamp}-${kind}`, + kind: /** @type {DumpKind} */ (kind), + timestamp, + role: role === "dump" ? "dump" : "manifest", + variant: + suffix === ".partial" + ? "partial" + : suffix === ".enc" + ? "encrypted" + : "final", + }; +} + +// ── Configuration ────────────────────────────────────────────────────────── + +/** @typedef {Record} Env */ + +/** @param {string | undefined} raw */ +function isOff(raw) { + const v = (raw ?? "").trim().toLowerCase(); + return v === "false" || v === "0" || v === "no" || v === "off"; +} + +/** `DB_BACKUP_ENABLED`, on unless it says off. @param {Env} env */ +export function databaseBackupEnabled(env = process.env) { + return !isOff(env.DB_BACKUP_ENABLED); +} + +/** `DB_BACKUP_BEFORE_MIGRATE`, on unless it says off. @param {Env} env */ +export function preMigrationDumpEnabled(env = process.env) { + return !isOff(env.DB_BACKUP_BEFORE_MIGRATE); +} + +/** `DB_BACKUP_DIR`, or the image's default. @param {Env} env */ +export function resolveBackupDir(env = process.env) { + return (env.DB_BACKUP_DIR ?? "").trim() || DEFAULT_BACKUP_DIR; +} + +/** + * `DB_BACKUP_KEEP`: how many daily dumps stay on disk. At least one, so the + * newest good dump is never deleted; a value that is not a whole number + * keeps the default. + * + * @param {Env} env + */ +export function resolveDailyKeep(env = process.env) { + const raw = (env.DB_BACKUP_KEEP ?? "").trim(); + if (!/^\d+$/.test(raw)) return DEFAULT_DAILY_KEEP; + return Math.min(Math.max(Number.parseInt(raw, 10), 1), 366); +} + +const CRON_FIELD = /^[0-9*,/-]+$/; + +/** + * `DB_BACKUP_SCHEDULE` when it is a five-field cron of digits, `*`, `,`, `/` + * and `-`, otherwise the default. `valid` says which, so the boot summary can + * name a schedule it ignored. + * + * @param {Env} env + * @returns {{ cron: string; valid: boolean; configured: boolean }} + */ +export function resolveBackupSchedule(env = process.env) { + const raw = (env.DB_BACKUP_SCHEDULE ?? "").trim(); + if (!raw) { + return { cron: DEFAULT_BACKUP_SCHEDULE, valid: true, configured: false }; + } + const fields = raw.split(/\s+/); + const valid = fields.length === 5 && fields.every((f) => CRON_FIELD.test(f)); + return { + cron: valid ? fields.join(" ") : DEFAULT_BACKUP_SCHEDULE, + valid, + configured: true, + }; +} + +/** + * Whether `dir` exists (or can be created) and this process may write in it. + * Never throws. + * + * @param {string} dir + */ +export function isWritableDirectory(dir) { + try { + mkdirSync(dir, { recursive: true }); + if (!statSync(dir).isDirectory()) return false; + accessSync(dir, fsConstants.W_OK | fsConstants.X_OK); + return true; + } catch { + return false; + } +} + +// ── Keys ─────────────────────────────────────────────────────────────────── + +/** + * Decode a 32-byte key the way the app does (`decodeKey` in + * `src/lib/crypto.ts`): 64 hex characters, or base64 of 32 bytes. Shorter hex + * is padded with its own SHA-256 there, outside production only; it is + * decoded the same way here so a development instance's fingerprints match + * what its admin page shows. Null when the value is not a key at all. + * + * @param {string | undefined} raw + * @returns {Buffer | null} + */ +export function decodeKeyMaterial(raw) { + const value = (raw ?? "").trim(); + if (!value) return null; + if (/^[0-9a-fA-F]{64}$/.test(value)) return Buffer.from(value, "hex"); + if (/^[A-Za-z0-9+/=]+$/.test(value)) { + const buf = Buffer.from(value, "base64"); + if (buf.length === 32) return buf; + } + if (/^[0-9a-fA-F]+$/.test(value) && value.length >= 32) { + const padded = + value + createHash("sha256").update(value, "utf8").digest("hex"); + return Buffer.from(padded.slice(0, 64), "hex"); + } + return null; +} + +/** + * The public fingerprint of a key: the first twelve hex characters of + * SHA-256 over its bytes. The same rule as `fingerprintKeyBytes` in + * `src/lib/crypto.ts` (what the admin's key-backup step shows) and as the key + * id an off-host object records, so one number names one key everywhere. + * + * @param {Buffer} bytes + */ +export function fingerprintBytes(bytes) { + return createHash("sha256").update(bytes).digest("hex").slice(0, 12); +} + +/** + * @typedef {{ + * encryption: { + * activeKeyId: string | null; + * keys: Array<{ id: string; fingerprint: string }>; + * }; + * backupEncryptionKey: string | null; + * backupEncryptionPreviousKeys: string[]; + * apiTokenHmacKey: string | null; + * }} KeyFingerprints + */ + +/** + * Fingerprints of every key a restored database needs, read from an + * environment. Never the key material. A malformed value is left out, which + * the comparison then reports as missing. + * + * `ENCRYPTION_KEY` joins the map as `v1` unless the map names `v1` itself, + * exactly as the app's key loader does. + * + * @param {Env} env + * @returns {KeyFingerprints} + */ +export function keyFingerprints(env = process.env) { + /** @type {Map} */ + const keys = new Map(); + let activeKeyId = null; + const rawMap = (env.ENCRYPTION_KEYS ?? "").trim(); + if (rawMap) { + try { + const parsed = JSON.parse(rawMap); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + for (const [id, value] of Object.entries(parsed)) { + const bytes = + typeof value === "string" ? decodeKeyMaterial(value) : null; + if (bytes) keys.set(id, fingerprintBytes(bytes)); + } + } + } catch { + // Malformed JSON: no keys from the map. The entrypoint refuses to boot + // on it anyway; here it only means nothing matches. + } + activeKeyId = (env.ENCRYPTION_ACTIVE_KEY_ID ?? "").trim() || null; + if (!activeKeyId && keys.size === 1) activeKeyId = [...keys.keys()][0]; + } + const legacy = decodeKeyMaterial(env.ENCRYPTION_KEY); + if (legacy && !keys.has("v1")) keys.set("v1", fingerprintBytes(legacy)); + if (!activeKeyId && legacy) activeKeyId = "v1"; + + const backup = decodeKeyMaterial(env.BACKUP_ENCRYPTION_KEY); + const previous = (env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "") + .split(",") + .map((raw) => decodeKeyMaterial(raw)) + .filter((b) => b !== null) + .map((b) => fingerprintBytes(/** @type {Buffer} */ (b))); + const hmac = (env.API_TOKEN_HMAC_KEY ?? "").trim(); + + return { + encryption: { + activeKeyId, + keys: [...keys.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([id, fingerprint]) => ({ id, fingerprint })), + }, + backupEncryptionKey: backup ? fingerprintBytes(backup) : null, + backupEncryptionPreviousKeys: previous, + // Used as a string by the app (`createHmac("sha256", key)`), so its + // fingerprint is over the string's bytes. + apiTokenHmacKey: hmac ? fingerprintBytes(Buffer.from(hmac, "utf8")) : null, + }; +} + +/** + * @typedef {{ + * key: string; + * state: "missing" | "different"; + * severity: "refuse" | "warn"; + * detail: string; + * }} KeyFinding + */ + +/** + * Compare the keys a dump was written with against the keys an environment + * holds. The active encryption key is the one that refuses: without it the + * app's start-up key check fails and nothing encrypted reads. Every other + * difference is a warning that says what stops working. + * + * @param {KeyFingerprints} expected the manifest's + * @param {KeyFingerprints} actual the environment's + * @returns {{ findings: KeyFinding[]; refuse: boolean }} + */ +export function compareKeyFingerprints(expected, actual) { + /** @type {KeyFinding[]} */ + const findings = []; + const have = new Map( + actual.encryption.keys.map((k) => [k.id, k.fingerprint]), + ); + const active = expected.encryption.activeKeyId; + for (const { id, fingerprint } of expected.encryption.keys) { + const mine = have.get(id); + if (mine === fingerprint) continue; + const isActive = id === active; + findings.push({ + key: `ENCRYPTION_KEYS[${id}]`, + state: mine === undefined ? "missing" : "different", + severity: isActive ? "refuse" : "warn", + detail: isActive + ? `the dump's active encryption key ${id} (fingerprint ${fingerprint}) is ${mine === undefined ? "not configured" : `configured with a different value (fingerprint ${mine})`}; nothing encrypted in this database can be read without it` + : `encryption key ${id} (fingerprint ${fingerprint}) is ${mine === undefined ? "not configured" : `different here (fingerprint ${mine})`}; rows still sealed under it will not open`, + }); + } + if ( + expected.backupEncryptionKey && + expected.backupEncryptionKey !== actual.backupEncryptionKey + ) { + findings.push({ + key: "BACKUP_ENCRYPTION_KEY", + state: actual.backupEncryptionKey ? "different" : "missing", + severity: "warn", + detail: `the off-host backup key (fingerprint ${expected.backupEncryptionKey}) is ${actual.backupEncryptionKey ? `different here (fingerprint ${actual.backupEncryptionKey})` : "not configured"}; off-host copies written before the restore will not open`, + }); + } + if ( + expected.apiTokenHmacKey && + expected.apiTokenHmacKey !== actual.apiTokenHmacKey + ) { + findings.push({ + key: "API_TOKEN_HMAC_KEY", + state: actual.apiTokenHmacKey ? "different" : "missing", + severity: "warn", + detail: `API_TOKEN_HMAC_KEY (fingerprint ${expected.apiTokenHmacKey}) is ${actual.apiTokenHmacKey ? `different here (fingerprint ${actual.apiTokenHmacKey})` : "not configured"}; every session, app token and share link will stop working, no data is lost`, + }); + } + return { + findings, + refuse: findings.some((f) => f.severity === "refuse"), + }; +} + +// ── Connection ───────────────────────────────────────────────────────────── + +/** + * Query parameters libpq understands, by the environment variable that + * carries each to `pg_dump` / `pg_restore`. `DATABASE_URL` also carries + * Prisma's own (`schema`, `connection_limit`, `pool_timeout`), which libpq + * rejects, so the URL is never handed over as it is. + */ +const LIBPQ_PARAMS = { + host: "PGHOST", + sslmode: "PGSSLMODE", + sslcert: "PGSSLCERT", + sslkey: "PGSSLKEY", + sslrootcert: "PGSSLROOTCERT", + connect_timeout: "PGCONNECT_TIMEOUT", + application_name: "PGAPPNAME", +}; + +/** + * @typedef {{ + * pgEnv: Record; + * clientConfig: Record; + * schema: string; + * }} Connection + */ + +/** + * The connection `DATABASE_URL` describes, as libpq environment variables + * (the password travels there, never in a process argument) and as a `pg` + * client config. + * + * @param {string} databaseUrl + * @returns {Connection} + */ +export function connectionFromUrl(databaseUrl) { + const url = new URL(databaseUrl); + const host = decodeURIComponent(url.hostname).replace(/^\[|\]$/g, ""); + /** @type {Record} */ + const pgEnv = { + PGHOST: host, + PGPORT: url.port || "5432", + PGUSER: decodeURIComponent(url.username), + PGPASSWORD: decodeURIComponent(url.password), + PGDATABASE: decodeURIComponent(url.pathname.replace(/^\//, "")), + }; + for (const [param, value] of url.searchParams) { + const name = /** @type {Record} */ (LIBPQ_PARAMS)[param]; + if (name) pgEnv[name] = value; + } + const clean = new URL(databaseUrl); + for (const param of [...clean.searchParams.keys()]) { + if (!(param in LIBPQ_PARAMS)) clean.searchParams.delete(param); + } + return { + pgEnv, + clientConfig: { + connectionString: clean.toString(), + connectionTimeoutMillis: 10_000, + }, + schema: url.searchParams.get("schema") || "public", + }; +} + +/** + * A connected `pg` client for `DATABASE_URL`. + * + * @param {string} databaseUrl + */ +export async function connect(databaseUrl) { + const { default: pg } = await import("pg"); + const client = new pg.Client(connectionFromUrl(databaseUrl).clientConfig); + await client.connect(); + // The server may carry a default statement timeout (the app sets one per + // session through `options`); counting a large table must not hit it. + await client.query("SET statement_timeout = 0"); + return client; +} + +// ── Facts about a database ───────────────────────────────────────────────── + +/** + * @typedef {{ query: (sql: string, params?: unknown[]) => Promise<{ rows: any[] }> }} Queryable + */ + +/** Tables outside the system schemas, `schema.table`, sorted. @param {Queryable} client */ +export async function listUserTables(client) { + const { rows } = await client.query( + `SELECT format('%I.%I', n.nspname, c.relname) AS name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg\\_%' + ORDER BY 1`, + ); + return rows.map((r) => /** @type {string} */ (r.name)); +} + +/** + * Exact row counts of every user table, keyed `schema.table`. + * + * @param {Queryable} client + * @returns {Promise>} + */ +export async function countRows(client) { + /** @type {Record} */ + const counts = {}; + for (const table of await listUserTables(client)) { + const { rows } = await client.query( + `SELECT count(*)::bigint AS n FROM ${table}`, + ); + counts[table] = Number(rows[0].n); + } + return counts; +} + +/** + * The newest finished migration, or null on a database Prisma never touched. + * + * @param {Queryable} client + * @param {string} schema + */ +export async function latestMigration(client, schema) { + const table = await client.query( + "SELECT to_regclass(format('%I._prisma_migrations', $1::text)) AS t", + [schema], + ); + if (!table.rows[0]?.t) return null; + const { rows } = await client.query( + `SELECT migration_name FROM ${table.rows[0].t} + WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL + ORDER BY migration_name DESC LIMIT 1`, + ); + return rows[0]?.migration_name ?? null; +} + +/** @param {Queryable} client */ +export async function serverVersion(client) { + const { rows } = await client.query( + "SELECT current_setting('server_version') AS v, current_setting('server_version_num')::int AS n", + ); + return { version: String(rows[0].v), versionNum: Number(rows[0].n) }; +} + +// ── Running the Postgres client tools ────────────────────────────────────── + +/** + * Run one Postgres client tool. Standard output goes to `stdout` when given + * (a file stream) and is collected otherwise; standard input comes from + * `stdin` when given. Rejects with the tool's own last lines on a non-zero + * exit, and kills it at `deadline` (epoch ms). + * + * @param {string} tool + * @param {string[]} args + * @param {{ + * env?: Record; + * stdout?: NodeJS.WritableStream; + * stdin?: NodeJS.ReadableStream; + * deadline?: number; + * }} [options] + * @returns {Promise} + */ +export function runTool(tool, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(tool, args, { + env: { ...process.env, ...(options.env ?? {}) }, + stdio: [options.stdin ? "pipe" : "ignore", "pipe", "pipe"], + }); + /** @type {Buffer[]} */ + const out = []; + let err = ""; + let timedOut = false; + const timer = + options.deadline !== undefined + ? setTimeout( + () => { + timedOut = true; + child.kill("SIGTERM"); + }, + Math.max(0, options.deadline - Date.now()), + ) + : null; + if (options.stdout) { + // A destination that fails (a full disk) must stop the tool: unpiped, + // it would block on a full pipe until its deadline, or forever. + options.stdout.on("error", (e) => { + child.kill("SIGTERM"); + reject(e); + }); + child.stdout.pipe(options.stdout); + } else { + child.stdout.on("data", (c) => out.push(c)); + } + child.stderr.on("data", (c) => { + err = (err + c.toString("utf8")).slice(-4000); + }); + if (options.stdin && child.stdin) { + options.stdin.on("error", (e) => { + child.kill("SIGTERM"); + reject(e); + }); + child.stdin.on("error", () => { + // The tool closed its input early; its exit code says why. + }); + options.stdin.pipe(child.stdin); + } + child.on("error", (e) => { + if (timer) clearTimeout(timer); + reject( + /** @type {NodeJS.ErrnoException} */ (e).code === "ENOENT" + ? new Error( + `${tool} is not installed in this environment (the HealthLog image ships it; on another host install the PostgreSQL client tools)`, + ) + : e, + ); + }); + child.on("close", (code, signal) => { + if (timer) clearTimeout(timer); + if (timedOut) { + reject(new Error(`${tool} ran past its time limit and was stopped`)); + } else if (code === 0) { + resolve(Buffer.concat(out).toString("utf8")); + } else { + const tail = err.trim().split("\n").slice(-3).join(" | "); + reject( + new Error( + `${tool} exited ${code ?? signal}${tail ? `: ${tail}` : ""}`, + ), + ); + } + }); + }); +} + +/** `pg_dump (PostgreSQL) 18.6`, or null when it cannot be asked. */ +export async function pgDumpVersion() { + try { + return (await runTool("pg_dump", ["--version"])).trim(); + } catch { + return null; + } +} + +/** + * Read an archive's table of contents back. A file `pg_restore` cannot list + * is not a dump, whatever its name. Answers the number of entries. + * + * @param {string} path + */ +export async function verifyArchive(path) { + const listing = await runTool("pg_restore", ["--list"], { + stdin: createReadStream(path), + }); + const entries = listing + .split("\n") + .filter((line) => line.trim() && !line.startsWith(";")).length; + if (entries === 0) throw new Error("the archive lists no entries"); + return entries; +} + +/** SHA-256 and size of a file, streamed. @param {string} path */ +export async function hashFile(path) { + const hash = createHash("sha256"); + let bytes = 0; + for await (const chunk of createReadStream(path)) { + hash.update(chunk); + bytes += chunk.length; + } + return { sha256: hash.digest("hex"), bytes }; +} + +// ── Writing a dump ───────────────────────────────────────────────────────── + +/** + * @typedef {{ + * format: string; + * formatVersion: number; + * kind: DumpKind; + * file: string; + * createdAt: string; + * appVersion: string | null; + * buildSha: string | null; + * latestMigration: string | null; + * postgres: { serverVersion: string; serverVersionNum: number; pgDump: string | null }; + * dump: { bytes: number; sha256: string; entries: number }; + * rowCounts: Record; + * keys: KeyFingerprints; + * }} DumpManifest + */ + +/** + * Build the manifest of one dump. Pure, so the shape is tested on its own. + * + * @param {{ + * kind: DumpKind; + * file: string; + * createdAt: Date; + * appVersion?: string | null; + * buildSha?: string | null; + * latestMigration: string | null; + * server: { version: string; versionNum: number }; + * pgDump: string | null; + * archive: { bytes: number; sha256: string; entries: number }; + * rowCounts: Record; + * env: Env; + * }} input + * @returns {DumpManifest} + */ +export function buildManifest(input) { + return { + format: MANIFEST_FORMAT, + formatVersion: MANIFEST_FORMAT_VERSION, + kind: input.kind, + file: input.file, + createdAt: input.createdAt.toISOString(), + appVersion: input.appVersion || null, + buildSha: input.buildSha || null, + latestMigration: input.latestMigration, + postgres: { + serverVersion: input.server.version, + serverVersionNum: input.server.versionNum, + pgDump: input.pgDump, + }, + dump: { ...input.archive }, + rowCounts: Object.fromEntries( + Object.entries(input.rowCounts).sort(([a], [b]) => a.localeCompare(b)), + ), + keys: keyFingerprints(input.env), + }; +} + +/** + * Check a parsed manifest has the fields a restore reads. + * + * @param {unknown} value + * @returns {DumpManifest} + */ +export function parseManifest(value) { + const m = /** @type {Record} */ (value); + if ( + !m || + m.format !== MANIFEST_FORMAT || + m.formatVersion !== MANIFEST_FORMAT_VERSION || + typeof m.file !== "string" || + typeof m.dump?.sha256 !== "string" || + typeof m.dump?.bytes !== "number" || + typeof m.rowCounts !== "object" || + !m.keys?.encryption + ) { + throw new Error("not a HealthLog database dump manifest"); + } + return /** @type {DumpManifest} */ (m); +} + +/** + * Take one dump into `dir`, with its manifest, and answer both paths. + * + * The archive and the row counts read one exported snapshot. The archive is + * written as `.partial`, hashed and listed back; the manifest is written as + * `.partial` and renamed; the archive is renamed last. On any failure every + * file this call created is removed and the error is rethrown. + * + * @param {{ + * databaseUrl: string; + * dir: string; + * kind: DumpKind; + * now?: Date; + * env?: Env; + * appVersion?: string | null; + * buildSha?: string | null; + * deadline?: number; + * }} options + * @returns {Promise<{ base: string; dumpPath: string; manifestPath: string; manifest: DumpManifest }>} + */ +export async function createDatabaseDump(options) { + const now = options.now ?? new Date(); + const env = options.env ?? process.env; + const base = dumpBaseName(options.kind, now); + const dumpPath = join(options.dir, dumpFileName(base)); + const manifestPath = join(options.dir, manifestFileName(base)); + const partialDump = `${dumpPath}.partial`; + const partialManifest = `${manifestPath}.partial`; + const connection = connectionFromUrl(options.databaseUrl); + // Two runs in the same second would share a name. The second one must not + // replace, or on failure remove, what the first one wrote. + for (const taken of [dumpPath, manifestPath]) { + if (await statOrNull(taken)) { + throw new Error(`a dump named ${basename(taken)} already exists`); + } + } + + const client = await connect(options.databaseUrl); + let finished = false; + /** The files this call created, and only those, go on a failure. */ + const created = []; + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY"); + const { rows } = await client.query( + "SELECT pg_export_snapshot() AS snapshot", + ); + const snapshot = String(rows[0].snapshot); + + const out = createWriteStream(partialDump, { flags: "wx", mode: 0o600 }); + const opened = new Promise((resolve, reject) => { + out.once("open", resolve); + out.once("error", reject); + }); + await opened; + created.push(partialDump); + const closed = new Promise((resolve, reject) => { + out.once("close", resolve); + out.once("error", reject); + }); + // Awaited below on success; on a failure the run is already rejecting + // and this must not surface a second time as unhandled. + closed.catch(() => {}); + // `pg_dump` reads the snapshot in its own session while this one counts + // the rows of the same snapshot; the queries on this client run one + // after another (a `pg` client does not overlap them). + const facts = async () => ({ + counts: await countRows(client), + server: await serverVersion(client), + migration: await latestMigration(client, connection.schema), + }); + const [, { counts, server, migration }, pgDump] = await Promise.all([ + runTool( + "pg_dump", + [ + "--format=custom", + "--no-password", + `--snapshot=${snapshot}`, + "--lock-wait-timeout=120000", + ], + { env: connection.pgEnv, stdout: out, deadline: options.deadline }, + ), + facts(), + pgDumpVersion(), + ]); + await closed; + await client.query("COMMIT"); + + const { sha256, bytes } = await hashFile(partialDump); + const entries = await verifyArchive(partialDump); + const manifest = buildManifest({ + kind: options.kind, + file: dumpFileName(base), + createdAt: now, + appVersion: options.appVersion, + buildSha: options.buildSha, + latestMigration: migration, + server, + pgDump, + archive: { bytes, sha256, entries }, + rowCounts: counts, + env, + }); + await writeFile(partialManifest, `${JSON.stringify(manifest, null, 2)}\n`, { + flag: "wx", + mode: 0o600, + }); + created.push(partialManifest); + await rename(partialManifest, manifestPath); + created.push(manifestPath); + await rename(partialDump, dumpPath); + finished = true; + return { base, dumpPath, manifestPath, manifest }; + } finally { + await client.end().catch(() => {}); + if (!finished) { + // Only what this call created: a `wx` open that failed created nothing. + await Promise.all(created.map((p) => rm(p, { force: true }))); + } + } +} + +// ── What is on disk, and what may go ─────────────────────────────────────── + +/** + * @typedef {{ + * base: string; + * kind: DumpKind; + * timestamp: Date; + * dump: { path: string; bytes: number } | null; + * manifest: string | null; + * encryptedDump: string | null; + * encryptedManifest: string | null; + * leftovers: Array<{ path: string; mtimeMs: number }>; + * }} DumpSet + */ + +/** + * Every dump this module wrote into `dir`, newest first, grouped by stem. + * Files with any other name are not looked at. + * + * @param {string} dir + * @returns {DumpSet[]} + */ +export function listDumpSets(dir) { + /** @type {Map} */ + const sets = new Map(); + let names = []; + try { + names = readdirSync(dir); + } catch { + return []; + } + for (const name of names) { + const parsed = parseDumpFileName(name); + if (!parsed) continue; + const path = join(dir, name); + let info; + try { + info = statSync(path); + } catch { + continue; + } + if (!info.isFile()) continue; + const set = sets.get(parsed.base) ?? { + base: parsed.base, + kind: parsed.kind, + timestamp: parsed.timestamp, + dump: null, + manifest: null, + encryptedDump: null, + encryptedManifest: null, + leftovers: [], + }; + if (parsed.variant === "partial") { + set.leftovers.push({ path, mtimeMs: info.mtimeMs }); + } else if (parsed.variant === "encrypted") { + if (parsed.role === "dump") set.encryptedDump = path; + else set.encryptedManifest = path; + } else if (parsed.role === "dump") { + set.dump = { path, bytes: info.size }; + } else { + set.manifest = path; + } + sets.set(parsed.base, set); + } + return [...sets.values()].sort( + (a, b) => b.timestamp.getTime() - a.timestamp.getTime(), + ); +} + +/** A set is good when its archive and its manifest both carry final names. @param {DumpSet} set */ +export function isGoodSet(set) { + return set.dump !== null && set.manifest !== null; +} + +/** + * Which files retention removes. Pure. + * + * Per kind, the newest `keep` good dumps stay (at least one, so the newest + * good dump is never removed) and every older good dump goes, archive and + * manifest together. A `.partial` or a manifest without its archive goes once + * it is older than twelve hours, so a run still in progress keeps its files. + * A final-named archive without a manifest is not something this module + * leaves behind, so it is not touched. Encrypted copies (`.enc`) an operator + * placed here to restore from are never touched either. + * + * @param {DumpSet[]} sets + * @param {{ keep: Record; now: number }} options + * @returns {string[]} + */ +export function selectForDeletion(sets, options) { + /** @type {string[]} */ + const remove = []; + for (const kind of DUMP_KINDS) { + const keep = Math.max(1, options.keep[kind] ?? 1); + const good = sets + .filter((s) => s.kind === kind && isGoodSet(s)) + .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()); + for (const set of good.slice(keep)) { + remove.push( + /** @type {{ path: string }} */ (set.dump).path, + /** @type {string} */ (set.manifest), + ); + } + } + for (const set of sets) { + for (const leftover of set.leftovers) { + if (options.now - leftover.mtimeMs > STALE_LEFTOVER_MS) { + remove.push(leftover.path); + } + } + if (set.manifest && !set.dump && set.leftovers.length === 0) { + try { + if (options.now - statSync(set.manifest).mtimeMs > STALE_LEFTOVER_MS) { + remove.push(set.manifest); + } + } catch { + // Gone already. + } + } + } + return remove; +} + +/** + * Apply retention to `dir`. Answers the paths removed. + * + * @param {string} dir + * @param {{ dailyKeep?: number; now?: number }} [options] + */ +export async function applyRetention(dir, options = {}) { + const doomed = selectForDeletion(listDumpSets(dir), { + keep: { + daily: options.dailyKeep ?? DEFAULT_DAILY_KEEP, + "pre-migration": PRE_MIGRATION_KEEP, + }, + now: options.now ?? Date.now(), + }); + for (const path of doomed) await rm(path, { force: true }); + return doomed; +} + +/** @param {string} path */ +export async function readManifestFile(path) { + return parseManifest(JSON.parse(await readFile(path, "utf8"))); +} + +// ── Off-host copies ──────────────────────────────────────────────────────── + +/** + * The bucket key of an off-host database object. It is also the associated + * data its encryption is bound to, so a copy cannot be passed off under + * another name. + * + * @param {string} fileName e.g. `healthlog-…-daily.dump.enc` + */ +export function offhostObjectKey(fileName) { + return `${OFFHOST_DATABASE_PREFIX}${fileName}`; +} + +const ENVELOPE_MAGIC = "HLBK"; +const ENVELOPE_KEYED = 0x04; +const AAD_PREFIX = "healthlog/offhost-backup/v4|"; +const IV_LENGTH = 12; +const TAG_LENGTH = 16; + +/** + * Open an off-host object written in the version-4 envelope + * (`src/lib/jobs/offhost-backup.ts`: `HLBK`, 0x04, key id, IV, gzip + * ciphertext, tag, the object key as associated data) into `outPath`, + * streaming, without holding the file. + * + * GCM's tag sits at the end, so the plaintext is written before the tag is + * checked; it goes to `outPath` and is removed again unless the tag verifies. + * Nothing reads `outPath` before this resolves. + * + * @param {{ inPath: string; outPath: string; objectKey: string; keys: Buffer[] }} options + */ +export async function decryptOffhostFile(options) { + const size = statSync(options.inPath).size; + const fd = openSync(options.inPath, "r"); + let header; + let tag; + try { + const head = Buffer.alloc(Math.min(size, 6 + 255 + IV_LENGTH)); + readSync(fd, head, 0, head.length, 0); + if ( + head.subarray(0, 4).toString("latin1") !== ENVELOPE_MAGIC || + head[4] !== ENVELOPE_KEYED + ) { + throw new Error( + "not an off-host database copy (unknown envelope); expected a .dump.enc written by HealthLog", + ); + } + const idLen = head[5]; + const keyId = head.subarray(6, 6 + idLen).toString("latin1"); + const iv = head.subarray(6 + idLen, 6 + idLen + IV_LENGTH); + header = { keyId, iv, length: 6 + idLen + IV_LENGTH }; + tag = Buffer.alloc(TAG_LENGTH); + readSync(fd, tag, 0, TAG_LENGTH, size - TAG_LENGTH); + } finally { + closeSync(fd); + } + if (size < header.length + TAG_LENGTH) { + throw new Error("the off-host copy is truncated"); + } + const key = options.keys.find((k) => fingerprintBytes(k) === header.keyId); + if (!key) { + throw new Error( + `the off-host copy was written under backup key ${header.keyId}, which is neither BACKUP_ENCRYPTION_KEY nor in BACKUP_ENCRYPTION_PREVIOUS_KEYS`, + ); + } + const decipher = createDecipheriv("aes-256-gcm", key, header.iv); + decipher.setAAD(Buffer.from(`${AAD_PREFIX}${options.objectKey}`, "utf8")); + decipher.setAuthTag(tag); + try { + await pipeline( + createReadStream(options.inPath, { + start: header.length, + end: size - TAG_LENGTH - 1, + }), + decipher, + createGunzip(), + createWriteStream(options.outPath, { flags: "wx", mode: 0o600 }), + ); + } catch (err) { + await rm(options.outPath, { force: true }); + const reason = err instanceof Error ? err.message : String(err); + throw new Error( + /auth|unable to authenticate/i.test(reason) + ? "the off-host copy does not open with this key under this name (altered, renamed, or written with another key)" + : `could not open the off-host copy: ${reason}`, + ); + } +} + +/** + * The off-host keys an environment holds, current first. + * + * @param {Env} env + * @returns {Buffer[]} + */ +export function offhostKeysFromEnv(env = process.env) { + return [ + env.BACKUP_ENCRYPTION_KEY, + ...(env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "").split(","), + ] + .map((raw) => decodeKeyMaterial(raw)) + .filter((k) => k !== null) + .map((k) => /** @type {Buffer} */ (k)); +} + +// ── Restore ──────────────────────────────────────────────────────────────── + +/** + * Compare the counts a restore produced with the manifest's. Every table the + * manifest names must be there with its count, and no table may be there that + * the manifest does not name. + * + * @param {Record} expected + * @param {Record} actual + * @returns {string[]} one line per difference + */ +export function compareRowCounts(expected, actual) { + /** @type {string[]} */ + const problems = []; + for (const [table, count] of Object.entries(expected)) { + if (!(table in actual)) { + problems.push(`${table}: in the dump with ${count} rows, missing now`); + } else if (actual[table] !== count) { + problems.push( + `${table}: ${count} rows in the dump, ${actual[table]} restored`, + ); + } + } + for (const table of Object.keys(actual)) { + if (!(table in expected)) { + problems.push(`${table}: restored but not in the dump's manifest`); + } + } + return problems; +} + +/** + * Drop every non-system schema and recreate an empty `public`, so a restore + * that failed its check leaves the database as empty as it found it. + * + * @param {Queryable} client + */ +export async function emptyDatabase(client) { + const { rows } = await client.query( + `SELECT quote_ident(nspname) AS s FROM pg_namespace + WHERE nspname NOT LIKE 'pg\\_%' AND nspname <> 'information_schema'`, + ); + for (const { s } of rows) await client.query(`DROP SCHEMA ${s} CASCADE`); + await client.query("CREATE SCHEMA IF NOT EXISTS public"); +} + +/** Stat helper that answers null for a missing file. @param {string} path */ +export async function statOrNull(path) { + try { + return await stat(path); + } catch { + return null; + } +} diff --git a/src/lib/database-backup/health.ts b/src/lib/database-backup/health.ts new file mode 100644 index 000000000..2039df249 --- /dev/null +++ b/src/lib/database-backup/health.ts @@ -0,0 +1,117 @@ +/** + * Whether the nightly whole-database dump is happening, as facts a page or an + * alert can act on. + * + * Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of + * this process: whether `DB_BACKUP_DIR` is writable here, and whether the + * off-host bucket is configured. A ledger row that says a copy went off-host + * outranks this process's environment, for the reason the off-host card gives + * (a split install can hand the bucket variables to the worker only). + */ +import { prisma } from "@/lib/db"; +import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue"; +import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; + +import { + databaseBackupEnabled, + isWritableDirectory, + resolveBackupDir, +} from "./dump-core.mjs"; + +export interface DatabaseBackupHealth { + /** `DB_BACKUP_ENABLED` is not off. */ + enabled: boolean; + /** The directory dumps are kept in (`DB_BACKUP_DIR`). */ + directory: string; + /** This process can write there. False means dumps only go off-host. */ + localDirWritable: boolean; + /** The off-host bucket is configured, or a dump has been uploaded to it. */ + offhostConfigured: boolean; + /** The newest dump that was kept on the server or reached the bucket. */ + lastGood: { + at: string; + fileName: string; + sizeBytes: number | null; + local: boolean; + offhost: boolean; + } | null; + /** The newest run that failed, or that a dying process left unfinished. */ + lastFailure: { at: string; error: string } | null; +} + +/** + * A run still `running` this long after it started was cut off by its + * process: the job's expiry, past which pg-boss has failed it anyway. + */ +const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000; + +export async function getDatabaseBackupHealth( + now: Date = new Date(), + env: NodeJS.ProcessEnv = process.env, +): Promise { + const directory = resolveBackupDir(env); + const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all( + [ + prisma.databaseBackupRun.findFirst({ + where: { + fileName: { not: null }, + OR: [{ localResult: "written" }, { offhostResult: "uploaded" }], + }, + orderBy: { startedAt: "desc" }, + select: { + startedAt: true, + finishedAt: true, + fileName: true, + dumpBytes: true, + localResult: true, + offhostResult: true, + }, + }), + prisma.databaseBackupRun.findFirst({ + where: { status: "failed" }, + orderBy: { startedAt: "desc" }, + select: { startedAt: true, finishedAt: true, error: true }, + }), + prisma.databaseBackupRun.findFirst({ + where: { + status: "running", + startedAt: { lt: new Date(now.getTime() - ABANDONED_AFTER_MS) }, + }, + orderBy: { startedAt: "desc" }, + select: { startedAt: true }, + }), + prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }), + ], + ); + + const failures = [ + lastFailed && { + at: (lastFailed.finishedAt ?? lastFailed.startedAt).toISOString(), + error: lastFailed.error ?? "failed without a reason", + }, + lastAbandoned && { + at: lastAbandoned.startedAt.toISOString(), + error: "the run did not finish (the process stopped under it)", + }, + ] + .filter((f): f is { at: string; error: string } => Boolean(f)) + .sort((a, b) => b.at.localeCompare(a.at)); + + return { + enabled: databaseBackupEnabled(env), + directory, + localDirWritable: isWritableDirectory(directory), + offhostConfigured: offhostBackupConfigured() || everUploaded > 0, + lastGood: lastGood + ? { + at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(), + fileName: lastGood.fileName ?? "", + sizeBytes: + lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes), + local: lastGood.localResult === "written", + offhost: lastGood.offhostResult === "uploaded", + } + : null, + lastFailure: failures[0] ?? null, + }; +} diff --git a/src/lib/database-backup/run.ts b/src/lib/database-backup/run.ts new file mode 100644 index 000000000..4f3e3d99e --- /dev/null +++ b/src/lib/database-backup/run.ts @@ -0,0 +1,243 @@ +/** + * One run of the nightly whole-database dump. + * + * The dump is the artifact that rebuilds a lost server: every account, every + * login, every grant and managed profile, the job queue and the instance + * settings, in one `pg_dump --format=custom` file with a manifest beside it + * (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the + * bundled compose file) and, when the off-host bucket is configured, uploaded + * under `database/` in the same envelope the per-account copies use, sealed + * with `BACKUP_ENCRYPTION_KEY`. + * + * Without a writable directory (an install that runs the image without the + * volume) the dump is written to a temporary directory, uploaded and removed: + * the off-host copy still happens, nothing is kept on the server, and the + * ledger and the boot summary say so. With neither, the run is skipped and + * recorded as such. + * + * Every run leaves one `DatabaseBackupRun` row: started, then finished with + * what happened on each leg. `getDatabaseBackupHealth` reads them. + */ +import { createReadStream } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; + +import type { PrismaClient } from "@/generated/prisma/client"; +import { + getS3Client, + loadOffhostConfig, + uploadEncryptedBackup, + type S3Like, +} from "@/lib/jobs/offhost-backup"; +import { annotate } from "@/lib/logging/context"; +import packageJson from "../../../package.json"; + +import { + applyRetention, + createDatabaseDump, + databaseBackupEnabled, + isWritableDirectory, + offhostObjectKey, + resolveBackupDir, + resolveDailyKeep, +} from "./dump-core.mjs"; + +/** How long ledger rows are kept. A run a day; a year and a bit of history. */ +const LEDGER_RETENTION_DAYS = 400; + +export type DatabaseBackupLocalResult = + "written" | "no_directory" | "failed" | "skipped"; +export type DatabaseBackupOffhostResult = + "uploaded" | "not_configured" | "failed" | "skipped"; + +export interface DatabaseBackupReport { + status: "succeeded" | "failed" | "skipped"; + fileName: string | null; + dumpBytes: number | null; + local: DatabaseBackupLocalResult; + offhost: DatabaseBackupOffhostResult; + /** Dumps and manifests retention removed. */ + removed: number; + error: string | null; +} + +export interface RunDatabaseBackupOptions { + now?: Date; + /** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */ + deadline?: number; + env?: NodeJS.ProcessEnv; + /** A bucket double for tests; the configured bucket otherwise. */ + s3?: S3Like; +} + +/** The running build's version, as `/api/version` answers it. */ +function appVersion(env: NodeJS.ProcessEnv): string { + return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version; +} + +/** An error as one line for the ledger and the admin page. */ +function reason(err: unknown): string { + const text = err instanceof Error ? err.message : String(err); + return text.replace(/\s+/g, " ").trim().slice(0, 500); +} + +/** Upload one local file as an off-host object, streamed. */ +async function uploadFile( + s3: S3Like, + key: Buffer, + path: string, +): Promise { + return uploadEncryptedBackup( + s3, + offhostObjectKey(`${basename(path)}.enc`), + key, + async (write) => { + for await (const chunk of createReadStream(path)) { + await write(chunk as Buffer); + } + }, + ); +} + +/** + * Take tonight's dump, keep it, copy it off-host, prune the old ones, and + * record what happened. Never throws: a failure is the report's `status`. + */ +export async function runDatabaseBackup( + prisma: PrismaClient, + options: RunDatabaseBackupOptions = {}, +): Promise { + const env = options.env ?? process.env; + const now = options.now ?? new Date(); + const dir = resolveBackupDir(env); + const localWritable = isWritableDirectory(dir); + const offhostConfig = (() => { + try { + return loadOffhostConfig(); + } catch { + // A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the + // same value with their own message; here it reads as an upload error. + return "invalid" as const; + } + })(); + + // Each leg reads "failed" until it has succeeded, so a run the process died + // under says so once its expiry has passed. + const report: DatabaseBackupReport = { + status: "failed", + fileName: null, + dumpBytes: null, + local: localWritable ? "failed" : "no_directory", + offhost: offhostConfig ? "failed" : "not_configured", + removed: 0, + error: null, + }; + const run = await prisma.databaseBackupRun.create({ + data: { + startedAt: now, + status: "running", + localResult: report.local, + offhostResult: report.offhost, + }, + select: { id: true }, + }); + + let scratch: string | null = null; + try { + if (!databaseBackupEnabled(env)) { + report.status = "skipped"; + report.local = "skipped"; + report.offhost = "skipped"; + report.error = "DB_BACKUP_ENABLED is false"; + } else if (!localWritable && !offhostConfig) { + report.status = "skipped"; + report.error = `no writable backup directory (${dir}) and no off-host bucket configured`; + } else { + const target = localWritable + ? dir + : (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-"))); + const dump = await createDatabaseDump({ + databaseUrl: env.DATABASE_URL ?? "", + dir: target, + kind: "daily", + now, + env, + appVersion: appVersion(env), + buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null, + deadline: options.deadline, + }); + if (localWritable) report.local = "written"; + report.fileName = basename(dump.dumpPath); + report.dumpBytes = dump.manifest.dump.bytes; + + let uploadError: string | null = null; + if (offhostConfig === "invalid") { + uploadError = + "BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64"; + } else if (offhostConfig) { + try { + const s3 = options.s3 ?? (await getS3Client(offhostConfig)); + // The manifest goes up first: a dump object in the bucket always + // has the manifest a restore checks it against. + await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath); + await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath); + report.offhost = "uploaded"; + } catch (err) { + uploadError = `off-host upload failed: ${reason(err)}`; + } + } + + if (localWritable) { + const removed = await applyRetention(dir, { + dailyKeep: resolveDailyKeep(env), + now: now.getTime(), + }); + report.removed = removed.length; + } + + // Good when the dump exists somewhere it survives: on the volume, or in + // the bucket. An upload that failed beside a kept dump still fails the + // run, because the operator asked for both. + report.status = uploadError ? "failed" : "succeeded"; + report.error = uploadError; + } + } catch (err) { + report.status = "failed"; + report.error = reason(err); + } finally { + if (scratch) await rm(scratch, { recursive: true, force: true }); + } + + await prisma.databaseBackupRun.update({ + where: { id: run.id }, + data: { + finishedAt: new Date(), + status: report.status, + fileName: report.fileName, + dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes), + localResult: report.local, + offhostResult: report.offhost, + error: report.error, + }, + }); + await prisma.databaseBackupRun.deleteMany({ + where: { + startedAt: { + lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000), + }, + }, + }); + + annotate({ + action: { name: "database.backup.run" }, + meta: { + status: report.status, + local: report.local, + offhost: report.offhost, + dump_bytes: report.dumpBytes ?? 0, + removed: report.removed, + }, + }); + return report; +} diff --git a/src/lib/jobs/database-backup-queue.ts b/src/lib/jobs/database-backup-queue.ts new file mode 100644 index 000000000..5285095eb --- /dev/null +++ b/src/lib/jobs/database-backup-queue.ts @@ -0,0 +1,40 @@ +/** + * The queue facts of the nightly whole-database dump: its name, its schedule + * and its send options. Apart from the job so the worker registry can read + * them without loading it (`job-registry-graph.test.ts`). + */ +import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy"; +import { resolveBackupSchedule } from "@/lib/database-backup/dump-core.mjs"; + +export const DATABASE_BACKUP_QUEUE = "database-backup"; + +/** + * `DB_BACKUP_SCHEDULE`, read once when the worker loads its registry, in the + * zone every cron runs in (`createAndSchedule`, Europe/Berlin). Default 01:30, + * an hour before the off-host account copies so the two uploads do not meet. + * A value that is not a five-field cron keeps the default, and the boot + * readiness summary says so. + */ +export const DATABASE_BACKUP_CRON = resolveBackupSchedule(process.env).cron; + +/** + * Four hours. `pg_dump` of a large instance takes minutes, the upload of the + * dump about as long as the dump; the window is there to be impossible to hit + * for a working run. The handler kills `pg_dump` at three quarters of it + * (`jobDeadline`), so a hung dump ends as a failed run with a reason rather + * than an expired job. + */ +export const DATABASE_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60; + +/** + * What the nightly schedule sends with: the expiry, the backups' heartbeat, + * and one retry. A dump that failed for a passing reason (a lock it waited + * out, a bucket that answered 503) gets a second attempt; one that fails + * again waits for the next night, and the failure stays on the admin page. + */ +export const DATABASE_BACKUP_SEND_OPTIONS = { + expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS, + heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS, + retryLimit: 1, + retryDelay: 15 * 60, +} as const; diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 250838b7d..ab79d94c1 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -78,6 +78,12 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "continued", "conversation_titles_migrated", "created", + // The nightly whole-database dump: where it went and how big. + "database_backup_bytes", + "database_backup_local", + "database_backup_offhost", + "database_backup_removed", + "database_backup_status", "daily_rows_retired", "days_consolidated", "days_rebuilt", diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index cf6fbf1ba..f34ded204 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -494,9 +494,9 @@ function createEnvelopeStream( return { stream, bytes: () => written }; } -/** Produces the backup JSON in pieces. Every piece is written in order. */ +/** Produces the plaintext (JSON text, or a dump's bytes) in ordered pieces. */ export type BackupJsonProducer = ( - write: (chunk: string) => Promise, + write: (chunk: string | Buffer) => Promise, ) => Promise; export interface UploadBackupOptions { @@ -572,9 +572,9 @@ export async function uploadEncryptedBackup( }, ); - const write = async (chunk: string): Promise => { + const write = async (chunk: string | Buffer): Promise => { if (failure) throw failure; - if (gzip.write(chunk, "utf8")) return; + if (gzip.write(chunk, "utf8")) return; // a Buffer ignores the encoding await new Promise((resolve, reject) => { const onDrain = (): void => { gzip.off("error", onError); diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 28ed26810..9a2503c1b 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -42,6 +42,7 @@ import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key- import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill"; import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run"; import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy"; +import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue"; import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue"; import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore"; import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue"; @@ -332,6 +333,17 @@ export const QUEUE_RUNTIME: Readonly> = { exclusive: "lockedPass", why: "Serialises every account's whole record.", }, + "database-backup": { + runtime: "long", + expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS, + expiryVia: ["DATABASE_BACKUP_SEND_OPTIONS"], + stop: { + file: "lib/jobs/reminder/backup-handlers.ts", + fn: "handleDatabaseBackup", + }, + exclusive: "lockedPass", + why: "Runs pg_dump over the whole database and uploads the dump off-host.", + }, "data-backup-offhost": { runtime: "long", expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS, diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts index 70429c156..6c84a3eaf 100644 --- a/src/lib/jobs/registry/maintenance.ts +++ b/src/lib/jobs/registry/maintenance.ts @@ -1,5 +1,6 @@ /** - * The housekeeping queues: backups (weekly in-database, nightly off-host), + * The housekeeping queues: backups (the nightly whole-database dump, weekly + * in-database account copies, nightly off-host), * the restore drill and restores, every ledger and retention cleanup, the * medication maintenance passes, the imports (Apple Health, Health Connect, * medication intake), the converging encryption backfills, the document vault @@ -47,6 +48,11 @@ import { DATA_BACKUP_QUEUE, DATA_BACKUP_SEND_OPTIONS, } from "@/lib/jobs/data-backup-policy"; +import { + DATABASE_BACKUP_CRON, + DATABASE_BACKUP_QUEUE, + DATABASE_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/database-backup-queue"; import { DAWARICH_SYNC_CRON, DAWARICH_SYNC_QUEUE, @@ -300,6 +306,16 @@ const jobs: readonly JobDefinition[] = [ lane: true, load: async () => (await backups()).handleDataBackup, }), + // The whole-database dump: the copy that rebuilds a lost server. Locked, so + // a retry that starts while a run is still dumping waits its turn. + defineJob({ + queue: DATABASE_BACKUP_QUEUE, + cron: DATABASE_BACKUP_CRON, + scheduleOptions: DATABASE_BACKUP_SEND_OPTIONS, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + load: async () => (await backups()).handleDatabaseBackup, + }), prune( RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index c8bc8fa83..845f03927 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -5,7 +5,7 @@ * schedules, and boss.work registrations. */ import { type Job, type JobWithMetadata } from "pg-boss"; -import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobBudget, jobDeadline } from "@/lib/jobs/job-budget"; import { recordError } from "@/lib/jobs/worker-status"; import { BackupBlobTooLargeError, @@ -31,6 +31,37 @@ import { } from "@/lib/jobs/backup-pass-attempts"; import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; +/** + * The nightly whole-database dump (`database-backup`). + * + * One `pg_dump` in a child process, so it is not in the heavy lane: the heap + * holds a part of the upload at a time, not the database. `pg_dump` is stopped + * at three quarters of the job's expiry (`jobDeadline`), so a dump that hangs + * on a lock ends as a failed run with its reason rather than an expired job. + * A skipped run (switched off, nowhere to put it) completes; the admin page + * and the boot summary carry why. + */ +export async function handleDatabaseBackup( + jobs: Job>[], +): Promise { + return withBackgroundEvent("job.database_backup", async () => { + const { runDatabaseBackup } = await import("@/lib/database-backup/run"); + const report = await runDatabaseBackup(getWorkerPrisma(), { + deadline: jobs[0] ? jobDeadline(jobs[0]) : undefined, + }); + const did = { + database_backup_status: report.status, + database_backup_local: report.local, + database_backup_offhost: report.offhost, + database_backup_bytes: report.dumpBytes ?? 0, + database_backup_removed: report.removed, + }; + return report.status === "failed" + ? jobFailed("database dump failed", report.error ?? undefined, did) + : jobDone(did); + }); +} + export interface DataBackupPayload { triggeredAt: string; } diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index d90560c84..c95d658a6 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -99,6 +99,7 @@ const subsystemSurface = { "audit-log-cleanup": { audience: "system" }, "step-up-elevation-cleanup": { audience: "system" }, "data-backup-offhost": { audience: "system" }, + "database-backup": { audience: "system" }, "data-restore-drill": { audience: "system" }, "offhost-backup-purge": { audience: "system" }, "backup-restore": { audience: "system" }, diff --git a/tests/integration/__golden__/queue-registration.json b/tests/integration/__golden__/queue-registration.json index d85705a21..31a41afe6 100644 --- a/tests/integration/__golden__/queue-registration.json +++ b/tests/integration/__golden__/queue-registration.json @@ -1,5 +1,5 @@ { - "registeredQueueCount": 122, + "registeredQueueCount": 123, "registeredQueues": [ "achievement-unlock-sweep", "apple-health-import", @@ -23,6 +23,7 @@ "data-backup", "data-backup-offhost", "data-restore-drill", + "database-backup", "dawarich-sync", "dense-intraday-hourly-rebuild", "dense-intraday-retention", @@ -455,6 +456,21 @@ "retry_delay_max": null, "retry_limit": 2 }, + { + "dead_letter": null, + "deletion_seconds": 604800, + "expire_seconds": 900, + "heartbeat_seconds": null, + "name": "database-backup", + "notify": false, + "partition": false, + "policy": "standard", + "retention_seconds": 1209600, + "retry_backoff": false, + "retry_delay": 0, + "retry_delay_max": null, + "retry_limit": 2 + }, { "dead_letter": null, "deletion_seconds": 604800, @@ -2137,6 +2153,21 @@ }, "timezone": "Europe/Berlin" }, + { + "cron": "30 1 * * *", + "data": {}, + "key": "", + "kind": "cron", + "name": "database-backup", + "options": { + "expireInSeconds": 14400, + "heartbeatSeconds": 300, + "retryDelay": 900, + "retryLimit": 1, + "tz": "Europe/Berlin" + }, + "timezone": "Europe/Berlin" + }, { "cron": "20 3 * * *", "data": {}, @@ -3022,6 +3053,13 @@ "pollingIntervalSeconds": 30 } }, + { + "queue": "database-backup", + "options": { + "localConcurrency": 1, + "pollingIntervalSeconds": 30 + } + }, { "queue": "dawarich-sync", "options": { diff --git a/tests/integration/database-backup.test.ts b/tests/integration/database-backup.test.ts new file mode 100644 index 000000000..58e778188 --- /dev/null +++ b/tests/integration/database-backup.test.ts @@ -0,0 +1,511 @@ +/** + * The nightly whole-database dump against a real Postgres: the job writes a + * dump and its manifest, records the run, copies both off-host in the + * envelope the restore opens, keeps only what retention allows; and the + * restore puts the dump into an empty database and refuses one with tables. + * + * `pg_dump` and `pg_restore` are the test container's own. A workstation may + * have none, or one of another major than the server (which `pg_dump` + * refuses), so the test puts two small forwarders on PATH that run the tool + * inside the database container with `docker exec`. Everything else, the + * snapshot, the counts, the manifest, the ledger and the restore's checks, + * runs exactly as in the image. + */ +import { execFileSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Readable } from "node:stream"; + +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +import { prisma } from "@/lib/db"; +import { getDatabaseBackupHealth } from "@/lib/database-backup/health"; +import { runDatabaseBackup } from "@/lib/database-backup/run"; +import { + connect, + countRows, + decryptOffhostFile, + offhostObjectKey, + readManifestFile, +} from "@/lib/database-backup/dump-core.mjs"; + +import { main as restoreMain } from "../../scripts/db-restore.mjs"; +import { shippedMigrations } from "../../scripts/pending-migrations.mjs"; + +const OFFHOST_KEY = "e".repeat(64); +const work = mkdtempSync(join(tmpdir(), "hl-db-backup-")); +const backups = join(work, "backups"); +const shims = join(work, "bin"); +const originalPath = process.env.PATH; + +/** The id of the container that serves DATABASE_URL. */ +function containerId(): string { + const port = new URL(process.env.DATABASE_URL ?? "").port; + const rows = execFileSync( + "docker", + ["ps", "--format", "{{.ID}} {{.Ports}}"], + { + encoding: "utf8", + }, + ).split("\n"); + const row = rows.find((r) => r.includes(`:${port}->5432/tcp`)); + if (!row) throw new Error(`no container publishes port ${port}`); + return row.split(" ")[0]; +} + +/** + * `pg_dump` / `pg_restore` that run inside the database container. The + * connection comes from the PG* variables the core sets; host and port become + * the container's own. A file argument that exists on this machine is sent on + * standard input, which both tools read when no file is named. + */ +function writeShims(container: string): void { + for (const tool of ["pg_dump", "pg_restore"]) { + const path = join(shims, tool); + writeFileSync( + path, + `#!/usr/bin/env node +const { spawn } = require("node:child_process"); +const { createReadStream, existsSync } = require("node:fs"); +const args = process.argv.slice(2); +let input = null; +if (args.length && !args[args.length - 1].startsWith("-") && existsSync(args[args.length - 1])) { + input = args.pop(); +} +const env = []; +for (const name of ["PGUSER", "PGPASSWORD", "PGDATABASE"]) { + if (process.env[name] !== undefined) env.push("-e", name + "=" + process.env[name]); +} +env.push("-e", "PGHOST=127.0.0.1", "-e", "PGPORT=5432"); +const child = spawn("docker", ["exec", "-i", ...env, ${JSON.stringify(container)}, ${JSON.stringify(tool)}, ...args], { + stdio: ["pipe", "inherit", "inherit"], +}); +(input ? createReadStream(input) : process.stdin).pipe(child.stdin); +child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0))); +`, + ); + chmodSync(path, 0o755); + } +} + +/** A bucket in memory, with the one call the job's uploader uses. */ +function memoryBucket() { + const objects = new Map(); + return { + objects, + putStream: async (key: string, body: Readable) => { + const chunks: Buffer[] = []; + for await (const c of body) chunks.push(c as Buffer); + objects.set(key, Buffer.concat(chunks)); + }, + }; +} + +function backupEnv(extra: Record = {}): NodeJS.ProcessEnv { + return { ...process.env, DB_BACKUP_DIR: backups, ...extra }; +} + +beforeAll(async () => { + const { mkdirSync } = await import("node:fs"); + mkdirSync(shims, { recursive: true }); + mkdirSync(backups, { recursive: true }); + writeShims(containerId()); + process.env.PATH = `${shims}:${originalPath}`; + for (const name of ["dump-a", "dump-b", "dump-c"]) { + await prisma.user.create({ + data: { username: `${name}-${Date.now()}` }, + }); + } +}); + +afterAll(() => { + process.env.PATH = originalPath; + vi.unstubAllEnvs(); + rmSync(work, { recursive: true, force: true }); +}); + +describe("the nightly database dump", () => { + let dumpFile = ""; + + it("writes a checked dump and manifest, records the run and copies both off-host", async () => { + vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid"); + vi.stubEnv("BACKUP_S3_BUCKET", "b"); + vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a"); + vi.stubEnv("BACKUP_S3_SECRET_KEY", "s"); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY); + const bucket = memoryBucket(); + const now = new Date("2026-10-20T01:30:00Z"); + + const report = await runDatabaseBackup(prisma, { + now, + env: backupEnv(), + s3: bucket as never, + }); + expect(report).toMatchObject({ + status: "succeeded", + local: "written", + offhost: "uploaded", + error: null, + fileName: "healthlog-20261020T013000Z-daily.dump", + }); + dumpFile = join(backups, report.fileName ?? ""); + + // Both files, under their final names, and nothing half-written. + expect(readdirSync(backups).sort()).toEqual([ + "healthlog-20261020T013000Z-daily.dump", + "healthlog-20261020T013000Z-daily.manifest.json", + ]); + expect(readFileSync(dumpFile).subarray(0, 5).toString("latin1")).toBe( + "PGDMP", + ); + + const manifest = await readManifestFile( + join(backups, "healthlog-20261020T013000Z-daily.manifest.json"), + ); + expect(manifest.kind).toBe("daily"); + expect(manifest.dump.bytes).toBe(report.dumpBytes); + expect(manifest.latestMigration).toBe( + shippedMigrations(join(process.cwd(), "prisma", "migrations")).at(-1), + ); + expect(manifest.postgres.serverVersionNum).toBeGreaterThanOrEqual(160000); + expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count()); + expect(Object.keys(manifest.rowCounts).length).toBeGreaterThan(100); + expect(manifest.keys.encryption.keys.length).toBeGreaterThan(0); + expect(manifest.keys.backupEncryptionKey).toMatch(/^[0-9a-f]{12}$/); + + // The ledger row and what the admin page reads from it. + const runs = await prisma.databaseBackupRun.findMany({ + where: { startedAt: now }, + }); + expect(runs).toHaveLength(1); + expect(runs[0]).toMatchObject({ + status: "succeeded", + fileName: "healthlog-20261020T013000Z-daily.dump", + localResult: "written", + offhostResult: "uploaded", + error: null, + }); + expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes); + const health = await getDatabaseBackupHealth( + new Date("2026-10-20T02:00:00Z"), + backupEnv(), + ); + expect(health).toMatchObject({ + enabled: true, + localDirWritable: true, + offhostConfigured: true, + lastGood: { + fileName: "healthlog-20261020T013000Z-daily.dump", + sizeBytes: manifest.dump.bytes, + local: true, + offhost: true, + }, + }); + + // The off-host pair opens, with the restore's own reader, to the very + // bytes on disk. + expect([...bucket.objects.keys()].sort()).toEqual([ + "database/healthlog-20261020T013000Z-daily.dump.enc", + "database/healthlog-20261020T013000Z-daily.manifest.json.enc", + ]); + const opened = join(work, "opened"); + const { mkdirSync } = await import("node:fs"); + mkdirSync(opened); + for (const name of [ + "healthlog-20261020T013000Z-daily.dump", + "healthlog-20261020T013000Z-daily.manifest.json", + ]) { + writeFileSync( + join(opened, `${name}.enc`), + bucket.objects.get(offhostObjectKey(`${name}.enc`)) as Buffer, + ); + await decryptOffhostFile({ + inPath: join(opened, `${name}.enc`), + outPath: join(opened, name), + objectKey: offhostObjectKey(`${name}.enc`), + keys: [Buffer.from(OFFHOST_KEY, "hex")], + }); + expect( + readFileSync(join(opened, name)).equals( + readFileSync(join(backups, name)), + ), + ).toBe(true); + } + vi.unstubAllEnvs(); + }); + + it("keeps DB_BACKUP_KEEP nightly dumps and removes the oldest", async () => { + for (const day of ["21", "22", "23"]) { + const report = await runDatabaseBackup(prisma, { + now: new Date(`2026-10-${day}T01:30:00Z`), + env: backupEnv({ DB_BACKUP_KEEP: "2" }), + }); + expect(report.status).toBe("succeeded"); + expect(report.offhost).toBe("not_configured"); + } + expect(readdirSync(backups).filter((n) => n.endsWith(".dump"))).toEqual([ + "healthlog-20261022T013000Z-daily.dump", + "healthlog-20261023T013000Z-daily.dump", + ]); + }); + + it("records a failed dump, leaves nothing that looks good, and the page says so", async () => { + const before = readdirSync(backups).sort(); + const url = new URL(process.env.DATABASE_URL ?? ""); + url.password = "not-the-password"; + const now = new Date("2026-10-24T01:30:00Z"); + const report = await runDatabaseBackup(prisma, { + now, + env: backupEnv({ DATABASE_URL: url.toString() }), + }); + expect(report.status).toBe("failed"); + expect(report.local).toBe("failed"); + expect(report.error).toMatch(/password authentication failed/); + expect(readdirSync(backups).sort()).toEqual(before); + const health = await getDatabaseBackupHealth( + new Date("2026-10-24T02:00:00Z"), + backupEnv(), + ); + expect(health.lastFailure?.error).toMatch(/password authentication/); + expect(health.lastGood?.fileName).toBe( + "healthlog-20261023T013000Z-daily.dump", + ); + }); + + it("skips, and says why, with nowhere to put a dump", async () => { + const report = await runDatabaseBackup(prisma, { + now: new Date("2026-10-24T03:00:00Z"), + env: backupEnv({ DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here" }), + }); + expect(report).toMatchObject({ + status: "skipped", + local: "no_directory", + offhost: "not_configured", + }); + expect(report.error).toMatch(/no writable backup directory/); + const off = await runDatabaseBackup(prisma, { + now: new Date("2026-10-24T04:00:00Z"), + env: backupEnv({ DB_BACKUP_ENABLED: "false" }), + }); + expect(off).toMatchObject({ + status: "skipped", + local: "skipped", + offhost: "skipped", + }); + }); + + describe("restoring it", () => { + const target = `restore_${process.pid}_${Date.now()}`; + let targetUrl = ""; + + beforeAll(async () => { + const admin = await connect(process.env.DATABASE_URL ?? ""); + await admin.query(`CREATE DATABASE ${target}`); + await admin.end(); + const url = new URL(process.env.DATABASE_URL ?? ""); + url.pathname = `/${target}`; + targetUrl = url.toString(); + }); + + afterAll(async () => { + const admin = await connect(process.env.DATABASE_URL ?? ""); + await admin.query(`DROP DATABASE IF EXISTS ${target} WITH (FORCE)`); + await admin.end(); + }); + + it("refuses the database the dump came from, which has tables", async () => { + const errors: string[] = []; + const spy = vi + .spyOn(console, "error") + .mockImplementation((l: string) => void errors.push(String(l))); + const code = await restoreMain(["latest"], backupEnv()); + spy.mockRestore(); + expect(code).toBe(1); + expect(errors.join("\n")).toMatch(/only into an empty database/); + }); + + it("restores the newest dump into an empty database with every row count", async () => { + const lines: string[] = []; + const spy = vi + .spyOn(console, "log") + .mockImplementation((l: string) => void lines.push(String(l))); + const code = await restoreMain( + ["latest"], + backupEnv({ DATABASE_URL: targetUrl }), + ); + spy.mockRestore(); + expect(code, lines.join("\n")).toBe(0); + expect(lines.join("\n")).toMatch( + /all \d+ tables hold the row counts the manifest names/, + ); + + const manifest = await readManifestFile( + join(backups, "healthlog-20261023T013000Z-daily.manifest.json"), + ); + const client = await connect(targetUrl); + try { + expect(await countRows(client)).toEqual(manifest.rowCounts); + const { rows } = await client.query( + "SELECT count(*)::int AS n FROM users WHERE username LIKE 'dump-%'", + ); + expect(rows[0].n).toBeGreaterThanOrEqual(3); + } finally { + await client.end(); + } + // Restoring again into the now full database is refused. + const again = await restoreMain( + ["latest"], + backupEnv({ DATABASE_URL: targetUrl }), + ); + expect(again).toBe(1); + expect(existsSync(dumpFile)).toBe(false); // the first dump aged out + }); + }); +}); + +describe("the dump before migrating", () => { + const preDir = join(work, "pre"); + const appDir = join(work, "app"); + const failing = join(work, "failing-bin"); + let cwd = ""; + + beforeAll(async () => { + const { cpSync, mkdirSync } = await import("node:fs"); + // An image that ships one migration the database has not applied. + mkdirSync(join(appDir, "prisma"), { recursive: true }); + cpSync( + join(process.cwd(), "prisma", "migrations"), + join(appDir, "prisma", "migrations"), + { recursive: true }, + ); + mkdirSync(join(appDir, "prisma", "migrations", "9999_not_applied_yet")); + writeFileSync( + join( + appDir, + "prisma", + "migrations", + "9999_not_applied_yet", + "migration.sql", + ), + "", + ); + mkdirSync(preDir, { recursive: true }); + mkdirSync(failing, { recursive: true }); + writeFileSync( + join(failing, "pg_dump"), + "#!/bin/sh\necho 'pg_dump: error: could not write to output file: No space left on device' >&2\nexit 1\n", + ); + chmodSync(join(failing, "pg_dump"), 0o755); + cwd = process.cwd(); + process.chdir(appDir); + }); + + afterAll(() => { + process.chdir(cwd); + process.env.PATH = `${shims}:${originalPath}`; + }); + + async function preMigration(env: Record) { + const { preMigrationDump } = await import("../../scripts/db-dump.mjs"); + const errors: string[] = []; + const spyErr = vi + .spyOn(console, "error") + .mockImplementation((l: string) => void errors.push(String(l))); + const spyLog = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + const code = await preMigrationDump({ ...process.env, ...env }); + return { code, errors: errors.join("\n") }; + } finally { + spyErr.mockRestore(); + spyLog.mockRestore(); + } + } + + it("writes a checked dump with its manifest, and keeps the last two", async () => { + for (let i = 0; i < 3; i++) { + if (i > 0) await new Promise((r) => setTimeout(r, 1100)); + const { code } = await preMigration({ DB_BACKUP_DIR: preDir }); + expect(code).toBe(0); + } + const names = readdirSync(preDir).sort(); + expect(names).toHaveLength(4); + expect(names.filter((n) => n.endsWith("-pre-migration.dump"))).toHaveLength( + 2, + ); + const manifest = await readManifestFile( + join(preDir, names.find((n) => n.endsWith(".manifest.json")) ?? ""), + ); + expect(manifest.kind).toBe("pre-migration"); + expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count()); + }); + + it("does nothing when switched off", async () => { + const before = readdirSync(preDir); + const { code } = await preMigration({ + DB_BACKUP_DIR: preDir, + DB_BACKUP_BEFORE_MIGRATE: "false", + }); + expect(code).toBe(0); + expect(readdirSync(preDir)).toEqual(before); + }); + + it("warns and lets the boot go on without a writable directory", async () => { + const { code, errors } = await preMigration({ + DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here", + }); + expect(code).toBe(0); + expect(errors).toMatch( + /is not writable, so no dump is taken before the 1 pending migration/, + ); + }); + + it("stops the boot when the dump fails on a writable directory, and leaves no dump behind", async () => { + // A second of its own, so the name cannot meet the last dump's. + await new Promise((r) => setTimeout(r, 1100)); + const before = readdirSync(preDir).sort(); + process.env.PATH = `${failing}:${originalPath}`; + const { code, errors } = await preMigration({ DB_BACKUP_DIR: preDir }); + process.env.PATH = `${shims}:${originalPath}`; + expect(code).toBe(1); + expect(errors).toMatch( + /the database dump before migrating failed: .*No space left/, + ); + expect(readdirSync(preDir).sort()).toEqual(before); + }); +}); + +describe("two dumps in the same second", () => { + it("refuses the second and leaves the first whole", async () => { + const { createDatabaseDump } = + await import("@/lib/database-backup/dump-core.mjs"); + const dir = join(work, "same-second"); + const { mkdirSync } = await import("node:fs"); + mkdirSync(dir); + const now = new Date("2026-10-26T01:30:00Z"); + const options = { + databaseUrl: process.env.DATABASE_URL ?? "", + dir, + kind: "daily" as const, + now, + }; + await createDatabaseDump(options); + const before = readdirSync(dir) + .sort() + .map((n) => [n, readFileSync(join(dir, n)).length]); + await expect(createDatabaseDump(options)).rejects.toThrow(/already exists/); + expect( + readdirSync(dir) + .sort() + .map((n) => [n, readFileSync(join(dir, n)).length]), + ).toEqual(before); + }); +}); From e665f17f521285f144828af8ce6bac49456a8e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 12:52:07 +0100 Subject: [PATCH 12/44] feat(backup): dump before migrating, and restore a dump into an empty database The image now carries postgresql18-client (pg_dump and pg_restore 18.6) and a /var/lib/healthlog/backups directory owned by the app user, which the bundled compose file backs with a new `backups` volume for `app` and the commented `app-worker`; a fresh named volume inherits that owner. Before `migrate deploy`, when migrations are pending on a database that has applied some, the entrypoint writes healthlog-

+ ); +} diff --git a/src/types/backups.ts b/src/types/backups.ts index f3c042999..5f2a28e2c 100644 --- a/src/types/backups.ts +++ b/src/types/backups.ts @@ -9,6 +9,7 @@ * both sides a route-handler-independent shared type home; the route * keeps owning the HTTP contract, the component keeps owning the UI. */ +import type { DatabaseBackupHealth } from "@/lib/database-backup/health"; import type { BackupPassActivity } from "@/lib/jobs/backup-pass-attempts"; import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status"; import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness"; @@ -101,6 +102,12 @@ export interface BackupsList { * the operator's bucket, and those two fail independently. */ offhost: OffhostBackupOverview; + /** + * The nightly whole-database dump, the copy that rebuilds a lost server: + * the newest good dump (kept here, off-host, or both), the newest failed + * run, and whether this server can keep dumps at all. + */ + database: DatabaseBackupHealth; /** * Soft retention hint — the worker is configured for weekly backups * (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and From ca644647a0153a1fa8d783960b25b9db537b2442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 12:52:56 +0100 Subject: [PATCH 14/44] docs(ops): describe disaster recovery from the database dump docs/ops/disaster-recovery.md says what is dumped and where, that local dumps are unencrypted and why, which keys a restore needs and what each one costs when missing, how to restore onto a new server or back to the state before a migration, and how the procedure is tested. The deploy runbook and the getting-started guide now describe the dump the entrypoint takes before migrating, and the off-host account copy page points at the dump for rebuilding a server. --- docs/ops/backup-restore.md | 5 + docs/ops/deploy.md | 35 ++++-- docs/ops/disaster-recovery.md | 164 +++++++++++++++++++++++++++ docs/self-hosting/getting-started.md | 28 ++++- 4 files changed, 215 insertions(+), 17 deletions(-) create mode 100644 docs/ops/disaster-recovery.md diff --git a/docs/ops/backup-restore.md b/docs/ops/backup-restore.md index 09524b2c8..f3d4c0571 100644 --- a/docs/ops/backup-restore.md +++ b/docs/ops/backup-restore.md @@ -5,6 +5,11 @@ user's JSON dump, encrypted with AES-256-GCM under a SEPARATE key (`BACKUP_ENCRYPTION_KEY`), to any S3-compatible bucket — Cloudflare R2, AWS S3, Backblaze B2, MinIO, etc. +These per-account copies roll one account back on the same server. To +rebuild a server, use the nightly database dump, which the same job +configuration also uploads (`database/healthlog-
- + {recentProof.dialog} - + {/* ── Coverage summary ───────────────────────────────────────── */} ` — the "Back up your encryption key" step on the admin - * Encryption page. - * - * Shows the active key id and its fingerprint (never the key), where the key - * lives on each platform, what is lost without it, a "Check my copy" field - * and the confirmation. The confirmation is bound to the key id and - * fingerprint shown here: if the server's key changed since the page loaded, - * the server answers 409 and the card reloads. - */ - -import { useState } from "react"; -import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { KeyRound, Loader2 } from "lucide-react"; -import { toast } from "sonner"; -import { showErrorToast } from "@/lib/errors/error-toast"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { SettingsCard } from "@/components/settings/settings-card"; -import { SettingsCardHeader } from "@/components/settings/_card-header"; -import { SettingsCardActions } from "@/components/settings/_card-actions"; -import { QueryErrorRow } from "@/components/ui/query-error-row"; -import { useFormatters, useTranslations } from "@/lib/i18n/context"; -import { queryKeys } from "@/lib/query-keys"; -import { ApiError, apiFetch, apiPost } from "@/lib/api/api-fetch"; -import { - useKeyBackupStatus, - type KeyBackupPlatform, - type KeyBackupStatus, -} from "./use-key-backup-status"; - -const PLATFORMS: KeyBackupPlatform[] = [ - "compose", - "truenas", - "coolify", - "unraid", - "portainer", -]; - -/** Literal keys per platform, so the i18n coverage guard sees every one. */ -function platformCopy( - t: (key: string) => string, - p: KeyBackupPlatform, -): { label: string; where: string } { - switch (p) { - case "truenas": - return { - label: t("admin.keyBackup.platform.truenasLabel"), - where: t("admin.keyBackup.platform.truenasWhere"), - }; - case "coolify": - return { - label: t("admin.keyBackup.platform.coolifyLabel"), - where: t("admin.keyBackup.platform.coolifyWhere"), - }; - case "unraid": - return { - label: t("admin.keyBackup.platform.unraidLabel"), - where: t("admin.keyBackup.platform.unraidWhere"), - }; - case "portainer": - return { - label: t("admin.keyBackup.platform.portainerLabel"), - where: t("admin.keyBackup.platform.portainerWhere"), - }; - case "compose": - return { - label: t("admin.keyBackup.platform.composeLabel"), - where: t("admin.keyBackup.platform.composeWhere"), - }; - } -} - -export function KeyBackupCard() { - const { t } = useTranslations(); - const fmt = useFormatters(); - const queryClient = useQueryClient(); - const status = useKeyBackupStatus(); - const [candidate, setCandidate] = useState(""); - const [checkResult, setCheckResult] = useState(null); - - const invalidate = () => - Promise.all([ - queryClient.invalidateQueries({ - queryKey: queryKeys.adminEncryptionKeyBackup(), - }), - queryClient.invalidateQueries({ - queryKey: queryKeys.adminEncryptionStatus(), - }), - ]); - - const confirm = useMutation({ - mutationFn: (s: KeyBackupStatus) => - apiPost( - "/api/admin/encryption/key-backup/confirm", - { keyId: s.activeKeyId, fingerprint: s.fingerprint }, - { credentials: "include" }, - ), - onSuccess: async () => { - toast(t("admin.keyBackup.confirmed")); - await invalidate(); - }, - onError: async (err: Error) => { - if (err instanceof ApiError && err.status === 409) { - showErrorToast(t("admin.keyBackup.stale"), { cause: err }); - await invalidate(); - return; - } - showErrorToast(err.message || t("admin.keyBackup.confirmFailed"), { - cause: err, - }); - }, - }); - - const verify = useMutation({ - // Not `apiPost`: no idempotency key, so no cached response row exists - // for a request whose body was a key. - mutationFn: (encryptionKey: string) => - apiFetch<{ matches: boolean }>( - "/api/admin/encryption/key-backup/verify", - { - method: "POST", - credentials: "include", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ encryptionKey }), - }, - ), - onSuccess: (data) => { - setCheckResult(data.matches); - setCandidate(""); - }, - onError: (err: Error) => { - setCheckResult(null); - showErrorToast(err.message || t("admin.keyBackup.checkFailed"), { - cause: err, - }); - }, - }); - - const insecure = - typeof window !== "undefined" && window.isSecureContext === false; - - if (!status.data) { - return ( - - - {status.isError ? ( - void status.refetch()} - /> - ) : ( -
- - {t("admin.keyBackup.loading")} -
- )} - - ); - } - - const s = status.data; - return ( - - - {t("admin.keyBackup.badgeDue")} - - ) : ( - {t("admin.keyBackup.badgeDone")} - ) - } - /> - -
-
-
- {t("admin.keyBackup.keyId")} -
-
{s.activeKeyId}
-
-
-
- {t("admin.keyBackup.fingerprint")} -
-
- {s.fingerprint} -
-
-
- -

{t("admin.keyBackup.whatIsLost")}

- - - - {PLATFORMS.map((p) => ( - - {platformCopy(t, p).label} - - ))} - - {PLATFORMS.map((p) => ( - - {platformCopy(t, p).where} - - ))} - - -

- {t("admin.keyBackup.fingerprintHint")} -

- -
{ - e.preventDefault(); - if (candidate.trim()) verify.mutate(candidate); - }} - > - -
- { - setCandidate(e.target.value); - setCheckResult(null); - }} - className="font-mono" - /> - -
-

- {insecure - ? t("admin.keyBackup.checkInsecureNote") - : t("admin.keyBackup.checkNote")} -

- {checkResult !== null && ( -

- {checkResult - ? t("admin.keyBackup.checkMatches") - : t("admin.keyBackup.checkNoMatch")} -

- )} -
- - {!s.due && s.confirmedAt && ( -

- {s.confirmedBy?.email - ? t("admin.keyBackup.confirmedAtBy", { - date: fmt.dateTime(new Date(s.confirmedAt)), - who: s.confirmedBy.email, - }) - : t("admin.keyBackup.confirmedAt", { - date: fmt.dateTime(new Date(s.confirmedAt)), - })} -

- )} - {s.due && s.confirmedKeyId && ( -

- {t("admin.keyBackup.dueAgain", { keyId: s.confirmedKeyId })} -

- )} - - {s.due && ( - - - - )} - - ); -} diff --git a/src/components/admin/recovery-kit-card.tsx b/src/components/admin/recovery-kit-card.tsx new file mode 100644 index 000000000..b08a9282f --- /dev/null +++ b/src/components/admin/recovery-kit-card.tsx @@ -0,0 +1,519 @@ +"use client"; + +/** + * `` — the recovery kit on the admin Encryption page. + * + * Lists every key a restore of this server needs: the application keys + * (including a retired one a backup still lists), the off-host backup keys and + * the token key. Each row shows the key's id and fingerprint (never the key), + * why the kit lists it and whether the confirmation covers it. Below them: + * where the keys are set on each platform, the short restore path with a link + * to the full guide, a "Check my copy" field and the confirmation. + * + * The confirmation sends the kit exactly as this card showed it. When the + * server's kit changed since the page loaded (a rotation, a new backup key), + * the server answers 409 and the card reloads. + */ + +import { useState } from "react"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { BookOpen, KeyRound, Loader2 } from "lucide-react"; +import { toast } from "sonner"; + +import { SettingsCardActions } from "@/components/settings/_card-actions"; +import { SettingsCardHeader } from "@/components/settings/_card-header"; +import { SettingsCard } from "@/components/settings/settings-card"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { InlineAction } from "@/components/ui/inline-action"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { NativeSelect } from "@/components/ui/native-select"; +import { QueryErrorRow } from "@/components/ui/query-error-row"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { ApiError, apiFetch, apiPost } from "@/lib/api/api-fetch"; +import { showErrorToast } from "@/lib/errors/error-toast"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import { queryKeys } from "@/lib/query-keys"; + +import { + useRecoveryKit, + type KitKey, + type KitPlatform, + type RecoveryKit, +} from "./use-recovery-kit"; + +const PLATFORMS: KitPlatform[] = [ + "compose", + "truenas", + "coolify", + "unraid", + "portainer", +]; + +/** The disaster recovery guide in the repository, the same on every host. */ +export const DISASTER_RECOVERY_GUIDE_URL = + "https://github.com/MBombeck/HealthLog/blob/main/docs/ops/disaster-recovery.md"; + +/** Literal keys per platform, so the i18n coverage guard sees every one. */ +function platformCopy( + t: (key: string) => string, + p: KitPlatform, +): { label: string; where: string } { + switch (p) { + case "truenas": + return { + label: t("admin.recoveryKit.platform.truenasLabel"), + where: t("admin.recoveryKit.platform.truenasWhere"), + }; + case "coolify": + return { + label: t("admin.recoveryKit.platform.coolifyLabel"), + where: t("admin.recoveryKit.platform.coolifyWhere"), + }; + case "unraid": + return { + label: t("admin.recoveryKit.platform.unraidLabel"), + where: t("admin.recoveryKit.platform.unraidWhere"), + }; + case "portainer": + return { + label: t("admin.recoveryKit.platform.portainerLabel"), + where: t("admin.recoveryKit.platform.portainerWhere"), + }; + case "compose": + return { + label: t("admin.recoveryKit.platform.composeLabel"), + where: t("admin.recoveryKit.platform.composeWhere"), + }; + } +} + +function keyLabel( + t: (key: string, params?: Record) => string, + key: KitKey, +): string { + switch (key.kind) { + case "application": + return t("admin.recoveryKit.kind.application", { id: key.id }); + case "backup": + return key.need.active + ? t("admin.recoveryKit.kind.backup") + : t("admin.recoveryKit.kind.backupPrevious"); + case "token": + return t("admin.recoveryKit.kind.token"); + } +} + +/** Why the kit lists a key, as short sentences. */ +function KeyNeeds({ k }: { k: KitKey }) { + const { t } = useTranslations(); + const fmt = useFormatters(); + const lines: string[] = []; + if (k.kind === "token") { + lines.push(t("admin.recoveryKit.need.token")); + } else if (k.kind === "backup") { + lines.push( + k.need.active + ? t("admin.recoveryKit.need.backupActive") + : t("admin.recoveryKit.need.backupPrevious"), + ); + } else { + if (k.need.active) lines.push(t("admin.recoveryKit.need.active")); + else if (k.need.configured) + lines.push(t("admin.recoveryKit.need.configured")); + if (k.need.storedCopies > 0) { + lines.push( + t("admin.recoveryKit.need.storedCopies", { + count: k.need.storedCopies, + }), + ); + } + if (k.need.offhostUntil === "unknown") { + lines.push(t("admin.recoveryKit.need.offhostUnknown")); + } else if (k.need.offhostUntil) { + lines.push( + t("admin.recoveryKit.need.offhostUntil", { + date: fmt.date(new Date(k.need.offhostUntil)), + }), + ); + } + } + lines.push(t("admin.recoveryKit.source", { variable: k.source })); + return

{lines.join(" ")}

; +} + +function KeyStatus({ k }: { k: KitKey }) { + const { t } = useTranslations(); + if (k.problem === "unreadable") { + return ( + + {t("admin.recoveryKit.status.unreadable")} + + ); + } + if (k.confirmed) { + return ( + {t("admin.recoveryKit.status.confirmed")} + ); + } + return ( + + {k.problem === "not_configured" + ? t("admin.recoveryKit.status.notConfigured") + : t("admin.recoveryKit.status.new")} + + ); +} + +/** One row per key: what it is, its id and fingerprint, why it is listed. */ +export function RecoveryKitKeyList({ kit }: { kit: RecoveryKit }) { + const { t } = useTranslations(); + return ( +
    + {kit.keys.map((k) => ( +
  • +
    +

    {keyLabel(t, k)}

    +
    + +
    +
    +
    +
    +
    + {t("admin.recoveryKit.keyId")} +
    +
    {k.id}
    +
    +
    +
    + {t("admin.recoveryKit.fingerprint")} +
    +
    + {k.fingerprint ?? t("admin.recoveryKit.noFingerprint")} +
    +
    +
    + + {k.problem === "not_configured" && k.kind !== "token" ? ( +

    + {t("admin.recoveryKit.notConfiguredHint")} +

    + ) : null} + {k.problem === "unreadable" ? ( +

    {t("admin.recoveryKit.unreadableHint")}

    + ) : null} +
    • + ))} +
+ ); +} + +function CheckMyCopy({ kit }: { kit: RecoveryKit }) { + const { t } = useTranslations(); + const checkable = kit.keys.filter((k) => k.fingerprint !== null); + const [selected, setSelected] = useState( + checkable[0] ? `${checkable[0].kind}:${checkable[0].id}` : "", + ); + const [candidate, setCandidate] = useState(""); + const [checkResult, setCheckResult] = useState(null); + + const verify = useMutation({ + // Not `apiPost`: no idempotency key, so no cached response row exists + // for a request whose body was a key. + mutationFn: (keyCopy: string) => { + const target = checkable.find((k) => `${k.kind}:${k.id}` === selected); + return apiFetch<{ matches: boolean }>("/api/admin/recovery-kit/verify", { + method: "POST", + credentials: "include", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + kind: target?.kind, + id: target?.id, + keyCopy, + }), + }); + }, + onSuccess: (data) => { + setCheckResult(data.matches); + setCandidate(""); + }, + onError: (err: Error) => { + setCheckResult(null); + showErrorToast(err.message || t("admin.recoveryKit.checkFailed"), { + cause: err, + }); + }, + }); + + if (checkable.length === 0) return null; + const insecure = + typeof window !== "undefined" && window.isSecureContext === false; + + return ( +
{ + e.preventDefault(); + if (candidate.trim() && selected) verify.mutate(candidate); + }} + > +

{t("admin.recoveryKit.checkLabel")}

+
+
+ + { + setSelected(e.target.value); + setCheckResult(null); + }} + > + {checkable.map((k) => ( + + ))} + +
+
+ + { + setCandidate(e.target.value); + setCheckResult(null); + }} + className="font-mono" + /> +
+ +
+

+ {insecure + ? t("admin.recoveryKit.checkInsecureNote") + : t("admin.recoveryKit.checkNote")} +

+ {checkResult !== null && ( +

+ {checkResult + ? t("admin.recoveryKit.checkMatches") + : t("admin.recoveryKit.checkNoMatch")} +

+ )} +
+ ); +} + +export function RecoveryKitCard() { + const { t } = useTranslations(); + const fmt = useFormatters(); + const queryClient = useQueryClient(); + const status = useRecoveryKit(); + + const invalidate = () => + Promise.all([ + queryClient.invalidateQueries({ + queryKey: queryKeys.adminRecoveryKit(), + }), + queryClient.invalidateQueries({ + queryKey: queryKeys.adminRecoveryReadiness(), + }), + ]); + + const confirm = useMutation({ + mutationFn: (kit: RecoveryKit) => + apiPost( + "/api/admin/recovery-kit/confirm", + { + entries: kit.keys.map(({ kind, id, fingerprint }) => ({ + kind, + id, + fingerprint, + })), + }, + { credentials: "include" }, + ), + onSuccess: async () => { + toast(t("admin.recoveryKit.confirmed")); + await invalidate(); + }, + onError: async (err: Error) => { + if (err instanceof ApiError && err.status === 409) { + showErrorToast(t("admin.recoveryKit.stale"), { cause: err }); + await invalidate(); + return; + } + showErrorToast(err.message || t("admin.recoveryKit.confirmFailed"), { + cause: err, + }); + }, + }); + + if (!status.data) { + return ( + + + {status.isError ? ( + void status.refetch()} + /> + ) : ( +
+ + {t("admin.recoveryKit.loading")} +
+ )} + + ); + } + + const kit = status.data; + const unconfirmedIds = kit.unconfirmed.map((k) => k.id).join(", "); + return ( + + + {t("admin.recoveryKit.badgeDue")} + + ) : ( + {t("admin.recoveryKit.badgeDone")} + ) + } + /> + +

{t("admin.recoveryKit.intro")}

+ + + +

+ {t("admin.recoveryKit.fingerprintHint")} +

+ +
+

+ {t("admin.recoveryKit.whereTitle")} +

+ + + {PLATFORMS.map((p) => ( + + {platformCopy(t, p).label} + + ))} + + {PLATFORMS.map((p) => ( + + {platformCopy(t, p).where} + + ))} + +
+ + + + + + {!kit.due && kit.confirmedAt && ( +

+ {kit.confirmedBy?.email + ? t("admin.recoveryKit.confirmedAtBy", { + date: fmt.dateTime(new Date(kit.confirmedAt)), + who: kit.confirmedBy.email, + }) + : t("admin.recoveryKit.confirmedAt", { + date: fmt.dateTime(new Date(kit.confirmedAt)), + })} +

+ )} + {kit.due && kit.confirmedAt && ( +

+ {t("admin.recoveryKit.dueAgain", { + date: fmt.dateTime(new Date(kit.confirmedAt)), + keys: unconfirmedIds, + })} +

+ )} + + {kit.due && ( + + + + )} + + ); +} diff --git a/src/components/admin/system-status-summary.tsx b/src/components/admin/system-status-summary.tsx index 7c412113c..a02d64609 100644 --- a/src/components/admin/system-status-summary.tsx +++ b/src/components/admin/system-status-summary.tsx @@ -12,27 +12,17 @@ * don't re-fetch. */ -import { - Clock, - Cog, - Database, - Globe, - KeyRound, - Loader2, - Server, -} from "lucide-react"; +import { Clock, Cog, Database, Globe, Loader2, Server } from "lucide-react"; import { formatDateTime } from "@/lib/format"; import { SettingsCard } from "@/components/settings/settings-card"; import { SettingsCardHeader } from "@/components/settings/_card-header"; import { QueryErrorRow } from "@/components/ui/query-error-row"; import { useTranslations } from "@/lib/i18n/context"; import { geoLookupState, StatusItem, useSystemStatus } from "./_shared"; -import { useKeyBackupStatus } from "./use-key-backup-status"; export function SystemStatusSummary() { const { t } = useTranslations(); const { data: status, isError, error, refetch } = useSystemStatus(); - const { data: keyBackup } = useKeyBackupStatus(); return ( - {/* v1.40.0 — the encryption key backup step, warning while due. */} - {keyBackup && ( - - )} {/* v1.4.27 — surface the offline-geo state so the maintainer spots the missing GeoLite2 databases without crawling logs. Read from the admin status, not the public /api/version. */} diff --git a/src/components/admin/use-key-backup-status.ts b/src/components/admin/use-key-backup-status.ts deleted file mode 100644 index 6f4304924..000000000 --- a/src/components/admin/use-key-backup-status.ts +++ /dev/null @@ -1,36 +0,0 @@ -"use client"; - -import { useQuery } from "@tanstack/react-query"; -import { apiGet } from "@/lib/api/api-fetch"; -import { queryKeys } from "@/lib/query-keys"; - -export type KeyBackupPlatform = - "compose" | "truenas" | "coolify" | "unraid" | "portainer"; - -/** `GET /api/admin/encryption/key-backup`; key id and fingerprint only. */ -export interface KeyBackupStatus { - due: boolean; - activeKeyId: string; - fingerprint: string; - confirmedAt: string | null; - confirmedKeyId: string | null; - confirmedFingerprint: string | null; - confirmedBy: { id: string; email: string | null } | null; - platformHint: KeyBackupPlatform; -} - -/** - * The key backup step's state. `enabled` lets the dashboard banner ask only - * for an admin; the route itself is admin-only and cookie-only. - */ -export function useKeyBackupStatus(enabled = true) { - return useQuery({ - queryKey: queryKeys.adminEncryptionKeyBackup(), - queryFn: () => - apiGet("/api/admin/encryption/key-backup", { - credentials: "include", - }), - enabled, - staleTime: 60_000, - }); -} diff --git a/src/components/admin/use-recovery-kit.ts b/src/components/admin/use-recovery-kit.ts new file mode 100644 index 000000000..534888758 --- /dev/null +++ b/src/components/admin/use-recovery-kit.ts @@ -0,0 +1,30 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { apiGet } from "@/lib/api/api-fetch"; +import { queryKeys } from "@/lib/query-keys"; +import type { RecoveryKit } from "@/lib/recovery/kit-rules"; + +export type { + KitKey, + KitKeyKind, + KitPlatform, + RecoveryKit, +} from "@/lib/recovery/kit-rules"; + +/** + * `GET /api/admin/recovery-kit`: key ids and fingerprints only. `enabled` lets + * the dashboard banner ask only for an admin; the route itself is admin-only + * and cookie-only. + */ +export function useRecoveryKit(enabled = true) { + return useQuery({ + queryKey: queryKeys.adminRecoveryKit(), + queryFn: () => + apiGet("/api/admin/recovery-kit", { + credentials: "include", + }), + enabled, + staleTime: 60_000, + }); +} diff --git a/src/components/dashboard/key-backup-banner.tsx b/src/components/dashboard/recovery-kit-banner.tsx similarity index 51% rename from src/components/dashboard/key-backup-banner.tsx rename to src/components/dashboard/recovery-kit-banner.tsx index 8b8901b2b..953d2e3fc 100644 --- a/src/components/dashboard/key-backup-banner.tsx +++ b/src/components/dashboard/recovery-kit-banner.tsx @@ -1,38 +1,39 @@ "use client"; /** - * Dashboard banner for an admin while the encryption key backup step is due. - * Not dismissable by design: it goes away when the step is done, and comes - * back when the key changes. Renders nothing for anyone who is not an admin, - * and asks the server nothing for them either. + * Dashboard banner for an admin while the recovery kit is due: never + * confirmed, or listing a key the confirmation does not cover (a rotation, a + * new backup key). Not dismissable by design: it goes away when the kit is + * confirmed, and comes back when a key changes. Renders nothing for anyone + * who is not an admin, and asks the server nothing for them either. */ import Link from "next/link"; import { KeyRound } from "lucide-react"; import { useTranslations } from "@/lib/i18n/context"; -import { useKeyBackupStatus } from "@/components/admin/use-key-backup-status"; +import { useRecoveryKit } from "@/components/admin/use-recovery-kit"; import { InlineAction } from "@/components/ui/inline-action"; -export function KeyBackupBanner({ isAdmin }: { isAdmin: boolean }) { +export function RecoveryKitBanner({ isAdmin }: { isAdmin: boolean }) { const { t } = useTranslations(); - const { data } = useKeyBackupStatus(isAdmin); + const { data } = useRecoveryKit(isAdmin); if (!isAdmin || !data?.due) return null; return (
diff --git a/src/lib/crypto/__tests__/key-backup.test.ts b/src/lib/crypto/__tests__/key-backup.test.ts deleted file mode 100644 index e82929db8..000000000 --- a/src/lib/crypto/__tests__/key-backup.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { isKeyBackupDue, resolvePlatformHint } from "../key-backup"; - -const confirmed = (keyId: string, fingerprint: string) => ({ - encryptionKeyBackupConfirmedAt: new Date("2026-10-03T10:00:00Z"), - encryptionKeyBackupConfirmedKeyId: keyId, - encryptionKeyBackupConfirmedFingerprint: fingerprint, -}); - -describe("key backup step", () => { - it("is due without any confirmation", () => { - expect(isKeyBackupDue(null, "v1", "abcdefabcdef")).toBe(true); - expect( - isKeyBackupDue( - { - encryptionKeyBackupConfirmedAt: null, - encryptionKeyBackupConfirmedKeyId: null, - encryptionKeyBackupConfirmedFingerprint: null, - }, - "v1", - "abcdefabcdef", - ), - ).toBe(true); - }); - - it("is done only for the exact key that was confirmed", () => { - expect( - isKeyBackupDue(confirmed("v1", "abcdefabcdef"), "v1", "abcdefabcdef"), - ).toBe(false); - }); - - it("is due again after a rotation (new active id)", () => { - expect( - isKeyBackupDue(confirmed("v1", "abcdefabcdef"), "v2", "abcdefabcdef"), - ).toBe(true); - }); - - it("is due again after a re-keyed install (same id, other key)", () => { - expect( - isKeyBackupDue(confirmed("v1", "abcdefabcdef"), "v1", "123456123456"), - ).toBe(true); - }); - - it("reads the platform hint and falls back to compose", () => { - expect(resolvePlatformHint("truenas")).toBe("truenas"); - expect(resolvePlatformHint(" Unraid ")).toBe("unraid"); - expect(resolvePlatformHint(undefined)).toBe("compose"); - expect(resolvePlatformHint("kubernetes")).toBe("compose"); - }); -}); diff --git a/src/lib/crypto/key-backup.ts b/src/lib/crypto/key-backup.ts deleted file mode 100644 index 6ea14555c..000000000 --- a/src/lib/crypto/key-backup.ts +++ /dev/null @@ -1,142 +0,0 @@ -/** - * The "Back up your encryption key" step: instance state and its rules. - * - * The confirmation lives on the `AppSettings` singleton, bound to the active - * key id and that key's fingerprint, so it is an instance fact (it survives - * the admin who gave it) and a rotation or a re-keyed install makes the step - * due again. Nothing here ever reads, returns or stores the key itself. - */ -import { prisma } from "@/lib/db"; -import { envValue } from "@/lib/env"; -import { getActiveKeyId, getKeyFingerprint } from "@/lib/crypto"; - -/** Where the operator sets the key; picks the tab the step opens on. */ -export const KEY_BACKUP_PLATFORMS = [ - "compose", - "truenas", - "coolify", - "unraid", - "portainer", -] as const; -export type KeyBackupPlatform = (typeof KEY_BACKUP_PLATFORMS)[number]; - -/** Wire code for a confirmation given for a key that is no longer active. */ -export const KEY_BACKUP_STALE_CODE = "encryption.keyBackup.stale"; - -/** - * `HEALTHLOG_PLATFORM`, when it names a known platform; `compose` otherwise. - * Catalog templates set it so the step opens on the right instructions. - */ -export function resolvePlatformHint( - raw: string | undefined = envValue("HEALTHLOG_PLATFORM"), -): KeyBackupPlatform { - const value = raw?.trim().toLowerCase(); - return (KEY_BACKUP_PLATFORMS as readonly string[]).includes(value ?? "") - ? (value as KeyBackupPlatform) - : "compose"; -} - -export interface KeyBackupStatus { - /** True until a confirmation exists for the active key id AND fingerprint. */ - due: boolean; - activeKeyId: string; - /** Fingerprint of the active key (first 12 hex of SHA-256 over its bytes). */ - fingerprint: string; - confirmedAt: string | null; - confirmedKeyId: string | null; - confirmedFingerprint: string | null; - /** The confirming admin, while that account exists. */ - confirmedBy: { id: string; email: string | null } | null; - platformHint: KeyBackupPlatform; -} - -interface StoredConfirmation { - encryptionKeyBackupConfirmedAt: Date | null; - encryptionKeyBackupConfirmedKeyId: string | null; - encryptionKeyBackupConfirmedFingerprint: string | null; -} - -/** Pure rule: is the stored confirmation for this exact key? */ -export function isKeyBackupDue( - stored: StoredConfirmation | null, - activeKeyId: string, - fingerprint: string, -): boolean { - return !( - stored?.encryptionKeyBackupConfirmedAt && - stored.encryptionKeyBackupConfirmedKeyId === activeKeyId && - stored.encryptionKeyBackupConfirmedFingerprint === fingerprint - ); -} - -function activeKey(): { activeKeyId: string; fingerprint: string } { - const activeKeyId = getActiveKeyId(); - // The active id is always configured, so the fingerprint is never null. - return { activeKeyId, fingerprint: getKeyFingerprint(activeKeyId) ?? "" }; -} - -export async function readKeyBackupStatus(): Promise { - const { activeKeyId, fingerprint } = activeKey(); - const row = await prisma.appSettings.findUnique({ - where: { id: "singleton" }, - select: { - encryptionKeyBackupConfirmedAt: true, - encryptionKeyBackupConfirmedKeyId: true, - encryptionKeyBackupConfirmedFingerprint: true, - encryptionKeyBackupConfirmedByUserId: true, - }, - }); - const confirmedBy = row?.encryptionKeyBackupConfirmedByUserId - ? await prisma.user.findUnique({ - where: { id: row.encryptionKeyBackupConfirmedByUserId }, - select: { id: true, email: true }, - }) - : null; - return { - due: isKeyBackupDue(row, activeKeyId, fingerprint), - activeKeyId, - fingerprint, - confirmedAt: row?.encryptionKeyBackupConfirmedAt?.toISOString() ?? null, - confirmedKeyId: row?.encryptionKeyBackupConfirmedKeyId ?? null, - confirmedFingerprint: row?.encryptionKeyBackupConfirmedFingerprint ?? null, - confirmedBy: confirmedBy ?? null, - platformHint: resolvePlatformHint(), - }; -} - -export type ConfirmKeyBackupResult = - | { ok: true; status: KeyBackupStatus } - | { ok: false; activeKeyId: string; fingerprint: string }; - -/** - * Record the confirmation, but only for the key the admin was shown: a page - * opened before a rotation must not confirm the new key. - */ -export async function confirmKeyBackup(input: { - keyId: string; - fingerprint: string; - userId: string; -}): Promise { - const { activeKeyId, fingerprint } = activeKey(); - if (input.keyId !== activeKeyId || input.fingerprint !== fingerprint) { - return { ok: false, activeKeyId, fingerprint }; - } - const now = new Date(); - await prisma.appSettings.upsert({ - where: { id: "singleton" }, - create: { - id: "singleton", - encryptionKeyBackupConfirmedAt: now, - encryptionKeyBackupConfirmedKeyId: activeKeyId, - encryptionKeyBackupConfirmedFingerprint: fingerprint, - encryptionKeyBackupConfirmedByUserId: input.userId, - }, - update: { - encryptionKeyBackupConfirmedAt: now, - encryptionKeyBackupConfirmedKeyId: activeKeyId, - encryptionKeyBackupConfirmedFingerprint: fingerprint, - encryptionKeyBackupConfirmedByUserId: input.userId, - }, - }); - return { ok: true, status: await readKeyBackupStatus() }; -} diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 92cead915..371c9c70b 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -353,6 +353,12 @@ export const INSTANCE_SCOPED: Readonly> = { HostMetric: "instance-wide host telemetry, not attributable to an account", OffhostBackupKeyUse: "per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account", + RestoreDrillOutcome: + "the monthly restore drill's last verdict for the instance's off-host bucket; says nothing about any account", + RecoveryReadinessNotice: + "per backup signal, what the admins were last told; a fact about the instance's backups, not about any account", + RecoveryKitConfirmation: + "the instance's recovery kit confirmation (key ids and fingerprints); the confirming admin is a plain id, not a relation", OffhostPurgeRequest: "the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account", CycleSymptomCategory: diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index cf6fbf1ba..4c802b4df 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -23,12 +23,7 @@ */ import { envOr } from "@/lib/env"; import { Buffer } from "node:buffer"; -import { - createCipheriv, - createDecipheriv, - createHash, - randomBytes, -} from "node:crypto"; +import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; import { Readable, Transform } from "node:stream"; import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib"; import type { PrismaClient } from "@/generated/prisma/client"; @@ -44,6 +39,7 @@ import { import { annotate, getEvent } from "@/lib/logging/context"; import { envValue } from "@/lib/env"; import { OFFHOST_BACKUP_QUEUE } from "./offhost-backup-queue"; +import { decodeBackupKey, offhostKeyId } from "./offhost-key"; const ALGORITHM = "aes-256-gcm"; const IV_LENGTH = 12; @@ -74,17 +70,6 @@ export class OffhostBackupNotConfiguredError extends Error { } } -function decodeBackupKey(raw: string): Buffer { - if (/^[0-9a-fA-F]{64}$/.test(raw)) return Buffer.from(raw, "hex"); - if (/^[A-Za-z0-9+/=]+$/.test(raw)) { - const buf = Buffer.from(raw, "base64"); - if (buf.length === 32) return buf; - } - throw new Error( - "BACKUP_ENCRYPTION_KEY must be 64 hex chars or 32-byte base64", - ); -} - /** * Whether this host has the five variables the nightly job needs. * @@ -200,11 +185,6 @@ function asKeyRing(key: Buffer | OffhostKeyRing): OffhostKeyRing { return Buffer.isBuffer(key) ? { active: key, previous: [] } : key; } -/** The id a version-4 object records for the key it was written under. */ -export function offhostKeyId(key: Buffer): string { - return createHash("sha256").update(key).digest("hex").slice(0, 12); -} - function offhostAad(objectKey: string): Buffer { return Buffer.from(`${OFFHOST_AAD_PREFIX}${objectKey}`, "utf8"); } diff --git a/src/lib/jobs/offhost-key.ts b/src/lib/jobs/offhost-key.ts new file mode 100644 index 000000000..3e28700de --- /dev/null +++ b/src/lib/jobs/offhost-key.ts @@ -0,0 +1,28 @@ +/** + * The off-host backup key as a value: how the operator writes it and the id + * an object records for it. Kept apart from the uploader so the recovery kit + * can name the keys without loading the backup machinery. + */ +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; + +/** One off-host key as the operator writes it: 64 hex or 32-byte base64. */ +export function decodeBackupKey(raw: string): Buffer { + if (/^[0-9a-fA-F]{64}$/.test(raw)) return Buffer.from(raw, "hex"); + if (/^[A-Za-z0-9+/=]+$/.test(raw)) { + const buf = Buffer.from(raw, "base64"); + if (buf.length === 32) return buf; + } + throw new Error( + "BACKUP_ENCRYPTION_KEY must be 64 hex chars or 32-byte base64", + ); +} + +/** + * The id a version-4 object records for the key it was written under: the + * first twelve hex characters of the key's SHA-256. The recovery kit uses the + * same value as the key's fingerprint, so a kit entry matches an object header. + */ +export function offhostKeyId(key: Buffer): string { + return createHash("sha256").update(key).digest("hex").slice(0, 12); +} diff --git a/src/lib/observability/redact-payload.ts b/src/lib/observability/redact-payload.ts index 848ff09e3..d47a0aa00 100644 --- a/src/lib/observability/redact-payload.ts +++ b/src/lib/observability/redact-payload.ts @@ -96,11 +96,13 @@ export const SENSITIVE_KEY_PATTERNS: readonly RegExp[] = [ // `client_secret` are already covered by `/token/i` + `/secret/i`; the // verifier is the one secret-shaped key those patterns miss. /verifier/i, - // v1.40.0 — the encryption key an admin pastes into "Check my copy" - // (`POST /api/admin/encryption/key-backup/verify`). The route never logs its - // body; this keeps it out of any excerpt a future change might take. - // Anchored so `encryptionKeyId` and friends stay readable. + // v1.40.0 — the encryption key an admin pastes into "Check my copy", and the + // copy of any recovery kit key, sent as `keyCopy` + // (`POST /api/admin/recovery-kit/verify`). The route never logs its body; + // this keeps it out of any excerpt a future change might take. Anchored so + // `encryptionKeyId` and friends stay readable. /^encryption[_-]?key$/i, + /^key[_-]?copy$/i, ]; /** diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index eefaaf11d..58cc2786e 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -199,7 +199,7 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "encounter.reminder-conflict", "encounter.reminder-not-found", ], - encryption: ["encryption.keyBackup.stale", "encryption.key_mismatch"], + encryption: ["encryption.key_mismatch"], environment: [ "environment.backfill_pending", "environment.backfill_rate_limited", @@ -398,6 +398,7 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "profile_claim.taken", ], record_write: ["record_write.rate_limited"], + recovery: ["recovery.kit.stale"], "report-selection": [ "report-selection.body.invalid_json", "report-selection.body.invalid_shape", diff --git a/src/lib/query-keys/admin.ts b/src/lib/query-keys/admin.ts index 62d6efd73..8a3e28cf4 100644 --- a/src/lib/query-keys/admin.ts +++ b/src/lib/query-keys/admin.ts @@ -55,9 +55,10 @@ export const adminKeys = { ["admin", "maintenance", "measurements"] as const, /** v1.43 — the pulse day-mean method, its dry run and apply step. */ adminPulseDayMean: () => ["admin", "maintenance", "pulse-day-mean"] as const, - /** v1.40.0 — the "Back up your encryption key" step (admin + dashboard banner). */ - adminEncryptionKeyBackup: () => - ["admin", "encryption", "key-backup"] as const, + /** The recovery kit (admin Encryption section + dashboard banner). */ + adminRecoveryKit: () => ["admin", "recovery-kit"] as const, + /** Every backup signal's verdict (admin overview + Backups). */ + adminRecoveryReadiness: () => ["admin", "recovery-readiness"] as const, adminCoachFeedback: () => ["admin", "coach-feedback"] as const, adminFeedback: (status: string) => ["admin", "feedback", status] as const, adminFeedbackRoot: () => ["admin", "feedback"] as const, diff --git a/src/lib/recovery/__tests__/kit-rules.test.ts b/src/lib/recovery/__tests__/kit-rules.test.ts new file mode 100644 index 000000000..f05e4b624 --- /dev/null +++ b/src/lib/recovery/__tests__/kit-rules.test.ts @@ -0,0 +1,231 @@ +/** + * The recovery kit rules: which keys the kit lists, what a confirmation + * covers, and when the kit is due again. + * + * Mutation check: compare fingerprints out of the covered test (id only) and + * the re-keyed-install case goes red; drop the backup needs from the + * application list and the retired-key case goes red. + */ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; + +import { offhostKeyId } from "@/lib/jobs/offhost-key"; +import type { BackupKeyNeeds } from "@/lib/recovery/backup-key-needs"; +import { + applicationKitKeys, + assessRecoveryKit, + backupKitKeys, + fingerprintText, + neededKeyIds, + parseKitEntries, + resolvePlatformHint, + sameKit, + tokenKitKeys, + type KitEntry, +} from "@/lib/recovery/kit-rules"; + +const NOW = new Date("2026-10-25T12:00:00Z"); +const DAY = 86_400_000; + +const NO_NEEDS: BackupKeyNeeds = { + stored: [], + unrecorded: { copies: 0, oldestAt: null }, + offhost: [], + offhostExpirationDays: null, + retiredKeysStillNeeded: [], +}; + +const FINGERPRINTS: Record = { + v1: "111111111111", + v2: "222222222222", +}; + +function appKeys(configured: string[], active: string, needs = NO_NEEDS) { + return applicationKitKeys({ + activeKeyId: active, + configuredKeyIds: configured, + fingerprintOf: (id) => (configured.includes(id) ? FINGERPRINTS[id] : null), + needs, + now: NOW, + }); +} + +const BACKUP_HEX = "ab".repeat(32); +const BACKUP_OLD_HEX = "cd".repeat(32); + +describe("which keys the kit lists", () => { + it("lists every configured application key, the active one marked", () => { + const keys = appKeys(["v1", "v2"], "v2"); + expect(keys.map((k) => [k.id, k.need.active, k.fingerprint])).toEqual([ + ["v1", false, "111111111111"], + ["v2", true, "222222222222"], + ]); + }); + + it("keeps a retired key a stored copy lists, even once the server dropped it", () => { + const needs: BackupKeyNeeds = { + ...NO_NEEDS, + stored: [{ keyId: "v1", copies: 3, oldestAt: NOW.toISOString() }], + }; + const [v1] = appKeys(["v2"], "v2", needs); + expect(v1).toMatchObject({ + id: "v1", + fingerprint: null, + problem: "not_configured", + need: { configured: false, storedCopies: 3 }, + }); + }); + + it("keeps an off-host key while objects inside the retention need it", () => { + const row = (neededUntil: string | null) => ({ + ...NO_NEEDS, + offhost: [ + { + keyId: "v1", + firstWrittenAt: NOW.toISOString(), + lastWrittenAt: NOW.toISOString(), + neededUntil, + }, + ], + }); + const until = new Date(NOW.getTime() + 10 * DAY).toISOString(); + expect(appKeys(["v2"], "v2", row(until))[0].need.offhostUntil).toBe(until); + expect(appKeys(["v2"], "v2", row(null))[0].need.offhostUntil).toBe( + "unknown", + ); + // Past the retention the objects are gone, so the key drops out. + const past = new Date(NOW.getTime() - DAY).toISOString(); + expect(appKeys(["v2"], "v2", row(past)).map((k) => k.id)).toEqual(["v2"]); + expect([...neededKeyIds(row(past), NOW)]).toEqual([]); + }); + + it("names the off-host keys by the id their objects record", () => { + const keys = backupKitKeys(BACKUP_HEX, ` ${BACKUP_OLD_HEX} ,`); + expect(keys.map((k) => [k.id, k.need.active, k.source])).toEqual([ + [ + offhostKeyId(Buffer.from(BACKUP_HEX, "hex")), + true, + "BACKUP_ENCRYPTION_KEY", + ], + [ + offhostKeyId(Buffer.from(BACKUP_OLD_HEX, "hex")), + false, + "BACKUP_ENCRYPTION_PREVIOUS_KEYS", + ], + ]); + expect(keys.every((k) => k.fingerprint === k.id)).toBe(true); + expect(backupKitKeys(undefined, undefined)).toEqual([]); + }); + + it("flags an off-host key that does not decode instead of hiding it", () => { + const [bad] = backupKitKeys("not-a-key", undefined); + expect(bad).toMatchObject({ fingerprint: null, problem: "unreadable" }); + }); + + it("fingerprints the token key as the text the operator set", () => { + const value = "x".repeat(40); + const [token] = tokenKitKeys(value); + expect(token.fingerprint).toBe( + createHash("sha256").update(value).digest("hex").slice(0, 12), + ); + expect(token.fingerprint).toBe(fingerprintText(value)); + expect(JSON.stringify(token)).not.toContain(value); + }); +}); + +describe("what a confirmation covers", () => { + const listed = [...appKeys(["v1"], "v1"), ...tokenKitKeys("t".repeat(40))]; + const entries = (keys: typeof listed): KitEntry[] => + keys.map(({ kind, id, fingerprint }) => ({ kind, id, fingerprint })); + + it("is due without any confirmation, and done for the exact set", () => { + expect(assessRecoveryKit(listed, null).due).toBe(true); + expect(assessRecoveryKit(listed, entries(listed)).due).toBe(false); + }); + + it("keeps a confirmation of the active key only, from the old step, for that key", () => { + const legacy: KitEntry[] = [ + { kind: "application", id: "v1", fingerprint: "111111111111" }, + ]; + const verdict = assessRecoveryKit(listed, legacy); + expect(verdict.due).toBe(true); + expect(verdict.keys.find((k) => k.id === "v1")?.confirmed).toBe(true); + expect(verdict.unconfirmed.map((k) => k.id)).toEqual([ + "API_TOKEN_HMAC_KEY", + ]); + }); + + it("is due again after a rotation, a re-keyed install or a new backup key", () => { + const confirmed = entries(listed); + // Rotation: a new active id. + const rotated = [ + ...appKeys(["v1", "v2"], "v2"), + ...tokenKitKeys("t".repeat(40)), + ]; + expect( + assessRecoveryKit(rotated, confirmed).unconfirmed.map((k) => k.id), + ).toEqual(["v2"]); + // Same id, other key. + const rekeyed = listed.map((k) => + k.id === "v1" ? { ...k, fingerprint: "999999999999" } : k, + ); + expect(assessRecoveryKit(rekeyed, confirmed).due).toBe(true); + // A backup key appears. + const withBackup = [...listed, ...backupKitKeys(BACKUP_HEX, undefined)]; + expect(assessRecoveryKit(withBackup, confirmed).unconfirmed).toHaveLength( + 1, + ); + }); + + it("is not due when a key merely drops out of the kit", () => { + const wider = [ + ...appKeys(["v1", "v2"], "v2"), + ...tokenKitKeys("t".repeat(40)), + ]; + const narrower = [ + ...appKeys(["v2"], "v2"), + ...tokenKitKeys("t".repeat(40)), + ]; + expect(assessRecoveryKit(narrower, entries(wider)).due).toBe(false); + }); + + it("covers a key this server does not hold by its id", () => { + const needs: BackupKeyNeeds = { + ...NO_NEEDS, + stored: [{ keyId: "v0", copies: 1, oldestAt: NOW.toISOString() }], + }; + const kit = appKeys(["v1"], "v1", needs); + const confirmed = entries(kit); + expect(confirmed.find((e) => e.id === "v0")?.fingerprint).toBeNull(); + expect(assessRecoveryKit(kit, confirmed).due).toBe(false); + }); + + it("matches the shown kit against the current one as a set", () => { + const a = entries(listed); + expect(sameKit([...a].reverse(), a)).toBe(true); + expect(sameKit(a.slice(1), a)).toBe(false); + expect( + sameKit( + a.map((e) => + e.id === "v1" ? { ...e, fingerprint: "000000000000" } : e, + ), + a, + ), + ).toBe(false); + }); + + it("reads a stored confirmation it cannot parse as none", () => { + expect(parseKitEntries({ not: "a list" })).toEqual([]); + expect( + parseKitEntries([{ kind: "nope", id: "v1", fingerprint: null }]), + ).toEqual([]); + }); +}); + +describe("platform hint", () => { + it("opens on a known platform and falls back to compose", () => { + expect(resolvePlatformHint("TrueNAS")).toBe("truenas"); + expect(resolvePlatformHint("kubernetes")).toBe("compose"); + expect(resolvePlatformHint(undefined)).toBe("compose"); + }); +}); diff --git a/src/lib/recovery/backup-key-needs.ts b/src/lib/recovery/backup-key-needs.ts new file mode 100644 index 000000000..fa2940577 --- /dev/null +++ b/src/lib/recovery/backup-key-needs.ts @@ -0,0 +1,99 @@ +/** + * Which application keys the backups still need. + * + * Rotation re-encrypts every row in the database and re-seals every stored + * backup's envelope, but the content INSIDE a backup keeps the key it was + * written under: a restore writes it back verbatim. So "every row is on the + * active key" is not the same as "the old key can go". This answers the second + * question per key id, from what each copy recorded as it was written: the + * stored copies that need it and the oldest of them, and for the off-host + * bucket the last night an object needing it went in, plus how long the + * bucket's lifecycle rule keeps objects after that. + * + * Two readers ask it: the admin encryption view, which shows the table, and + * the recovery kit, which lists every key a restore needs. One definition, so + * the kit cannot drop a key the encryption view still calls needed. + */ +import { prisma } from "@/lib/db"; +import { + offhostBackupConfigured, + probeOffhostLifecycle, +} from "@/lib/jobs/offhost-backup"; +import { neededKeyIds } from "@/lib/recovery/kit-rules"; + +export interface BackupKeyNeeds { + stored: Array<{ keyId: string; copies: number; oldestAt: string }>; + /** Copies written before v1.39.3, which recorded no key ids. */ + unrecorded: { copies: number; oldestAt: string | null }; + offhost: Array<{ + keyId: string; + firstWrittenAt: string; + lastWrittenAt: string; + /** When the last object needing this key expires; null when unknown. */ + neededUntil: string | null; + }>; + /** The bucket's expiry, when it has one the credential may read. */ + offhostExpirationDays: number | null; + /** Retired key ids some backup still needs (never the active one). */ + retiredKeysStillNeeded: string[]; +} + +export async function readBackupKeyNeeds( + activeKeyId: string, + now: Date, +): Promise { + const [stored, unrecorded, offhostRows] = await Promise.all([ + prisma.$queryRaw< + Array<{ key_id: string; copies: number; oldest_at: Date }> + >` + SELECT k AS key_id, count(*)::int AS copies, min(created_at) AS oldest_at + FROM data_backups, unnest(inner_key_ids) AS k + WHERE inner_key_ids_recorded + GROUP BY k + ORDER BY k + `, + prisma.dataBackup.aggregate({ + where: { innerKeyIdsRecorded: false }, + _count: { _all: true }, + _min: { createdAt: true }, + }), + prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }), + ]); + const lifecycle = + offhostRows.length > 0 && offhostBackupConfigured() + ? await probeOffhostLifecycle() + : null; + const expirationDays = + lifecycle?.state === "configured" ? lifecycle.expirationDays : null; + + const offhost = offhostRows.map((row) => ({ + keyId: row.keyId, + firstWrittenAt: row.firstWrittenAt.toISOString(), + lastWrittenAt: row.lastWrittenAt.toISOString(), + neededUntil: + expirationDays === null + ? null + : new Date( + row.lastWrittenAt.getTime() + (expirationDays + 1) * 86_400_000, + ).toISOString(), + })); + + const needs: BackupKeyNeeds = { + stored: stored.map((row) => ({ + keyId: row.key_id, + copies: row.copies, + oldestAt: row.oldest_at.toISOString(), + })), + unrecorded: { + copies: unrecorded._count._all, + oldestAt: unrecorded._min.createdAt?.toISOString() ?? null, + }, + offhost, + offhostExpirationDays: expirationDays, + retiredKeysStillNeeded: [], + }; + const retired = neededKeyIds(needs, now); + retired.delete(activeKeyId); + needs.retiredKeysStillNeeded = [...retired].sort(); + return needs; +} diff --git a/src/lib/recovery/kit-rules.ts b/src/lib/recovery/kit-rules.ts new file mode 100644 index 000000000..e7edfd791 --- /dev/null +++ b/src/lib/recovery/kit-rules.ts @@ -0,0 +1,302 @@ +/** + * The recovery kit: every key a restore of this instance needs, by id and + * fingerprint, and the admin's confirmation that a copy of each is kept + * outside the server. + * + * It replaced a step that confirmed one key, the active application key. An + * operator could do everything that step asked and still be unable to read a + * single backup: the off-host copies are sealed under `BACKUP_ENCRYPTION_KEY`, + * a copy written before a rotation needs the retired application key it was + * written under, and every session, token and share link hangs off + * `API_TOKEN_HMAC_KEY`. The kit lists all of them: + * + * - `application`: every id in `ENCRYPTION_KEYS` this server holds, plus + * every retired id a stored copy or an off-host object inside the + * bucket's retention still lists (`backup-key-needs.ts`); + * - `backup`: `BACKUP_ENCRYPTION_KEY` and each key in + * `BACKUP_ENCRYPTION_PREVIOUS_KEYS`, when set; + * - `token`: `API_TOKEN_HMAC_KEY`. + * + * The confirmation records the set the admin was shown. The kit is due again + * as soon as a key it needs is not in that set: a rotation, a new backup key, + * a re-keyed install. A key that drops out of the kit (a retired key whose + * last copy expired) does not make it due, because nothing is missing. + * + * Fingerprints only, never key material: the first twelve hex characters of + * SHA-256 over the key, the same rule each key's own surface already uses (the + * application key's boot check, the off-host object header). The token key is + * hashed as the text the operator set, so `printf %s "$API_TOKEN_HMAC_KEY" | + * sha256sum` reproduces it. + */ +import { createHash } from "node:crypto"; +import { z } from "zod/v4"; + +import { envValue } from "@/lib/env"; +import { decodeBackupKey, offhostKeyId } from "@/lib/jobs/offhost-key"; +import type { BackupKeyNeeds } from "@/lib/recovery/backup-key-needs"; + +/** Where the operator sets the keys; picks the tab the kit opens on. */ +export const KIT_PLATFORMS = [ + "compose", + "truenas", + "coolify", + "unraid", + "portainer", +] as const; +export type KitPlatform = (typeof KIT_PLATFORMS)[number]; + +/** Wire code for a confirmation of a kit that has changed since it was shown. */ +export const RECOVERY_KIT_STALE_CODE = "recovery.kit.stale"; + +export const KIT_KEY_KINDS = ["application", "backup", "token"] as const; +export type KitKeyKind = (typeof KIT_KEY_KINDS)[number]; + +/** The variable each kind of key is set in. */ +export const KIT_KEY_SOURCES = { + application: "ENCRYPTION_KEYS", + backup: "BACKUP_ENCRYPTION_KEY", + backupPrevious: "BACKUP_ENCRYPTION_PREVIOUS_KEYS", + token: "API_TOKEN_HMAC_KEY", +} as const; +export type KitKeySource = + (typeof KIT_KEY_SOURCES)[keyof typeof KIT_KEY_SOURCES]; + +/** One key as a confirmation stores it. */ +export interface KitEntry { + kind: KitKeyKind; + id: string; + /** Null for a key this server does not hold but a copy still needs. */ + fingerprint: string | null; +} + +/** Why the kit lists a key. */ +export interface KitKeyNeed { + /** The key new data is written under. */ + active: boolean; + /** This server holds the key. */ + configured: boolean; + /** Stored copies in this database whose content needs it. */ + storedCopies: number; + /** + * Off-host objects need it until this instant; `"unknown"` when the bucket's + * retention cannot be read, so they may need it indefinitely. + */ + offhostUntil: string | null; +} + +/** One key the kit lists, with its verdict. */ +export interface KitKey extends KitEntry { + source: KitKeySource; + need: KitKeyNeed; + /** The confirmation covers this key. */ + confirmed: boolean; + /** + * `not_configured`: a copy needs it and this server does not hold it, so + * only a copy kept elsewhere can read those backups. `unreadable`: the + * variable is set but does not decode as a key. + */ + problem: "not_configured" | "unreadable" | null; +} + +export interface RecoveryKit { + keys: KitKey[]; + /** A key the kit lists is not covered by the confirmation. */ + due: boolean; + /** Keys the confirmation does not cover, by kind and id. */ + unconfirmed: Array>; + confirmedAt: string | null; + /** The confirming admin, while that account exists. */ + confirmedBy: { id: string; email: string | null } | null; + platformHint: KitPlatform; +} + +/** + * `HEALTHLOG_PLATFORM`, when it names a known platform; `compose` otherwise. + * Catalog templates set it so the kit opens on the right instructions. + */ +export function resolvePlatformHint( + raw: string | undefined = envValue("HEALTHLOG_PLATFORM"), +): KitPlatform { + const value = raw?.trim().toLowerCase(); + return (KIT_PLATFORMS as readonly string[]).includes(value ?? "") + ? (value as KitPlatform) + : "compose"; +} + +/** The fingerprint rule for a key that is text rather than decoded bytes. */ +export function fingerprintText(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex").slice(0, 12); +} + +const kitEntrySchema = z.object({ + kind: z.enum(KIT_KEY_KINDS), + id: z.string().min(1).max(64), + fingerprint: z + .string() + .regex(/^[0-9a-f]{12}$/) + .nullable(), +}); + +/** The request body of a confirmation: the kit as the admin saw it. */ +export const kitConfirmationSchema = z + .object({ entries: z.array(kitEntrySchema).min(1).max(64) }) + .strict(); + +/** + * The stored entries, validated. A row that does not parse confirms nothing, + * which leaves the kit due rather than trusting a shape it cannot read. + */ +export function parseKitEntries(raw: unknown): KitEntry[] { + const parsed = z.array(kitEntrySchema).safeParse(raw); + return parsed.success ? parsed.data : []; +} + +const entryKey = (e: Pick) => `${e.kind}:${e.id}`; + +/** + * Pure rule: which listed keys a confirmation covers. + * + * A key this server holds is covered by an entry with the same kind, id and + * fingerprint, so a re-keyed install under the same id is due again. A key it + * does not hold has no fingerprint to compare, so an entry with its kind and id + * covers it: the admin said a copy of that id is kept. + */ +export function assessRecoveryKit( + listed: ReadonlyArray>, + confirmation: readonly KitEntry[] | null, +): { keys: KitKey[]; due: boolean; unconfirmed: KitEntry[] } { + const exact = new Set( + (confirmation ?? []).map((e) => `${entryKey(e)}:${e.fingerprint ?? ""}`), + ); + const byId = new Set((confirmation ?? []).map(entryKey)); + const keys = listed.map((key) => ({ + ...key, + confirmed: + key.fingerprint === null + ? byId.has(entryKey(key)) + : exact.has(`${entryKey(key)}:${key.fingerprint}`), + })); + const unconfirmed = keys + .filter((key) => !key.confirmed) + .map(({ kind, id, fingerprint }) => ({ kind, id, fingerprint })); + return { keys, due: unconfirmed.length > 0, unconfirmed }; +} + +/** Pure rule: does what the admin was shown match what the kit lists now? */ +export function sameKit( + shown: readonly KitEntry[], + listed: readonly KitEntry[], +): boolean { + const canon = (entries: readonly KitEntry[]) => + [ + ...new Set(entries.map((e) => `${entryKey(e)}:${e.fingerprint ?? ""}`)), + ].sort(); + const a = canon(shown); + const b = canon(listed); + return a.length === b.length && a.every((value, i) => value === b[i]); +} + +/** + * Every key id a stored copy lists, and every key id an off-host object still + * inside the bucket's retention needs. An off-host key whose expiry cannot be + * read stays needed: without the lifecycle rule nothing proves the objects are + * gone. + */ +export function neededKeyIds(needs: BackupKeyNeeds, now: Date): Set { + const needed = new Set(); + for (const row of needs.stored) needed.add(row.keyId); + for (const row of needs.offhost) { + if (row.neededUntil === null || Date.parse(row.neededUntil) > +now) { + needed.add(row.keyId); + } + } + return needed; +} + +/** + * Pure: the application keys the kit lists, from the ids this server holds + * and what the backups still need. + */ +export function applicationKitKeys(args: { + activeKeyId: string; + configuredKeyIds: readonly string[]; + fingerprintOf: (id: string) => string | null; + needs: BackupKeyNeeds; + now: Date; +}): Array> { + const needed = neededKeyIds(args.needs, args.now); + const ids = new Set([...args.configuredKeyIds, ...needed]); + return [...ids].sort().map((id) => { + const fingerprint = args.fingerprintOf(id); + const offhost = args.needs.offhost.find((row) => row.keyId === id); + return { + kind: "application" as const, + id, + fingerprint, + source: KIT_KEY_SOURCES.application, + need: { + active: id === args.activeKeyId, + configured: fingerprint !== null, + storedCopies: + args.needs.stored.find((row) => row.keyId === id)?.copies ?? 0, + offhostUntil: + offhost && needed.has(id) ? (offhost.neededUntil ?? "unknown") : null, + }, + problem: fingerprint === null ? ("not_configured" as const) : null, + }; + }); +} + +function noNeed(active: boolean): KitKeyNeed { + return { active, configured: true, storedCopies: 0, offhostUntil: null }; +} + +/** Pure: the off-host keys, from the two variables as the operator set them. */ +export function backupKitKeys( + activeRaw: string | undefined, + previousRaw: string | undefined, +): Array> { + const out: Array> = []; + const add = (raw: string, source: KitKeySource, active: boolean) => { + let fingerprint: string | null = null; + try { + fingerprint = offhostKeyId(decodeBackupKey(raw)); + } catch { + fingerprint = null; + } + out.push({ + kind: "backup", + // The fingerprint is the id off-host objects record for their key, so + // the operator matches a kit entry to an object header directly. A key + // that does not decode has neither; it is named by where it is set. + id: fingerprint ?? `${source}#${out.length + 1}`, + fingerprint, + source, + need: noNeed(active), + problem: fingerprint === null ? "unreadable" : null, + }); + }; + const active = activeRaw?.trim(); + if (active) add(active, KIT_KEY_SOURCES.backup, true); + for (const raw of (previousRaw ?? "").split(",")) { + if (raw.trim()) add(raw.trim(), KIT_KEY_SOURCES.backupPrevious, false); + } + return out; +} + +/** Pure: the token key, which the app cannot run without. */ +export function tokenKitKeys( + raw: string | undefined, +): Array> { + const value = raw ?? ""; + return [ + { + kind: "token", + id: KIT_KEY_SOURCES.token, + fingerprint: value ? fingerprintText(value) : null, + source: KIT_KEY_SOURCES.token, + need: { ...noNeed(true), configured: value.length > 0 }, + problem: value ? null : "not_configured", + }, + ]; +} diff --git a/src/lib/recovery/kit.ts b/src/lib/recovery/kit.ts new file mode 100644 index 000000000..194b4c898 --- /dev/null +++ b/src/lib/recovery/kit.ts @@ -0,0 +1,173 @@ +/** + * The recovery kit on this server: which keys it lists right now, what the + * stored confirmation covers, recording a confirmation, and "Check my copy". + * The rules themselves are pure and live in `kit-rules.ts`. + */ +import { createHash, timingSafeEqual } from "node:crypto"; + +import { prisma } from "@/lib/db"; +import { envValue } from "@/lib/env"; +import { + candidateMatchesKey, + getActiveKeyId, + getConfiguredKeyIds, + getKeyFingerprint, +} from "@/lib/crypto"; +import { decodeBackupKey, offhostKeyId } from "@/lib/jobs/offhost-key"; +import { readBackupKeyNeeds } from "@/lib/recovery/backup-key-needs"; +import { + applicationKitKeys, + assessRecoveryKit, + backupKitKeys, + KIT_KEY_SOURCES, + parseKitEntries, + resolvePlatformHint, + sameKit, + tokenKitKeys, + type KitEntry, + type KitKey, + type KitKeyKind, + type RecoveryKit, +} from "@/lib/recovery/kit-rules"; + +/** Every key the kit lists on this server right now, without a verdict. */ +export async function listKitKeys( + now: Date = new Date(), +): Promise>> { + const activeKeyId = getActiveKeyId(); + const needs = await readBackupKeyNeeds(activeKeyId, now); + return [ + ...applicationKitKeys({ + activeKeyId, + configuredKeyIds: getConfiguredKeyIds(), + fingerprintOf: getKeyFingerprint, + needs, + now, + }), + ...backupKitKeys( + envValue("BACKUP_ENCRYPTION_KEY"), + envValue("BACKUP_ENCRYPTION_PREVIOUS_KEYS"), + ), + ...tokenKitKeys(envValue("API_TOKEN_HMAC_KEY")), + ]; +} + +export async function readRecoveryKit( + now: Date = new Date(), +): Promise { + const [listed, row] = await Promise.all([ + listKitKeys(now), + prisma.recoveryKitConfirmation.findUnique({ + where: { id: "singleton" }, + }), + ]); + const verdict = assessRecoveryKit( + listed, + row ? parseKitEntries(row.entries) : null, + ); + const confirmedBy = row?.confirmedByUserId + ? await prisma.user.findUnique({ + where: { id: row.confirmedByUserId }, + select: { id: true, email: true }, + }) + : null; + return { + keys: verdict.keys, + due: verdict.due, + unconfirmed: verdict.unconfirmed.map(({ kind, id }) => ({ kind, id })), + confirmedAt: row?.confirmedAt.toISOString() ?? null, + confirmedBy: confirmedBy ?? null, + platformHint: resolvePlatformHint(), + }; +} + +export type ConfirmRecoveryKitResult = + { ok: true; kit: RecoveryKit } | { ok: false; kit: RecoveryKit }; + +/** + * Record the confirmation, but only for the kit the admin was shown: a page + * opened before a rotation or a new backup key must not confirm a key the + * admin never saw. + */ +export async function confirmRecoveryKit(input: { + shown: readonly KitEntry[]; + userId: string; + now?: Date; +}): Promise { + const now = input.now ?? new Date(); + const listed = await listKitKeys(now); + const entries: KitEntry[] = listed.map(({ kind, id, fingerprint }) => ({ + kind, + id, + fingerprint, + })); + if (!sameKit(input.shown, entries)) { + return { ok: false, kit: await readRecoveryKit(now) }; + } + const data = { + confirmedAt: now, + confirmedByUserId: input.userId, + entries: entries.map(({ kind, id, fingerprint }) => ({ + kind, + id, + fingerprint, + })), + }; + await prisma.recoveryKitConfirmation.upsert({ + where: { id: "singleton" }, + create: { id: "singleton", ...data }, + update: data, + }); + return { ok: true, kit: await readRecoveryKit(now) }; +} + +/** + * "Check my copy": does `value` decode to the key the kit lists as + * `kind`/`id` on this server? Constant time on the compared bytes; a value + * that does not decode is simply not a match. + */ +export function verifyKitKey(input: { + kind: KitKeyKind; + id: string; + value: string; +}): boolean { + const value = input.value.trim(); + switch (input.kind) { + case "application": + return candidateMatchesKey(value, input.id); + case "backup": { + let candidate: Buffer; + try { + candidate = decodeBackupKey(value); + } catch { + return false; + } + const configured = [ + envValue("BACKUP_ENCRYPTION_KEY"), + ...(envValue("BACKUP_ENCRYPTION_PREVIOUS_KEYS") ?? "").split(","), + ] + .map((raw) => raw?.trim() ?? "") + .filter((raw) => raw.length > 0) + .flatMap((raw) => { + try { + return [decodeBackupKey(raw)]; + } catch { + return []; + } + }) + .find((key) => offhostKeyId(key) === input.id); + return ( + configured !== undefined && + configured.length === candidate.length && + timingSafeEqual(configured, candidate) + ); + } + case "token": { + const configured = envValue("API_TOKEN_HMAC_KEY"); + if (!configured || input.id !== KIT_KEY_SOURCES.token) return false; + // Digests, so the comparison is constant time whatever the lengths. + const digest = (s: string) => createHash("sha256").update(s).digest(); + return timingSafeEqual(digest(value), digest(configured)); + } + } +} diff --git a/tests/integration/__golden__/backup-parity/export-disaster-recovery.json b/tests/integration/__golden__/backup-parity/export-disaster-recovery.json index 5825e765a..f597f3954 100644 --- a/tests/integration/__golden__/backup-parity/export-disaster-recovery.json +++ b/tests/integration/__golden__/backup-parity/export-disaster-recovery.json @@ -54,10 +54,6 @@ }, "documentMaxFileBytes": 3, "documentQuotaBytes": "5", - "encryptionKeyBackupConfirmedAt": "2026-06-30T12:34:56.789Z", - "encryptionKeyBackupConfirmedKeyId": "rt-AppSettings-encryptionKeyBackupConfirmedKeyId", - "encryptionKeyBackupConfirmedFingerprint": "rt-AppSettings-encryptionKeyBackupConfirmedFingerprint", - "encryptionKeyBackupConfirmedByUserId": "rt-AppSettings-encryptionKeyBackupConfirmedByUserId", "correlationSeasonalEngineSince": "2026-06-30T12:34:56.789Z", "pulseDayMeanMethod": "hour_mean_v1", "pulseDayMeanAppliedAt": "2026-06-30T12:34:56.789Z", @@ -2504,10 +2500,6 @@ "aiUsageMaxLevel": "FRUGAL", "aiUsageDefaultLevel": "FRUGAL", "aiSyncRegenGapHours": 3, - "encryptionKeyBackupConfirmedAt": "2026-06-30T12:34:56.789Z", - "encryptionKeyBackupConfirmedKeyId": "rt-AppSettings-encryptionKeyBackupConfirmedKeyId", - "encryptionKeyBackupConfirmedFingerprint": "rt-AppSettings-encryptionKeyBackupConfirmedFingerprint", - "encryptionKeyBackupConfirmedByUserId": "rt-AppSettings-encryptionKeyBackupConfirmedByUserId", "correlationSeasonalEngineSince": "2026-06-30T12:34:56.789Z", "pulseDayMeanMethod": "hour_mean_v1", "pulseDayMeanAppliedAt": "2026-06-30T12:34:56.789Z", diff --git a/tests/integration/__golden__/backup-parity/restore-disaster-recovery.json b/tests/integration/__golden__/backup-parity/restore-disaster-recovery.json index ff5e45346..bc50cba10 100644 --- a/tests/integration/__golden__/backup-parity/restore-disaster-recovery.json +++ b/tests/integration/__golden__/backup-parity/restore-disaster-recovery.json @@ -2902,10 +2902,6 @@ }, "documentMaxFileBytes": 3, "documentQuotaBytes": "bigint:5", - "encryptionKeyBackupConfirmedAt": "2026-06-30T12:34:56.789Z", - "encryptionKeyBackupConfirmedKeyId": "rt-AppSettings-encryptionKeyBackupConfirmedKeyId", - "encryptionKeyBackupConfirmedFingerprint": "rt-AppSettings-encryptionKeyBackupConfirmedFingerprint", - "encryptionKeyBackupConfirmedByUserId": "rt-AppSettings-encryptionKeyBackupConfirmedByUserId", "correlationSeasonalEngineSince": "2026-06-30T12:34:56.789Z", "pulseDayMeanMethod": "hour_mean_v1", "pulseDayMeanAppliedAt": "2026-06-30T12:34:56.789Z", @@ -2970,10 +2966,6 @@ }, "documentMaxFileBytes": 3, "documentQuotaBytes": "5", - "encryptionKeyBackupConfirmedAt": "2026-06-30T12:34:56.789Z", - "encryptionKeyBackupConfirmedKeyId": "rt-AppSettings-encryptionKeyBackupConfirmedKeyId", - "encryptionKeyBackupConfirmedFingerprint": "rt-AppSettings-encryptionKeyBackupConfirmedFingerprint", - "encryptionKeyBackupConfirmedByUserId": "rt-AppSettings-encryptionKeyBackupConfirmedByUserId", "correlationSeasonalEngineSince": "2026-06-30T12:34:56.789Z", "pulseDayMeanMethod": "hour_mean_v1", "pulseDayMeanAppliedAt": "2026-06-30T12:34:56.789Z", diff --git a/tests/integration/__golden__/backup-parity/restore-portable.json b/tests/integration/__golden__/backup-parity/restore-portable.json index b73ea56a6..b66d44a9a 100644 --- a/tests/integration/__golden__/backup-parity/restore-portable.json +++ b/tests/integration/__golden__/backup-parity/restore-portable.json @@ -2396,10 +2396,6 @@ }, "documentMaxFileBytes": 3, "documentQuotaBytes": "bigint:5", - "encryptionKeyBackupConfirmedAt": "2026-06-30T12:34:56.789Z", - "encryptionKeyBackupConfirmedKeyId": "rt-AppSettings-encryptionKeyBackupConfirmedKeyId", - "encryptionKeyBackupConfirmedFingerprint": "rt-AppSettings-encryptionKeyBackupConfirmedFingerprint", - "encryptionKeyBackupConfirmedByUserId": "rt-AppSettings-encryptionKeyBackupConfirmedByUserId", "correlationSeasonalEngineSince": "2026-06-30T12:34:56.789Z", "pulseDayMeanMethod": "hour_mean_v1", "pulseDayMeanAppliedAt": "2026-06-30T12:34:56.789Z", diff --git a/tests/integration/__golden__/backup-parity/restore-skips.json b/tests/integration/__golden__/backup-parity/restore-skips.json index 901ab0a0f..2d983c8ca 100644 --- a/tests/integration/__golden__/backup-parity/restore-skips.json +++ b/tests/integration/__golden__/backup-parity/restore-skips.json @@ -1004,10 +1004,6 @@ }, "documentMaxFileBytes": 10000000, "documentQuotaBytes": "bigint:500000000", - "encryptionKeyBackupConfirmedAt": "2026-06-30T12:34:56.789Z", - "encryptionKeyBackupConfirmedKeyId": "rt-AppSettings-encryptionKeyBackupConfirmedKeyId", - "encryptionKeyBackupConfirmedFingerprint": "rt-AppSettings-encryptionKeyBackupConfirmedFingerprint", - "encryptionKeyBackupConfirmedByUserId": "rt-AppSettings-encryptionKeyBackupConfirmedByUserId", "correlationSeasonalEngineSince": "2026-06-30T12:34:56.789Z", "pulseDayMeanMethod": "hour_mean_v1", "pulseDayMeanAppliedAt": "2026-06-30T12:34:56.789Z", diff --git a/tests/integration/__golden__/backup-parity/restore-version-1.json b/tests/integration/__golden__/backup-parity/restore-version-1.json index bc26e8895..2d3534c66 100644 --- a/tests/integration/__golden__/backup-parity/restore-version-1.json +++ b/tests/integration/__golden__/backup-parity/restore-version-1.json @@ -1137,10 +1137,6 @@ }, "documentMaxFileBytes": 10000000, "documentQuotaBytes": "bigint:500000000", - "encryptionKeyBackupConfirmedAt": "2026-06-30T12:34:56.789Z", - "encryptionKeyBackupConfirmedKeyId": "rt-AppSettings-encryptionKeyBackupConfirmedKeyId", - "encryptionKeyBackupConfirmedFingerprint": "rt-AppSettings-encryptionKeyBackupConfirmedFingerprint", - "encryptionKeyBackupConfirmedByUserId": "rt-AppSettings-encryptionKeyBackupConfirmedByUserId", "correlationSeasonalEngineSince": "2026-06-30T12:34:56.789Z", "pulseDayMeanMethod": "hour_mean_v1", "pulseDayMeanAppliedAt": "2026-06-30T12:34:56.789Z", @@ -1203,10 +1199,6 @@ }, "documentMaxFileBytes": 10000000, "documentQuotaBytes": "500000000", - "encryptionKeyBackupConfirmedAt": "2026-06-30T12:34:56.789Z", - "encryptionKeyBackupConfirmedKeyId": "rt-AppSettings-encryptionKeyBackupConfirmedKeyId", - "encryptionKeyBackupConfirmedFingerprint": "rt-AppSettings-encryptionKeyBackupConfirmedFingerprint", - "encryptionKeyBackupConfirmedByUserId": "rt-AppSettings-encryptionKeyBackupConfirmedByUserId", "correlationSeasonalEngineSince": "2026-06-30T12:34:56.789Z", "pulseDayMeanMethod": "hour_mean_v1", "pulseDayMeanAppliedAt": "2026-06-30T12:34:56.789Z", diff --git a/tests/integration/__golden__/backup-parity/seeded-account.json b/tests/integration/__golden__/backup-parity/seeded-account.json index 1e9a3bc58..7b5206628 100644 --- a/tests/integration/__golden__/backup-parity/seeded-account.json +++ b/tests/integration/__golden__/backup-parity/seeded-account.json @@ -2515,10 +2515,6 @@ }, "documentMaxFileBytes": 3, "documentQuotaBytes": "bigint:5", - "encryptionKeyBackupConfirmedAt": "2026-06-30T12:34:56.789Z", - "encryptionKeyBackupConfirmedKeyId": "rt-AppSettings-encryptionKeyBackupConfirmedKeyId", - "encryptionKeyBackupConfirmedFingerprint": "rt-AppSettings-encryptionKeyBackupConfirmedFingerprint", - "encryptionKeyBackupConfirmedByUserId": "rt-AppSettings-encryptionKeyBackupConfirmedByUserId", "correlationSeasonalEngineSince": "2026-06-30T12:34:56.789Z", "pulseDayMeanMethod": "hour_mean_v1", "pulseDayMeanAppliedAt": "2026-06-30T12:34:56.789Z", diff --git a/tests/integration/encryption-key-check.test.ts b/tests/integration/encryption-key-check.test.ts index f7d1a8ea5..7e4700609 100644 --- a/tests/integration/encryption-key-check.test.ts +++ b/tests/integration/encryption-key-check.test.ts @@ -1,6 +1,5 @@ /** - * The boot encryption key check and the "Back up your encryption key" step, - * against real Postgres. + * The boot encryption key check, against real Postgres. * * - a first boot writes a canary; a second boot with the same key opens it; * - a boot with another key under the same id refuses: the state is set, @@ -9,9 +8,9 @@ * - a database written before the canary existed is probed: data that does * not open under the configured key refuses rather than sealing the wrong * key as the right one; - * - the confirmation binds to the active key id and fingerprint, refuses a - * stale one with 409, and is due again after a re-keyed install; - * - "Check my copy" answers a boolean, stores nothing, and is rate limited. + * + * The recovery kit that replaced the key backup step is pinned in + * `recovery-kit.test.ts`. */ import { NextRequest } from "next/server"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -83,7 +82,6 @@ import { _resetCryptoCacheForTests, encrypt, encryptUnderKeyId, - getKeyFingerprint, } from "@/lib/crypto"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { runBootKeyCheck } from "@/lib/boot/key-check"; @@ -145,7 +143,6 @@ beforeEach(async () => { const prisma = getPrismaClient(); await truncateAllTables(prisma); await prisma.$executeRaw`DELETE FROM encryption_key_canaries`; - await prisma.$executeRaw`DELETE FROM rate_limits WHERE key LIKE 'key-backup-verify:%'`; cookieJar.clear(); headerJar.clear(); useKey(KEY_A); @@ -185,10 +182,10 @@ describe("boot encryption key check (real Postgres)", () => { expect(await prisma.encryptionKeyCanary.findMany()).toEqual(before); await seedAdmin(); - const { GET: keyBackupGet } = - await import("@/app/api/admin/encryption/key-backup/route"); - const refused = await asRoute(keyBackupGet)( - new NextRequest("http://localhost/api/admin/encryption/key-backup"), + const { GET: recoveryKitGet } = + await import("@/app/api/admin/recovery-kit/route"); + const refused = await asRoute(recoveryKitGet)( + new NextRequest("http://localhost/api/admin/recovery-kit"), ); expect(refused.status).toBe(503); const body = await refused.json(); @@ -526,10 +523,10 @@ describe("ENCRYPTION_KEY_CHECK (real Postgres)", () => { // Requests are served, through apiHandler and outside it. await seedAdmin(); - const { GET: keyBackupGet } = - await import("@/app/api/admin/encryption/key-backup/route"); - const served = await asRoute(keyBackupGet)( - new NextRequest("http://localhost/api/admin/encryption/key-backup"), + const { GET: recoveryKitGet } = + await import("@/app/api/admin/recovery-kit/route"); + const served = await asRoute(recoveryKitGet)( + new NextRequest("http://localhost/api/admin/recovery-kit"), ); expect(served.status).toBe(200); const { POST: mcp } = await import("@/app/mcp/route"); @@ -707,106 +704,3 @@ describe("routes outside apiHandler refuse while the key does not match", () => } }); }); - -describe("encryption key backup step (real Postgres)", () => { - it("is due, refuses a stale confirmation, records the right one, and is due again after a re-key", async () => { - const prisma = getPrismaClient(); - const adminId = await seedAdmin(); - const { GET } = await import("@/app/api/admin/encryption/key-backup/route"); - const { POST: confirm } = - await import("@/app/api/admin/encryption/key-backup/confirm/route"); - const status = async () => - ( - await ( - await asRoute(GET)( - new NextRequest("http://localhost/api/admin/encryption/key-backup"), - ) - ).json() - ).data; - - const first = await status(); - expect(first.due).toBe(true); - expect(first.activeKeyId).toBe("v1"); - expect(first.fingerprint).toBe(getKeyFingerprint("v1")); - expect(first.platformHint).toBe("compose"); - expect(JSON.stringify(first)).not.toContain(KEY_A); - - const stale = await confirm( - post("/api/admin/encryption/key-backup/confirm", { - keyId: "v1", - fingerprint: "000000000000", - }), - ); - expect(stale.status).toBe(409); - expect((await stale.json()).meta.errorCode).toBe( - "encryption.keyBackup.stale", - ); - expect( - (await prisma.appSettings.findUnique({ where: { id: "singleton" } })) - ?.encryptionKeyBackupConfirmedAt ?? null, - ).toBeNull(); - - const ok = await confirm( - post("/api/admin/encryption/key-backup/confirm", { - keyId: "v1", - fingerprint: first.fingerprint, - }), - ); - expect(ok.status).toBe(200); - const confirmed = (await ok.json()).data; - expect(confirmed.due).toBe(false); - expect(confirmed.confirmedBy).toEqual({ - id: adminId, - email: "key-admin@example.test", - }); - const audit = await prisma.auditLog.findFirst({ - where: { action: "encryption.keyBackup.confirmed" }, - }); - expect(audit).not.toBeNull(); - - vi.stubEnv("HEALTHLOG_PLATFORM", "truenas"); - useKey(KEY_B); - const after = await status(); - expect(after.due).toBe(true); - expect(after.confirmedKeyId).toBe("v1"); - expect(after.platformHint).toBe("truenas"); - }); - - it("checks a copy without storing it, and limits the checks", async () => { - const prisma = getPrismaClient(); - await seedAdmin(); - const { POST: verify } = - await import("@/app/api/admin/encryption/key-backup/verify/route"); - const check = (encryptionKey: string) => - verify( - post("/api/admin/encryption/key-backup/verify", { encryptionKey }), - ); - - const good = await check(`${KEY_A}\n`); - expect(good.status).toBe(200); - expect((await good.json()).data).toEqual({ matches: true, keyId: "v1" }); - - const bad = await check(KEY_B); - expect((await bad.json()).data.matches).toBe(false); - - const invalid = await verify( - post("/api/admin/encryption/key-backup/verify", { encryptionKey: "" }), - ); - expect(invalid.status).toBe(422); - expect(JSON.stringify(await invalid.json())).not.toContain("issues"); - - // Nothing of the candidate is stored anywhere a row could hold it. - const audits = await prisma.auditLog.findMany(); - expect(JSON.stringify(audits)).not.toContain(KEY_A); - expect(JSON.stringify(audits)).not.toContain(KEY_B); - expect( - (await prisma.appSettings.findUnique({ where: { id: "singleton" } })) - ?.encryptionKeyBackupConfirmedAt ?? null, - ).toBeNull(); - - await check(KEY_B); - await check(KEY_B); - const limited = await check(KEY_A); - expect(limited.status).toBe(429); - }); -}); diff --git a/tests/integration/recovery-kit.test.ts b/tests/integration/recovery-kit.test.ts new file mode 100644 index 000000000..d100f1d4e --- /dev/null +++ b/tests/integration/recovery-kit.test.ts @@ -0,0 +1,359 @@ +/** + * The recovery kit against real Postgres, through its routes. + * + * - the kit lists the application key, the token key and, when set, the + * off-host backup key, by id and fingerprint and never key material; + * - a confirmation of a kit the admin was not shown is refused with 409 + * and records nothing; the right one records the set and is audited; + * - a rotation makes the kit due again, and the retired key stays listed, + * named, for as long as a stored copy lists it, also once the server no + * longer holds it; when the last such copy is gone it drops out; + * - the readiness model names a needed key nobody confirmed by its id; + * - "Check my copy" answers a boolean for each kind of key, stores + * nothing, and is rate limited; + * - migration 0487 carries a confirmation given under the old one-key step + * forward as a kit holding that key. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { NextRequest } from "next/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +import { _resetCryptoCacheForTests, getKeyFingerprint } from "@/lib/crypto"; +import { offhostKeyId } from "@/lib/jobs/offhost-key"; +import type { RecoveryKit } from "@/lib/recovery/kit-rules"; + +const KEY_A = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const KEY_B = + "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"; +const BACKUP_KEY = "ab".repeat(32); +const DAY = 86_400_000; + +function useKeys(keys: Record, active: string) { + vi.stubEnv("ENCRYPTION_KEY", ""); + vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify(keys)); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", active); + _resetCryptoCacheForTests(); +} + +async function seedAdmin(createdAt = new Date()): Promise { + const prisma = getPrismaClient(); + const admin = await prisma.user.create({ + data: { + username: "kit-admin", + email: "kit-admin@example.test", + role: "ADMIN", + createdAt, + }, + }); + const session = await prisma.session.create({ + data: { userId: admin.id, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return admin.id; +} + +type RouteFn = (request: NextRequest) => Promise; +const asRoute = (fn: unknown) => fn as RouteFn; + +function post(path: string, body: unknown): NextRequest { + return new NextRequest(`http://localhost${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + +async function readKit(): Promise { + const { GET } = await import("@/app/api/admin/recovery-kit/route"); + const res = await asRoute(GET)( + new NextRequest("http://localhost/api/admin/recovery-kit"), + ); + expect(res.status).toBe(200); + return (await res.json()).data; +} + +const shown = (kit: RecoveryKit) => ({ + entries: kit.keys.map(({ kind, id, fingerprint }) => ({ + kind, + id, + fingerprint, + })), +}); + +async function confirm(body: unknown): Promise { + const { POST } = await import("@/app/api/admin/recovery-kit/confirm/route"); + return POST(post("/api/admin/recovery-kit/confirm", body)); +} + +async function storedCopyListing(userId: string, keyIds: string[]) { + await getPrismaClient().dataBackup.create({ + data: { + userId, + type: "WEEKLY_AUTO", + data: "sealed", + innerKeyIds: keyIds, + innerKeyIdsRecorded: true, + }, + }); +} + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + await prisma.$executeRaw`DELETE FROM rate_limits WHERE key LIKE 'recovery-kit-verify:%'`; + cookieJar.clear(); + headerJar.clear(); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", ""); + vi.stubEnv("BACKUP_ENCRYPTION_PREVIOUS_KEYS", ""); + vi.stubEnv("BACKUP_S3_ENDPOINT", ""); + useKeys({ v1: KEY_A }, "v1"); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + _resetCryptoCacheForTests(); +}); + +describe("recovery kit (real Postgres)", () => { + it("lists every key by id and fingerprint, refuses a stale confirmation, and records the right one", async () => { + const prisma = getPrismaClient(); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", BACKUP_KEY); + vi.stubEnv("HEALTHLOG_PLATFORM", "truenas"); + const adminId = await seedAdmin(); + + const kit = await readKit(); + expect(kit.due).toBe(true); + expect(kit.platformHint).toBe("truenas"); + expect(kit.keys.map((k) => [k.kind, k.id, k.fingerprint])).toEqual([ + ["application", "v1", getKeyFingerprint("v1")], + [ + "backup", + offhostKeyId(Buffer.from(BACKUP_KEY, "hex")), + offhostKeyId(Buffer.from(BACKUP_KEY, "hex")), + ], + ["token", "API_TOKEN_HMAC_KEY", expect.stringMatching(/^[0-9a-f]{12}$/)], + ]); + const wire = JSON.stringify(kit); + expect(wire).not.toContain(KEY_A); + expect(wire).not.toContain(BACKUP_KEY); + expect(wire).not.toContain(process.env.API_TOKEN_HMAC_KEY ?? ""); + + const tampered = shown(kit); + tampered.entries[0] = { + ...tampered.entries[0], + fingerprint: "000000000000", + }; + const stale = await confirm(tampered); + expect(stale.status).toBe(409); + expect((await stale.json()).meta.errorCode).toBe("recovery.kit.stale"); + expect(await prisma.recoveryKitConfirmation.count()).toBe(0); + + const ok = await confirm(shown(kit)); + expect(ok.status).toBe(200); + const confirmed: RecoveryKit = (await ok.json()).data; + expect(confirmed.due).toBe(false); + expect(confirmed.confirmedBy).toEqual({ + id: adminId, + email: "kit-admin@example.test", + }); + const audit = await prisma.auditLog.findFirst({ + where: { action: "recovery.kit.confirmed" }, + }); + expect(audit).not.toBeNull(); + expect(JSON.stringify(audit)).not.toContain(KEY_A); + }); + + it("is due again after a rotation and keeps naming the retired key while a copy lists it", async () => { + const adminId = await seedAdmin(); + await storedCopyListing(adminId, ["v1"]); + expect((await confirm(shown(await readKit()))).status).toBe(200); + + // Rotate: v2 becomes active, v1 stays configured for the old data. + useKeys({ v1: KEY_A, v2: KEY_B }, "v2"); + const rotated = await readKit(); + expect(rotated.due).toBe(true); + expect(rotated.unconfirmed).toEqual([{ kind: "application", id: "v2" }]); + expect(rotated.keys.find((k) => k.id === "v1")).toMatchObject({ + confirmed: true, + need: { active: false, storedCopies: 1 }, + }); + expect((await confirm(shown(rotated))).status).toBe(200); + expect((await readKit()).due).toBe(false); + + // The operator drops v1 from the server while the copy still lists it: + // the kit keeps naming it, covered by the earlier confirmation of v1. + useKeys({ v2: KEY_B }, "v2"); + const dropped = await readKit(); + expect(dropped.keys.find((k) => k.id === "v1")).toMatchObject({ + fingerprint: null, + problem: "not_configured", + confirmed: true, + }); + expect(dropped.due).toBe(false); + + // Once no copy lists v1, it leaves the kit. + await getPrismaClient().dataBackup.deleteMany({}); + expect((await readKit()).keys.map((k) => k.id)).not.toContain("v1"); + }); + + it("names a needed key nobody confirmed in the readiness verdict", async () => { + const adminId = await seedAdmin(new Date(Date.now() - 30 * DAY)); + useKeys({ v1: KEY_A, v2: KEY_B }, "v2"); + await storedCopyListing(adminId, ["v1"]); + // A confirmation that covers only the new key and the token key. + const kit = await readKit(); + await getPrismaClient().recoveryKitConfirmation.create({ + data: { + confirmedAt: new Date(), + entries: shown(kit).entries.filter((e) => e.id !== "v1"), + }, + }); + + const { readRecoveryReadiness } = await import("@/lib/recovery/readiness"); + const readiness = await readRecoveryReadiness(); + expect( + readiness.signals.find((s) => s.id === "recovery_kit"), + ).toMatchObject({ + state: "attention", + reason: "out_of_date", + params: { keys: "v1", count: 1 }, + }); + + expect((await confirm(shown(kit))).status).toBe(200); + const after = await readRecoveryReadiness(); + expect(after.signals.find((s) => s.id === "recovery_kit")?.state).toBe( + "ok", + ); + }); + + it("checks a copy of each kind of key without storing it, and limits the checks", async () => { + const prisma = getPrismaClient(); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", BACKUP_KEY); + await seedAdmin(); + const { POST: verify } = + await import("@/app/api/admin/recovery-kit/verify/route"); + const check = (kind: string, id: string, keyCopy: string) => + verify(post("/api/admin/recovery-kit/verify", { kind, id, keyCopy })); + const backupId = offhostKeyId(Buffer.from(BACKUP_KEY, "hex")); + const token = process.env.API_TOKEN_HMAC_KEY ?? ""; + expect(token.length).toBeGreaterThan(0); + + const good = await check("application", "v1", `${KEY_A}\n`); + expect(good.status).toBe(200); + expect((await good.json()).data).toEqual({ + matches: true, + kind: "application", + id: "v1", + }); + expect( + (await (await check("backup", backupId, BACKUP_KEY)).json()).data.matches, + ).toBe(true); + expect( + (await (await check("token", "API_TOKEN_HMAC_KEY", token)).json()).data + .matches, + ).toBe(true); + expect( + (await (await check("application", "v1", KEY_B)).json()).data.matches, + ).toBe(false); + + const invalid = await verify( + post("/api/admin/recovery-kit/verify", { kind: "application", id: "v1" }), + ); + expect(invalid.status).toBe(422); + expect(JSON.stringify(await invalid.json())).not.toContain("issues"); + + const audits = JSON.stringify(await prisma.auditLog.findMany()); + for (const secret of [KEY_A, KEY_B, BACKUP_KEY, token]) { + expect(audits).not.toContain(secret); + } + expect(await prisma.recoveryKitConfirmation.count()).toBe(0); + + const limited = await check("application", "v1", KEY_A); + expect(limited.status).toBe(429); + }); +}); + +describe("migration 0487", () => { + it("carries a confirmation of the active key forward as a kit holding that key", async () => { + const prisma = getPrismaClient(); + const adminId = await seedAdmin(); + const fingerprint = getKeyFingerprint("v1"); + // The shape before 0487: the four columns on the settings row. + await prisma.$executeRawUnsafe(` + ALTER TABLE app_settings + ADD COLUMN IF NOT EXISTS encryption_key_backup_confirmed_at TIMESTAMP(3), + ADD COLUMN IF NOT EXISTS encryption_key_backup_confirmed_key_id TEXT, + ADD COLUMN IF NOT EXISTS encryption_key_backup_confirmed_fingerprint TEXT, + ADD COLUMN IF NOT EXISTS encryption_key_backup_confirmed_by_user_id TEXT + `); + await prisma.$executeRaw` + INSERT INTO app_settings (id, encryption_key_backup_confirmed_at, + encryption_key_backup_confirmed_key_id, + encryption_key_backup_confirmed_fingerprint, + encryption_key_backup_confirmed_by_user_id) + VALUES ('singleton', NOW(), 'v1', ${fingerprint}, ${adminId}) + ON CONFLICT (id) DO UPDATE SET + encryption_key_backup_confirmed_at = NOW(), + encryption_key_backup_confirmed_key_id = 'v1', + encryption_key_backup_confirmed_fingerprint = ${fingerprint}, + encryption_key_backup_confirmed_by_user_id = ${adminId} + `; + + const sql = readFileSync( + join( + process.cwd(), + "prisma/migrations/0487_recovery_readiness/migration.sql", + ), + "utf8", + ); + await prisma.$executeRawUnsafe(sql); + // Running it again changes nothing. + await prisma.$executeRawUnsafe(sql); + + const columns = await prisma.$queryRaw>` + SELECT column_name FROM information_schema.columns + WHERE table_name = 'app_settings' + AND column_name LIKE 'encryption_key_backup%' + `; + expect(columns).toEqual([]); + + const kit = await readKit(); + expect(kit.confirmedBy?.id).toBe(adminId); + expect(kit.keys.find((k) => k.id === "v1")?.confirmed).toBe(true); + // The old step never covered the token key, so the kit asks for it. + expect(kit.unconfirmed).toEqual([ + { kind: "token", id: "API_TOKEN_HMAC_KEY" }, + ]); + }); +}); From 7dab9cc5e98b5ccb78e7a80662757547c0cdfa53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 13:32:43 +0100 Subject: [PATCH 17/44] feat(admin): judge every backup signal and push changes to admins Every backup failure was a pull, on a page nobody opens until the day it matters: the weekly copy stopped for 46 days and showed an ordinary row, a failed monthly drill was on the failing-jobs card three days in thirty (job-failures.ts reads 72 hours), and an off-host bucket that was never set up showed only on the backups page. One readiness model (src/lib/recovery/readiness-model.ts) judges six signals as ok, attention or failing, with a reason code, the time it is about and where to fix it: - database dump: failing once the newest good dump is older than 36 h while dumps are on, or when it can be kept nowhere; read through a provider that reports "not available" until the dump lands; - weekly account copies: worst account decides, attention past 9 days, failing past 14; an account without a copy ages from its creation; - off-host copies: attention when any account is two nights behind; - restore drill: failing until a later drill passes, with no time window; the drill now keeps its verdict in restore_drill_outcomes; - off-host setup: attention 7 days after the first admin exists; - recovery kit: attention when a needed key is not confirmed, by id, 7 days after setup or as soon as a confirmed kit is out of date. A "Recovery readiness" card shows them on the admin overview and the backups page. An hourly job (recovery-readiness, :33) sends each change to every admin through the localised admin path, under a new admin-only BACKUP_HEALTH event that the channel preferences route: once when a signal turns to attention or failing, once a day while it stays failing, once when it recovers. Each notice is claimed with a compare-and-set before it is sent, so two workers never send the same one, and nothing is claimed while no admin exists. The backups page calls the weekly copy stale at the same instant as the model (past nine days rather than past ten). --- docs/api/openapi.yaml | 54 +-- messages/de.json | 98 ++++ messages/en.json | 98 ++++ messages/es.json | 98 ++++ messages/fr.json | 98 ++++ messages/it.json | 98 ++++ messages/ko.json | 98 ++++ messages/nl.json | 98 ++++ messages/pl.json | 98 ++++ messages/pt-BR.json | 98 ++++ .../success-affordance-guard.test.ts | 7 + src/app/admin/[section]/renderer.tsx | 7 +- src/app/admin/page.tsx | 4 + src/app/api/admin/recovery-readiness/route.ts | 24 + .../api/notifications/preferences/route.ts | 9 +- .../backups-restore-preview.test.tsx | 2 +- .../__tests__/backups-section-empty.test.tsx | 2 +- .../recovery-readiness-card.test.tsx | 154 +++++++ .../admin/recovery-readiness-card.tsx | 208 +++++++++ .../admin/use-recovery-readiness.ts | 24 + .../__tests__/backup-schedule-status.test.ts | 13 + src/lib/jobs/__tests__/restore-drill.test.ts | 24 + src/lib/jobs/backup-schedule-status.ts | 12 +- src/lib/jobs/job-outcome.ts | 7 + src/lib/jobs/queue-runtime.ts | 3 + src/lib/jobs/recovery-readiness-queue.ts | 12 + src/lib/jobs/recovery-readiness.ts | 42 ++ src/lib/jobs/registry/maintenance.ts | 15 + src/lib/jobs/restore-drill-outcome.ts | 54 +++ src/lib/jobs/restore-drill.ts | 42 +- src/lib/jobs/subsystem-surface.ts | 1 + src/lib/notifications/tap-target.ts | 1 + src/lib/notifications/types.ts | 20 + .../__tests__/readiness-model.test.ts | 424 ++++++++++++++++++ .../__tests__/readiness-notify.test.ts | 287 ++++++++++++ src/lib/recovery/database-dump-health.ts | 50 +++ src/lib/recovery/readiness-model.ts | 422 +++++++++++++++++ src/lib/recovery/readiness-notify.ts | 205 +++++++++ src/lib/recovery/readiness.ts | 117 +++++ .../recovery-readiness-notify.test.ts | 295 ++++++++++++ 40 files changed, 3386 insertions(+), 37 deletions(-) create mode 100644 src/app/api/admin/recovery-readiness/route.ts create mode 100644 src/components/admin/__tests__/recovery-readiness-card.test.tsx create mode 100644 src/components/admin/recovery-readiness-card.tsx create mode 100644 src/components/admin/use-recovery-readiness.ts create mode 100644 src/lib/jobs/recovery-readiness-queue.ts create mode 100644 src/lib/jobs/recovery-readiness.ts create mode 100644 src/lib/jobs/restore-drill-outcome.ts create mode 100644 src/lib/recovery/__tests__/readiness-model.test.ts create mode 100644 src/lib/recovery/__tests__/readiness-notify.test.ts create mode 100644 src/lib/recovery/database-dump-health.ts create mode 100644 src/lib/recovery/readiness-model.ts create mode 100644 src/lib/recovery/readiness-notify.ts create mode 100644 src/lib/recovery/readiness.ts create mode 100644 tests/integration/recovery-readiness-notify.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 9255e81e3..70c23f6ee 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -37138,6 +37138,7 @@ components: - SECURITY_ALERT - DAILY_BRIEFING - SHARED_RECORD_INTAKE + - BACKUP_HEALTH enabled: type: boolean required: @@ -38519,7 +38520,7 @@ components: documents.sources.tokenRequired, documents.sources.unavailable, documents.sources.unreachable, documents.sources.versionTooOld; encounter — encounter.invalid, encounter.link.invalid, encounter.practitioner-not-found, encounter.reminder-conflict, encounter.reminder-not-found; encryption - — encryption.keyBackup.stale, encryption.key_mismatch; environment — environment.backfill_pending, + — encryption.key_mismatch; environment — environment.backfill_pending, environment.backfill_rate_limited, environment.invalid, environment.no_home, environment.range_too_large, environment.travel.not_found, environment.travel_suggestion.not_found; export — export.selection.empty, export.selection.unknown_leaf; familyHistory — familyHistory.invalid; @@ -38580,30 +38581,30 @@ components: profile.update.emailInUse, profile.update.emailRateLimited, profile.update.emailSecondFactorRequired, profile.update.invalidBody, profile.update.nothingSaved; profile_claim — profile_claim.invalid, profile_claim.oidc_only_unsupported, profile_claim.sso_unavailable, profile_claim.taken; record_write — - record_write.rate_limited; report-selection — report-selection.body.invalid_json, - report-selection.body.invalid_shape, report-selection.leaves.unknown; route — route.retired; search — - search.invalid; security_activity — security_activity.invalid; share-link — - share-link.document.decryptFailed, share-link.document.notFound, share-link.document.rateLimited, - share-link.documents.invalid, share-link.report.notFound, share-link.report.rateLimited, - share-link.selection.forbidden_leaf, share-link.selection.unknown_leaf; sharing — - sharing.accept.expired, sharing.accept.not_found, sharing.accept.not_pending, sharing.accept.revoked, - sharing.access.denied, sharing.invite.duplicate, sharing.invite.invalid, sharing.invite.invalid_scope, - sharing.invite.manage_browser_only, sharing.invite.no_account, sharing.invite.refused, - sharing.invite.self, sharing.not_permitted, sharing.renounce.already_ended, sharing.renounce.not_found, - sharing.revoke.already_ended, sharing.revoke.not_found, sharing.session.changed, sharing.switch.invalid, - sharing.switch.wrong_transport; source-priority — source-priority.body.invalid_json, - source-priority.body.invalid_shape; symptoms — symptoms.definition.limitReached, - symptoms.definition.notFound, symptoms.episode.notOpen, symptoms.event.notFound, symptoms.invalid; - timeline — timeline.invalid; tokens — tokens.documents.ceiling_reached, - tokens.measurements.ceiling_reached, tokens.mirror.not_measurement_token, - tokens.workouts.ceiling_reached; vaccination — vaccination.booster-invalid, - vaccination.booster-no-antigen, vaccination.booster-out-of-scope, vaccination.custom-vaccine-not-found, - vaccination.custom.invalid, vaccination.custom.name-taken, vaccination.custom.not-found, - vaccination.encounter-not-found, vaccination.identity-required, vaccination.invalid, - vaccination.link.invalid, vaccination.not-found, vaccination.practitioner-not-found; workout — - workout.batch.invalid, workout.batch.payload_too_large, workout.batch.source_not_permitted, - workout.batch.too_large, workout.delete.synced_source; (unprefixed) — about_me_conflict, - ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file, + record_write.rate_limited; recovery — recovery.kit.stale; report-selection — + report-selection.body.invalid_json, report-selection.body.invalid_shape, + report-selection.leaves.unknown; route — route.retired; search — search.invalid; security_activity — + security_activity.invalid; share-link — share-link.document.decryptFailed, share-link.document.notFound, + share-link.document.rateLimited, share-link.documents.invalid, share-link.report.notFound, + share-link.report.rateLimited, share-link.selection.forbidden_leaf, share-link.selection.unknown_leaf; + sharing — sharing.accept.expired, sharing.accept.not_found, sharing.accept.not_pending, + sharing.accept.revoked, sharing.access.denied, sharing.invite.duplicate, sharing.invite.invalid, + sharing.invite.invalid_scope, sharing.invite.manage_browser_only, sharing.invite.no_account, + sharing.invite.refused, sharing.invite.self, sharing.not_permitted, sharing.renounce.already_ended, + sharing.renounce.not_found, sharing.revoke.already_ended, sharing.revoke.not_found, + sharing.session.changed, sharing.switch.invalid, sharing.switch.wrong_transport; source-priority — + source-priority.body.invalid_json, source-priority.body.invalid_shape; symptoms — + symptoms.definition.limitReached, symptoms.definition.notFound, symptoms.episode.notOpen, + symptoms.event.notFound, symptoms.invalid; timeline — timeline.invalid; tokens — + tokens.documents.ceiling_reached, tokens.measurements.ceiling_reached, + tokens.mirror.not_measurement_token, tokens.workouts.ceiling_reached; vaccination — + vaccination.booster-invalid, vaccination.booster-no-antigen, vaccination.booster-out-of-scope, + vaccination.custom-vaccine-not-found, vaccination.custom.invalid, vaccination.custom.name-taken, + vaccination.custom.not-found, vaccination.encounter-not-found, vaccination.identity-required, + vaccination.invalid, vaccination.link.invalid, vaccination.not-found, + vaccination.practitioner-not-found; workout — workout.batch.invalid, workout.batch.payload_too_large, + workout.batch.source_not_permitted, workout.batch.too_large, workout.delete.synced_source; (unprefixed) + — about_me_conflict, ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file, health_score_config_conflict, insights_layout_conflict, invalid_base_updated_at, json_carries_no_intake_time, json_not_an_array, medication_layout_conflict, missing_required_columns, modules_conflict, mood_tag_layout_conflict, not_configured, notification_prefs_conflict, oidc_only, @@ -87952,6 +87953,7 @@ components: - SECURITY_ALERT - DAILY_BRIEFING - SHARED_RECORD_INTAKE + - BACKUP_HEALTH description: Notification category a per-channel preference can toggle. additionalProperties: type: boolean @@ -87973,6 +87975,7 @@ components: - SECURITY_ALERT - DAILY_BRIEFING - SHARED_RECORD_INTAKE + - BACKUP_HEALTH description: What each event does on a channel with no stored row, resolved for this account by the function delivery uses. Most are constant per category; the mood reminder follows the account's mood-reminder setting (`moodReminderEnabled` on `/api/user/profile`). Render a switch without a row from this map rather than from @@ -88043,6 +88046,7 @@ components: - SECURITY_ALERT - DAILY_BRIEFING - SHARED_RECORD_INTAKE + - BACKUP_HEALTH description: Notification category a per-channel preference can toggle. enabled: type: boolean diff --git a/messages/de.json b/messages/de.json index 11ced69c4..5bbe197e2 100644 --- a/messages/de.json +++ b/messages/de.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Bestätigt am {date}.", "confirmedAtBy": "Bestätigt am {date} von {who}.", "dueAgain": "Das Kit wurde am {date} bestätigt, führt jetzt aber Schlüssel auf, die diese Bestätigung nicht abdeckt: {keys}." + }, + "recoveryReadiness": { + "title": "Wiederherstellbarkeit", + "description": "Ob sich dieser Server nach einem Ausfall wiederherstellen ließe, beurteilt anhand jeder Sicherung, die er anlegt.", + "loading": "Sicherungen werden geprüft…", + "loadError": "Die Wiederherstellbarkeit konnte nicht geladen werden.", + "state": { + "ok": "Bereit", + "attention": "Braucht Aufmerksamkeit", + "failing": "Gefährdet" + }, + "signal": { + "database_dump": "Datenbank-Dump", + "weekly_copy": "Wöchentliche Kontokopien", + "offhost_copies": "Externe Kopien", + "restore_drill": "Wiederherstellungsprobe", + "offhost_setup": "Externe Sicherung", + "recovery_kit": "Wiederherstellungs-Kit" + }, + "at": { + "database_dump": "Neuester Dump: {date}", + "weekly_copy": "Älteste Kontokopie: {date}", + "offhost_copies": "Älteste externe Kopie: {date}", + "restore_drill": "Letzte Probe: {date}", + "recovery_kit": "Bestätigt: {date}" + }, + "dueBy": "Fällig bis {date}", + "fix": { + "database_dump": "Sicherungen öffnen", + "weekly_copy": "Sicherungen öffnen", + "offhost_copies": "Sicherungen öffnen", + "restore_drill": "Sicherungen öffnen", + "offhost_setup": "Sicherungen öffnen", + "recovery_kit": "Kit öffnen" + }, + "reason": { + "database_dump": { + "not_available": "Diese Version legt keinen geplanten Datenbank-Dump an.", + "disabled": "Der geplante Datenbank-Dump ist ausgeschaltet.", + "unreadable": "Der Zustand des Datenbank-Dumps konnte nicht gelesen werden.", + "never_succeeded": "Noch kein Datenbank-Dump ist gelungen, und der letzte Versuch ist fehlgeschlagen.", + "never_ran": "In den {hours} Stunden, seit Dumps eingeschaltet sind, wurde kein Datenbank-Dump geschrieben.", + "first_pending": "Der erste Datenbank-Dump ist noch nicht gelaufen.", + "stale": "Der neueste Datenbank-Dump ist {hours} Stunden alt.", + "last_failed": "Der letzte Datenbank-Dump ist fehlgeschlagen. Der davor ist noch aktuell.", + "fresh": "Der neueste Datenbank-Dump ist aktuell.", + "nowhere_to_write": "Das Dump-Verzeichnis {directory} ist nicht beschreibbar und es ist kein externer Bucket eingerichtet, daher kann kein Dump aufbewahrt werden." + }, + "weekly_copy": { + "no_accounts": "Es gibt noch keine Konten zum Kopieren.", + "stale": "Die älteste wöchentliche Kopie ist {days} Tage alt. Konten im Rückstand: {accounts}.", + "never": "Ein Konto hat seit {days} Tagen keine wöchentliche Kopie. Konten im Rückstand: {accounts}.", + "first_pending": "Die erste wöchentliche Kopie entsteht am frühen Sonntagmorgen.", + "fresh": "Jedes Konto hat eine Kopie aus der letzten Woche." + }, + "offhost_copies": { + "not_configured": "Es ist kein externer Bucket eingerichtet.", + "stale": "Externe Kopien liegen mehr als zwei Nächte zurück. Konten im Rückstand: {accounts}.", + "first_pending": "Der nächtliche Lauf für externe Kopien hat noch kein Konto erreicht.", + "fresh": "Jedes Konto hat eine aktuelle Kopie im externen Bucket." + }, + "restore_drill": { + "not_configured": "Die Probe liest den externen Bucket, und der ist nicht eingerichtet.", + "not_run_yet": "Die monatliche Wiederherstellungsprobe ist noch nicht gelaufen.", + "failed": "Die letzte Wiederherstellungsprobe ist fehlgeschlagen: {failed} von {checked} Kopien ließen sich nicht zurücklesen.", + "failed_run": "Die letzte Wiederherstellungsprobe konnte nicht zu Ende laufen.", + "passed": "Die letzte Wiederherstellungsprobe hat {checked} Kopien zurückgelesen." + }, + "offhost_setup": { + "configured": "Kopien verlassen diesen Server jede Nacht.", + "no_admin": "Es gibt noch kein Admin-Konto.", + "not_configured": "Es ist kein externer Bucket eingerichtet, daher liegt jede Kopie nur auf diesem Server.", + "not_configured_grace": "Es ist noch kein externer Bucket eingerichtet, daher liegt jede Kopie nur auf diesem Server." + }, + "recovery_kit": { + "confirmed": "Von jedem Schlüssel, den eine Wiederherstellung braucht, gibt es eine Kopie.", + "out_of_date": "Das Wiederherstellungs-Kit führt Schlüssel auf, für die niemand eine Kopie bestätigt hat: {keys}.", + "no_admin": "Es gibt noch kein Admin-Konto.", + "not_confirmed": "Niemand hat eine Kopie der Schlüssel bestätigt, die eine Wiederherstellung braucht: {keys}.", + "not_confirmed_grace": "Sichere eine Kopie der Schlüssel, die eine Wiederherstellung braucht: {keys}." + } + }, + "recovered": { + "database_dump": "Der Datenbank-Dump ist wieder aktuell.", + "weekly_copy": "Jedes Konto hat wieder eine aktuelle wöchentliche Kopie.", + "offhost_copies": "Jedes Konto hat wieder eine aktuelle externe Kopie.", + "restore_drill": "Die Wiederherstellungsprobe ist wieder bestanden.", + "offhost_setup": "Externe Kopien sind jetzt eingerichtet.", + "recovery_kit": "Das Wiederherstellungs-Kit ist wieder bestätigt." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Vorsorge-Erinnerungen", "description": "Schalte einzelne Vorsorge-Erinnerungen ein oder aus." + }, + "eventBackupHealth": "Zustand der Sicherungen", + "eventBackupHealthDesc": "Wenn eine Sicherung ausbleibt, zurückfällt oder wieder läuft. Nur für Admins.", + "recoveryReadiness": { + "titleAttention": "Sicherungen brauchen Aufmerksamkeit", + "titleFailing": "Sicherungen sind gefährdet", + "titleReminder": "Sicherungen sind weiterhin gefährdet", + "titleRecovered": "Sicherungen laufen wieder normal" } }, "i18n": { diff --git a/messages/en.json b/messages/en.json index 24d7846e2..4267cb9f5 100644 --- a/messages/en.json +++ b/messages/en.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confirmed on {date}.", "confirmedAtBy": "Confirmed on {date} by {who}.", "dueAgain": "The kit was confirmed on {date}, but it now lists keys that confirmation does not cover: {keys}." + }, + "recoveryReadiness": { + "title": "Recovery readiness", + "description": "Whether this server could be put back after a loss, judged from every backup it makes.", + "loading": "Checking the backups…", + "loadError": "Could not load the recovery readiness.", + "state": { + "ok": "Ready", + "attention": "Needs attention", + "failing": "At risk" + }, + "signal": { + "database_dump": "Database dump", + "weekly_copy": "Weekly account copies", + "offhost_copies": "Off-host copies", + "restore_drill": "Restore drill", + "offhost_setup": "Off-host backup", + "recovery_kit": "Recovery kit" + }, + "at": { + "database_dump": "Newest dump: {date}", + "weekly_copy": "Oldest account copy: {date}", + "offhost_copies": "Least recent off-host copy: {date}", + "restore_drill": "Last drill: {date}", + "recovery_kit": "Confirmed: {date}" + }, + "dueBy": "Due by {date}", + "fix": { + "database_dump": "Open backups", + "weekly_copy": "Open backups", + "offhost_copies": "Open backups", + "restore_drill": "Open backups", + "offhost_setup": "Open backups", + "recovery_kit": "Open the kit" + }, + "reason": { + "database_dump": { + "not_available": "This version does not schedule a database dump.", + "disabled": "The scheduled database dump is switched off.", + "unreadable": "The status of the database dump could not be read.", + "never_succeeded": "No database dump has succeeded yet, and the last attempt failed.", + "never_ran": "No database dump has been written in the {hours} hours since dumps were switched on.", + "first_pending": "The first database dump has not run yet.", + "stale": "The newest database dump is {hours} hours old.", + "last_failed": "The last database dump failed. The one before it is still recent.", + "fresh": "The newest database dump is recent.", + "nowhere_to_write": "The dump directory {directory} cannot be written to and no off-host bucket is set up, so no dump can be kept." + }, + "weekly_copy": { + "no_accounts": "There are no accounts to copy yet.", + "stale": "The oldest weekly copy is {days} days old. Accounts behind: {accounts}.", + "never": "An account has had no weekly copy for {days} days. Accounts behind: {accounts}.", + "first_pending": "The first weekly copy is made early on Sunday.", + "fresh": "Every account has a copy from the last week." + }, + "offhost_copies": { + "not_configured": "No off-host bucket is set up.", + "stale": "Off-host copies are more than two nights behind. Accounts behind: {accounts}.", + "first_pending": "The nightly off-host run has not reached any account yet.", + "fresh": "Every account has a recent copy in the off-host bucket." + }, + "restore_drill": { + "not_configured": "The drill reads the off-host bucket, which is not set up.", + "not_run_yet": "The monthly restore drill has not run yet.", + "failed": "The last restore drill failed: {failed} of {checked} copies could not be read back.", + "failed_run": "The last restore drill could not finish.", + "passed": "The last restore drill read back {checked} copies." + }, + "offhost_setup": { + "configured": "Copies leave this server every night.", + "no_admin": "There is no admin account yet.", + "not_configured": "No off-host bucket is set up, so every copy lives on this server only.", + "not_configured_grace": "No off-host bucket is set up yet, so every copy lives on this server only." + }, + "recovery_kit": { + "confirmed": "A copy of every key a restore needs is kept.", + "out_of_date": "The recovery kit lists keys nobody has confirmed a copy of: {keys}.", + "no_admin": "There is no admin account yet.", + "not_confirmed": "Nobody has confirmed a copy of the keys a restore needs: {keys}.", + "not_confirmed_grace": "Save a copy of the keys a restore needs: {keys}." + } + }, + "recovered": { + "database_dump": "The database dump is recent again.", + "weekly_copy": "Every account has a recent weekly copy again.", + "offhost_copies": "Every account has a recent off-host copy again.", + "restore_drill": "The restore drill passed again.", + "offhost_setup": "Off-host copies are set up now.", + "recovery_kit": "The recovery kit is confirmed again." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Checkup reminders", "description": "Switch individual checkup reminders on or off." + }, + "eventBackupHealth": "Backup health", + "eventBackupHealthDesc": "When a backup stops, falls behind or recovers. Sent to admins only.", + "recoveryReadiness": { + "titleAttention": "Backups need attention", + "titleFailing": "Backups are at risk", + "titleReminder": "Backups are still at risk", + "titleRecovered": "Backups are back to normal" } }, "i18n": { diff --git a/messages/es.json b/messages/es.json index 6f3591693..629461a17 100644 --- a/messages/es.json +++ b/messages/es.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confirmado el {date}.", "confirmedAtBy": "Confirmado el {date} por {who}.", "dueAgain": "El kit se confirmó el {date}, pero ahora muestra claves que esa confirmación no cubre: {keys}." + }, + "recoveryReadiness": { + "title": "Preparación para la recuperación", + "description": "Si este servidor se podría recuperar tras una pérdida, a juzgar por cada copia de seguridad que hace.", + "loading": "Comprobando las copias de seguridad…", + "loadError": "No se pudo cargar la preparación para la recuperación.", + "state": { + "ok": "Listo", + "attention": "Necesita atención", + "failing": "En riesgo" + }, + "signal": { + "database_dump": "Volcado de la base de datos", + "weekly_copy": "Copias semanales de las cuentas", + "offhost_copies": "Copias fuera del host", + "restore_drill": "Simulacro de restauración", + "offhost_setup": "Copia fuera del host", + "recovery_kit": "Kit de recuperación" + }, + "at": { + "database_dump": "Volcado más reciente: {date}", + "weekly_copy": "Copia de cuenta más antigua: {date}", + "offhost_copies": "Copia fuera del host más antigua: {date}", + "restore_drill": "Último simulacro: {date}", + "recovery_kit": "Confirmado: {date}" + }, + "dueBy": "Vence el {date}", + "fix": { + "database_dump": "Abrir copias de seguridad", + "weekly_copy": "Abrir copias de seguridad", + "offhost_copies": "Abrir copias de seguridad", + "restore_drill": "Abrir copias de seguridad", + "offhost_setup": "Abrir copias de seguridad", + "recovery_kit": "Abrir el kit" + }, + "reason": { + "database_dump": { + "not_available": "Esta versión no programa un volcado de la base de datos.", + "disabled": "El volcado programado de la base de datos está desactivado.", + "unreadable": "No se pudo leer el estado del volcado de la base de datos.", + "never_succeeded": "Ningún volcado de la base de datos ha salido bien todavía, y el último intento falló.", + "never_ran": "No se ha escrito ningún volcado de la base de datos en las {hours} horas transcurridas desde que se activaron los volcados.", + "first_pending": "El primer volcado de la base de datos aún no se ha ejecutado.", + "stale": "El volcado de la base de datos más reciente tiene {hours} horas.", + "last_failed": "El último volcado de la base de datos falló. El anterior sigue siendo reciente.", + "fresh": "El último volcado de la base de datos es reciente.", + "nowhere_to_write": "No se puede escribir en el directorio de volcados {directory} y no hay ningún bucket externo configurado, así que no se puede guardar ningún volcado." + }, + "weekly_copy": { + "no_accounts": "Todavía no hay cuentas que copiar.", + "stale": "La copia semanal más antigua tiene {days} días. Cuentas con retraso: {accounts}.", + "never": "Una cuenta lleva {days} días sin copia semanal. Cuentas con retraso: {accounts}.", + "first_pending": "La primera copia semanal se hace el domingo de madrugada.", + "fresh": "Cada cuenta tiene una copia de la última semana." + }, + "offhost_copies": { + "not_configured": "No hay ningún bucket fuera del host configurado.", + "stale": "Las copias fuera del host llevan más de dos noches de retraso. Cuentas con retraso: {accounts}.", + "first_pending": "La ejecución nocturna fuera del host aún no ha llegado a ninguna cuenta.", + "fresh": "Cada cuenta tiene una copia reciente en el bucket fuera del host." + }, + "restore_drill": { + "not_configured": "El simulacro lee el bucket fuera del host, que no está configurado.", + "not_run_yet": "El simulacro de restauración mensual aún no se ha ejecutado.", + "failed": "El último simulacro de restauración falló: no se pudieron volver a leer {failed} de {checked} copias.", + "failed_run": "El último simulacro de restauración no pudo terminar.", + "passed": "El último simulacro de restauración volvió a leer {checked} copias." + }, + "offhost_setup": { + "configured": "Las copias salen de este servidor cada noche.", + "no_admin": "Todavía no hay ninguna cuenta de administrador.", + "not_configured": "No hay ningún bucket fuera del host configurado, así que cada copia está solo en este servidor.", + "not_configured_grace": "Aún no hay ningún bucket fuera del host configurado, así que cada copia está solo en este servidor." + }, + "recovery_kit": { + "confirmed": "Se guarda una copia de cada clave que necesita una restauración.", + "out_of_date": "El kit de recuperación muestra claves de las que nadie ha confirmado una copia: {keys}.", + "no_admin": "Todavía no hay ninguna cuenta de administrador.", + "not_confirmed": "Nadie ha confirmado una copia de las claves que necesita una restauración: {keys}.", + "not_confirmed_grace": "Guarda una copia de las claves que necesita una restauración: {keys}." + } + }, + "recovered": { + "database_dump": "El volcado de la base de datos vuelve a ser reciente.", + "weekly_copy": "Cada cuenta vuelve a tener una copia semanal reciente.", + "offhost_copies": "Cada cuenta vuelve a tener una copia fuera del host reciente.", + "restore_drill": "El simulacro de restauración vuelve a superarse.", + "offhost_setup": "Las copias fuera del host ya están configuradas.", + "recovery_kit": "El kit de recuperación vuelve a estar confirmado." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Recordatorios de chequeos", "description": "Activa o desactiva recordatorios de chequeos concretos." + }, + "eventBackupHealth": "Estado de las copias de seguridad", + "eventBackupHealthDesc": "Cuando una copia de seguridad se detiene, se retrasa o se recupera. Solo se envía a los administradores.", + "recoveryReadiness": { + "titleAttention": "Las copias de seguridad necesitan atención", + "titleFailing": "Las copias de seguridad están en riesgo", + "titleReminder": "Las copias de seguridad siguen en riesgo", + "titleRecovered": "Las copias de seguridad vuelven a la normalidad" } }, "i18n": { diff --git a/messages/fr.json b/messages/fr.json index 6fd4d7820..17c13ff4b 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confirmé le {date}.", "confirmedAtBy": "Confirmé le {date} par {who}.", "dueAgain": "Le kit a été confirmé le {date}, mais il liste maintenant des clés que cette confirmation ne couvre pas : {keys}." + }, + "recoveryReadiness": { + "title": "Capacité de restauration", + "description": "Indique si ce serveur pourrait être remis en route après une perte, d'après chaque sauvegarde qu'il réalise.", + "loading": "Vérification des sauvegardes…", + "loadError": "Impossible de charger la capacité de restauration.", + "state": { + "ok": "Prêt", + "attention": "Demande de l'attention", + "failing": "Menacé" + }, + "signal": { + "database_dump": "Dump de la base de données", + "weekly_copy": "Copies hebdomadaires des comptes", + "offhost_copies": "Copies hors hôte", + "restore_drill": "Test de restauration", + "offhost_setup": "Sauvegarde hors hôte", + "recovery_kit": "Kit de récupération" + }, + "at": { + "database_dump": "Dump le plus récent : {date}", + "weekly_copy": "Copie de compte la plus ancienne : {date}", + "offhost_copies": "Copie hors hôte la plus ancienne : {date}", + "restore_drill": "Dernier test : {date}", + "recovery_kit": "Confirmé : {date}" + }, + "dueBy": "À faire avant le {date}", + "fix": { + "database_dump": "Ouvrir les sauvegardes", + "weekly_copy": "Ouvrir les sauvegardes", + "offhost_copies": "Ouvrir les sauvegardes", + "restore_drill": "Ouvrir les sauvegardes", + "offhost_setup": "Ouvrir les sauvegardes", + "recovery_kit": "Ouvrir le kit" + }, + "reason": { + "database_dump": { + "not_available": "Cette version ne planifie pas de dump de la base de données.", + "disabled": "Le dump planifié de la base de données est désactivé.", + "unreadable": "L'état du dump de la base de données n'a pas pu être lu.", + "never_succeeded": "Aucun dump de la base de données n'a encore réussi, et la dernière tentative a échoué.", + "never_ran": "Aucun dump de la base de données n'a été écrit dans les {hours} heures écoulées depuis l'activation des dumps.", + "first_pending": "Le premier dump de la base de données n'a pas encore été lancé.", + "stale": "Le dump de la base de données le plus récent date de {hours} heures.", + "last_failed": "Le dernier dump de la base de données a échoué. Le précédent est encore récent.", + "fresh": "Le dernier dump de la base de données est récent.", + "nowhere_to_write": "Le dossier des dumps {directory} n'est pas accessible en écriture et aucun bucket externe n'est configuré, donc aucun dump ne peut être conservé." + }, + "weekly_copy": { + "no_accounts": "Il n'y a pas encore de compte à copier.", + "stale": "La copie hebdomadaire la plus ancienne date de {days} jours. Comptes en retard : {accounts}.", + "never": "Un compte n'a pas eu de copie hebdomadaire depuis {days} jours. Comptes en retard : {accounts}.", + "first_pending": "La première copie hebdomadaire est faite tôt le dimanche matin.", + "fresh": "Chaque compte a une copie de la semaine dernière." + }, + "offhost_copies": { + "not_configured": "Aucun bucket hors hôte n'est configuré.", + "stale": "Les copies hors hôte ont plus de deux nuits de retard. Comptes en retard : {accounts}.", + "first_pending": "L'envoi nocturne hors hôte n'a encore atteint aucun compte.", + "fresh": "Chaque compte a une copie récente dans le bucket hors hôte." + }, + "restore_drill": { + "not_configured": "Le test lit le bucket hors hôte, qui n'est pas configuré.", + "not_run_yet": "Le test de restauration mensuel n'a pas encore été lancé.", + "failed": "Le dernier test de restauration a échoué : {failed} copies sur {checked} n'ont pas pu être relues.", + "failed_run": "Le dernier test de restauration n'a pas pu aller jusqu'au bout.", + "passed": "Le dernier test de restauration a relu {checked} copies." + }, + "offhost_setup": { + "configured": "Les copies quittent ce serveur chaque nuit.", + "no_admin": "Il n'y a pas encore de compte administrateur.", + "not_configured": "Aucun bucket hors hôte n'est configuré, donc chaque copie se trouve uniquement sur ce serveur.", + "not_configured_grace": "Aucun bucket hors hôte n'est encore configuré, donc chaque copie se trouve uniquement sur ce serveur." + }, + "recovery_kit": { + "confirmed": "Une copie de chaque clé nécessaire à une restauration est conservée.", + "out_of_date": "Le kit de récupération liste des clés dont personne n'a confirmé de copie : {keys}.", + "no_admin": "Il n'y a pas encore de compte administrateur.", + "not_confirmed": "Personne n'a confirmé de copie des clés nécessaires à une restauration : {keys}.", + "not_confirmed_grace": "Sauvegarde une copie des clés nécessaires à une restauration : {keys}." + } + }, + "recovered": { + "database_dump": "Le dump de la base de données est de nouveau récent.", + "weekly_copy": "Chaque compte a de nouveau une copie hebdomadaire récente.", + "offhost_copies": "Chaque compte a de nouveau une copie hors hôte récente.", + "restore_drill": "Le test de restauration réussit de nouveau.", + "offhost_setup": "Les copies hors hôte sont maintenant configurées.", + "recovery_kit": "Le kit de récupération est de nouveau confirmé." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Rappels de bilans", "description": "Activez ou désactivez chaque rappel de bilan." + }, + "eventBackupHealth": "État des sauvegardes", + "eventBackupHealthDesc": "Quand une sauvegarde s'arrête, prend du retard ou repart. Envoyé uniquement aux administrateurs.", + "recoveryReadiness": { + "titleAttention": "Les sauvegardes demandent de l'attention", + "titleFailing": "Les sauvegardes sont menacées", + "titleReminder": "Les sauvegardes sont toujours menacées", + "titleRecovered": "Les sauvegardes sont revenues à la normale" } }, "i18n": { diff --git a/messages/it.json b/messages/it.json index ee227f3a3..e8faea848 100644 --- a/messages/it.json +++ b/messages/it.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confermato il {date}.", "confirmedAtBy": "Confermato il {date} da {who}.", "dueAgain": "Il kit è stato confermato il {date}, ma ora elenca chiavi che quella conferma non copre: {keys}." + }, + "recoveryReadiness": { + "title": "Preparazione al ripristino", + "description": "Se questo server potrebbe essere rimesso in piedi dopo una perdita, valutato in base a ogni backup che esegue.", + "loading": "Controllo dei backup…", + "loadError": "Impossibile caricare la preparazione al ripristino.", + "state": { + "ok": "Pronto", + "attention": "Richiede attenzione", + "failing": "A rischio" + }, + "signal": { + "database_dump": "Dump del database", + "weekly_copy": "Copie settimanali degli account", + "offhost_copies": "Copie fuori host", + "restore_drill": "Prova di ripristino", + "offhost_setup": "Backup fuori host", + "recovery_kit": "Kit di recupero" + }, + "at": { + "database_dump": "Dump più recente: {date}", + "weekly_copy": "Copia di account più vecchia: {date}", + "offhost_copies": "Copia fuori host meno recente: {date}", + "restore_drill": "Ultima prova: {date}", + "recovery_kit": "Confermato: {date}" + }, + "dueBy": "Scadenza: {date}", + "fix": { + "database_dump": "Apri i backup", + "weekly_copy": "Apri i backup", + "offhost_copies": "Apri i backup", + "restore_drill": "Apri i backup", + "offhost_setup": "Apri i backup", + "recovery_kit": "Apri il kit" + }, + "reason": { + "database_dump": { + "not_available": "Questa versione non pianifica un dump del database.", + "disabled": "Il dump pianificato del database è disattivato.", + "unreadable": "Non è stato possibile leggere lo stato del dump del database.", + "never_succeeded": "Nessun dump del database è ancora riuscito, e l'ultimo tentativo non è andato a buon fine.", + "never_ran": "Nelle {hours} ore trascorse da quando i dump sono stati attivati non è stato scritto alcun dump del database.", + "first_pending": "Il primo dump del database non è ancora stato eseguito.", + "stale": "Il dump del database più recente ha {hours} ore.", + "last_failed": "L'ultimo dump del database non è riuscito. Quello precedente è ancora recente.", + "fresh": "L'ultimo dump del database è recente.", + "nowhere_to_write": "La cartella dei dump {directory} non è scrivibile e non è configurato nessun bucket esterno, quindi non si può conservare alcun dump." + }, + "weekly_copy": { + "no_accounts": "Non ci sono ancora account da copiare.", + "stale": "La copia settimanale più vecchia ha {days} giorni. Account in ritardo: {accounts}.", + "never": "Un account non ha una copia settimanale da {days} giorni. Account in ritardo: {accounts}.", + "first_pending": "La prima copia settimanale viene creata domenica mattina presto.", + "fresh": "Ogni account ha una copia dell'ultima settimana." + }, + "offhost_copies": { + "not_configured": "Non è configurato alcun bucket fuori host.", + "stale": "Le copie fuori host sono indietro di più di due notti. Account in ritardo: {accounts}.", + "first_pending": "Il giro notturno fuori host non ha ancora raggiunto alcun account.", + "fresh": "Ogni account ha una copia recente nel bucket fuori host." + }, + "restore_drill": { + "not_configured": "La prova legge il bucket fuori host, che non è configurato.", + "not_run_yet": "La prova di ripristino mensile non è ancora stata eseguita.", + "failed": "L'ultima prova di ripristino non è riuscita: {failed} copie su {checked} non si sono potute rileggere.", + "failed_run": "L'ultima prova di ripristino non è riuscita ad arrivare in fondo.", + "passed": "L'ultima prova di ripristino ha riletto {checked} copie." + }, + "offhost_setup": { + "configured": "Le copie lasciano questo server ogni notte.", + "no_admin": "Non esiste ancora un account amministratore.", + "not_configured": "Non è configurato alcun bucket fuori host, quindi ogni copia si trova solo su questo server.", + "not_configured_grace": "Non è ancora configurato alcun bucket fuori host, quindi ogni copia si trova solo su questo server." + }, + "recovery_kit": { + "confirmed": "Esiste una copia di ogni chiave necessaria per un ripristino.", + "out_of_date": "Il kit di recupero elenca chiavi di cui nessuno ha confermato una copia: {keys}.", + "no_admin": "Non esiste ancora un account amministratore.", + "not_confirmed": "Nessuno ha confermato una copia delle chiavi necessarie per un ripristino: {keys}.", + "not_confirmed_grace": "Salva una copia delle chiavi necessarie per un ripristino: {keys}." + } + }, + "recovered": { + "database_dump": "Il dump del database è di nuovo recente.", + "weekly_copy": "Ogni account ha di nuovo una copia settimanale recente.", + "offhost_copies": "Ogni account ha di nuovo una copia fuori host recente.", + "restore_drill": "La prova di ripristino è di nuovo riuscita.", + "offhost_setup": "Le copie fuori host ora sono configurate.", + "recovery_kit": "Il kit di recupero è di nuovo confermato." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Promemoria dei controlli", "description": "Attiva o disattiva i singoli promemoria dei controlli." + }, + "eventBackupHealth": "Stato dei backup", + "eventBackupHealthDesc": "Quando un backup si ferma, resta indietro o torna a funzionare. Inviato solo agli amministratori.", + "recoveryReadiness": { + "titleAttention": "I backup richiedono attenzione", + "titleFailing": "I backup sono a rischio", + "titleReminder": "I backup sono ancora a rischio", + "titleRecovered": "I backup sono tornati alla normalità" } }, "i18n": { diff --git a/messages/ko.json b/messages/ko.json index 26eafeeef..d70aa8aa4 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -8606,6 +8606,96 @@ "confirmedAt": "{date}에 확인됨.", "confirmedAtBy": "{date}에 {who}님이 확인함.", "dueAgain": "키트는 {date}에 확인되었지만 지금은 그 확인에 포함되지 않은 키가 있습니다: {keys}." + }, + "recoveryReadiness": { + "title": "복구 준비 상태", + "description": "이 서버가 만드는 모든 백업을 기준으로, 서버를 잃었을 때 되살릴 수 있는지 판단합니다.", + "loading": "백업을 확인하는 중…", + "loadError": "복구 준비 상태를 불러오지 못했습니다.", + "state": { + "ok": "준비됨", + "attention": "확인 필요", + "failing": "위험" + }, + "signal": { + "database_dump": "데이터베이스 덤프", + "weekly_copy": "주간 계정 사본", + "offhost_copies": "외부 보관 사본", + "restore_drill": "복원 훈련", + "offhost_setup": "외부 보관 백업", + "recovery_kit": "복구 키트" + }, + "at": { + "database_dump": "최신 덤프: {date}", + "weekly_copy": "가장 오래된 계정 사본: {date}", + "offhost_copies": "가장 오래된 외부 보관 사본: {date}", + "restore_drill": "마지막 훈련: {date}", + "recovery_kit": "확인: {date}" + }, + "dueBy": "{date}까지", + "fix": { + "database_dump": "백업 열기", + "weekly_copy": "백업 열기", + "offhost_copies": "백업 열기", + "restore_drill": "백업 열기", + "offhost_setup": "백업 열기", + "recovery_kit": "키트 열기" + }, + "reason": { + "database_dump": { + "not_available": "이 버전은 데이터베이스 덤프를 예약하지 않습니다.", + "disabled": "예약된 데이터베이스 덤프가 꺼져 있습니다.", + "unreadable": "데이터베이스 덤프 상태를 읽지 못했습니다.", + "never_succeeded": "아직 성공한 데이터베이스 덤프가 없으며 마지막 시도도 실패했습니다.", + "never_ran": "덤프를 켠 뒤 {hours}시간 동안 데이터베이스 덤프가 기록되지 않았습니다.", + "first_pending": "첫 데이터베이스 덤프가 아직 실행되지 않았습니다.", + "stale": "최신 데이터베이스 덤프가 {hours}시간 전 것입니다.", + "last_failed": "마지막 데이터베이스 덤프가 실패했습니다. 그 전 덤프는 아직 최근 것입니다.", + "fresh": "최신 데이터베이스 덤프가 최근 것입니다.", + "nowhere_to_write": "덤프 디렉터리 {directory}에 쓸 수 없고 외부 버킷도 설정되어 있지 않아 덤프를 보관할 수 없습니다." + }, + "weekly_copy": { + "no_accounts": "아직 복사할 계정이 없습니다.", + "stale": "가장 오래된 주간 사본이 {days}일 전 것입니다. 뒤처진 계정: {accounts}.", + "never": "{days}일 동안 주간 사본이 없는 계정이 있습니다. 뒤처진 계정: {accounts}.", + "first_pending": "첫 주간 사본은 일요일 이른 시간에 만들어집니다.", + "fresh": "모든 계정에 지난주 사본이 있습니다." + }, + "offhost_copies": { + "not_configured": "외부 보관 버킷이 설정되지 않았습니다.", + "stale": "외부 보관 사본이 이틀 밤 넘게 뒤처져 있습니다. 뒤처진 계정: {accounts}.", + "first_pending": "야간 외부 보관 실행이 아직 어떤 계정에도 도달하지 않았습니다.", + "fresh": "모든 계정의 최근 사본이 외부 보관 버킷에 있습니다." + }, + "restore_drill": { + "not_configured": "훈련은 외부 보관 버킷을 읽는데, 버킷이 설정되지 않았습니다.", + "not_run_yet": "월간 복원 훈련이 아직 실행되지 않았습니다.", + "failed": "마지막 복원 훈련이 실패했습니다: 사본 {checked}개 중 {failed}개를 다시 읽지 못했습니다.", + "failed_run": "마지막 복원 훈련을 끝까지 마치지 못했습니다.", + "passed": "마지막 복원 훈련에서 사본 {checked}개를 다시 읽었습니다." + }, + "offhost_setup": { + "configured": "사본이 매일 밤 이 서버 밖으로 나갑니다.", + "no_admin": "아직 관리자 계정이 없습니다.", + "not_configured": "외부 보관 버킷이 설정되지 않아 모든 사본이 이 서버에만 있습니다.", + "not_configured_grace": "외부 보관 버킷이 아직 설정되지 않아 모든 사본이 이 서버에만 있습니다." + }, + "recovery_kit": { + "confirmed": "복원에 필요한 모든 키의 사본이 보관되어 있습니다.", + "out_of_date": "복구 키트에 아무도 사본을 확인하지 않은 키가 있습니다: {keys}.", + "no_admin": "아직 관리자 계정이 없습니다.", + "not_confirmed": "복원에 필요한 키의 사본을 아무도 확인하지 않았습니다: {keys}.", + "not_confirmed_grace": "복원에 필요한 키의 사본을 보관하세요: {keys}." + } + }, + "recovered": { + "database_dump": "데이터베이스 덤프가 다시 최신 상태입니다.", + "weekly_copy": "모든 계정에 다시 최근 주간 사본이 있습니다.", + "offhost_copies": "모든 계정에 다시 최근 외부 보관 사본이 있습니다.", + "restore_drill": "복원 훈련을 다시 통과했습니다.", + "offhost_setup": "이제 외부 보관 사본이 설정되었습니다.", + "recovery_kit": "복구 키트가 다시 확인되었습니다." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "검진 리마인더", "description": "검진 리마인더를 개별로 켜거나 끕니다." + }, + "eventBackupHealth": "백업 상태", + "eventBackupHealthDesc": "백업이 멈추거나 뒤처지거나 다시 정상으로 돌아오면 보내요. 관리자에게만 보내요.", + "recoveryReadiness": { + "titleAttention": "백업에 확인이 필요해요", + "titleFailing": "백업이 위험해요", + "titleReminder": "백업이 여전히 위험해요", + "titleRecovered": "백업이 다시 정상이에요" } }, "i18n": { diff --git a/messages/nl.json b/messages/nl.json index 1bf30323a..a05952c4b 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Bevestigd op {date}.", "confirmedAtBy": "Bevestigd op {date} door {who}.", "dueAgain": "De kit is bevestigd op {date}, maar bevat nu sleutels die niet onder die bevestiging vallen: {keys}." + }, + "recoveryReadiness": { + "title": "Herstelbaarheid", + "description": "Of deze server na verlies weer op te bouwen is, beoordeeld op basis van elke back-up die hij maakt.", + "loading": "Back-ups controleren…", + "loadError": "De herstelbaarheid kon niet worden geladen.", + "state": { + "ok": "Gereed", + "attention": "Aandacht nodig", + "failing": "In gevaar" + }, + "signal": { + "database_dump": "Databasedump", + "weekly_copy": "Wekelijkse accountkopieën", + "offhost_copies": "Externe kopieën", + "restore_drill": "Hersteltest", + "offhost_setup": "Externe back-up", + "recovery_kit": "Herstelkit" + }, + "at": { + "database_dump": "Nieuwste dump: {date}", + "weekly_copy": "Oudste accountkopie: {date}", + "offhost_copies": "Oudste externe kopie: {date}", + "restore_drill": "Laatste test: {date}", + "recovery_kit": "Bevestigd: {date}" + }, + "dueBy": "Uiterlijk {date}", + "fix": { + "database_dump": "Back-ups openen", + "weekly_copy": "Back-ups openen", + "offhost_copies": "Back-ups openen", + "restore_drill": "Back-ups openen", + "offhost_setup": "Back-ups openen", + "recovery_kit": "Kit openen" + }, + "reason": { + "database_dump": { + "not_available": "Deze versie plant geen databasedump in.", + "disabled": "De geplande databasedump is uitgeschakeld.", + "unreadable": "De status van de databasedump kon niet worden gelezen.", + "never_succeeded": "Er is nog geen databasedump gelukt, en de laatste poging is mislukt.", + "never_ran": "In de {hours} uur sinds dumps zijn ingeschakeld, is er geen databasedump geschreven.", + "first_pending": "De eerste databasedump is nog niet uitgevoerd.", + "stale": "De nieuwste databasedump is {hours} uur oud.", + "last_failed": "De laatste databasedump is mislukt. De dump daarvoor is nog recent.", + "fresh": "De nieuwste databasedump is recent.", + "nowhere_to_write": "De dumpmap {directory} is niet beschrijfbaar en er is geen externe bucket ingesteld, dus er kan geen dump worden bewaard." + }, + "weekly_copy": { + "no_accounts": "Er zijn nog geen accounts om te kopiëren.", + "stale": "De oudste wekelijkse kopie is {days} dagen oud. Achterlopende accounts: {accounts}.", + "never": "Een account heeft al {days} dagen geen wekelijkse kopie. Achterlopende accounts: {accounts}.", + "first_pending": "De eerste wekelijkse kopie wordt vroeg op zondag gemaakt.", + "fresh": "Elk account heeft een kopie van de afgelopen week." + }, + "offhost_copies": { + "not_configured": "Er is geen externe bucket ingesteld.", + "stale": "Externe kopieën lopen meer dan twee nachten achter. Achterlopende accounts: {accounts}.", + "first_pending": "De nachtelijke run voor externe kopieën heeft nog geen enkel account bereikt.", + "fresh": "Elk account heeft een recente kopie in de externe bucket." + }, + "restore_drill": { + "not_configured": "De test leest de externe bucket, en die is niet ingesteld.", + "not_run_yet": "De maandelijkse hersteltest is nog niet uitgevoerd.", + "failed": "De laatste hersteltest is mislukt: {failed} van {checked} kopieën konden niet worden teruggelezen.", + "failed_run": "De laatste hersteltest kon niet worden afgerond.", + "passed": "De laatste hersteltest heeft {checked} kopieën teruggelezen." + }, + "offhost_setup": { + "configured": "Kopieën verlaten deze server elke nacht.", + "no_admin": "Er is nog geen beheerdersaccount.", + "not_configured": "Er is geen externe bucket ingesteld, dus elke kopie staat alleen op deze server.", + "not_configured_grace": "Er is nog geen externe bucket ingesteld, dus elke kopie staat alleen op deze server." + }, + "recovery_kit": { + "confirmed": "Van elke sleutel die een herstel nodig heeft, wordt een kopie bewaard.", + "out_of_date": "De herstelkit bevat sleutels waarvan niemand een kopie heeft bevestigd: {keys}.", + "no_admin": "Er is nog geen beheerdersaccount.", + "not_confirmed": "Niemand heeft een kopie bevestigd van de sleutels die een herstel nodig heeft: {keys}.", + "not_confirmed_grace": "Bewaar een kopie van de sleutels die een herstel nodig heeft: {keys}." + } + }, + "recovered": { + "database_dump": "De databasedump is weer recent.", + "weekly_copy": "Elk account heeft weer een recente wekelijkse kopie.", + "offhost_copies": "Elk account heeft weer een recente externe kopie.", + "restore_drill": "De hersteltest is weer geslaagd.", + "offhost_setup": "Externe kopieën zijn nu ingesteld.", + "recovery_kit": "De herstelkit is weer bevestigd." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Herinneringen voor controles", "description": "Zet afzonderlijke herinneringen voor controles aan of uit." + }, + "eventBackupHealth": "Status van back-ups", + "eventBackupHealthDesc": "Wordt verstuurd als een back-up stopt, achterloopt of weer bijgewerkt is. Alleen voor beheerders.", + "recoveryReadiness": { + "titleAttention": "Back-ups hebben aandacht nodig", + "titleFailing": "Back-ups lopen gevaar", + "titleReminder": "Back-ups lopen nog steeds gevaar", + "titleRecovered": "Back-ups zijn weer in orde" } }, "i18n": { diff --git a/messages/pl.json b/messages/pl.json index 9f07e3f4f..26d5d9e58 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Potwierdzono {date}.", "confirmedAtBy": "Potwierdzono {date}, potwierdził(a): {who}.", "dueAgain": "Zestaw potwierdzono {date}, ale teraz wymienia klucze, których to potwierdzenie nie obejmuje: {keys}." + }, + "recoveryReadiness": { + "title": "Gotowość do odzyskiwania", + "description": "Czy ten serwer dałoby się odtworzyć po utracie, oceniane na podstawie każdej kopii zapasowej, którą tworzy.", + "loading": "Sprawdzanie kopii zapasowych…", + "loadError": "Nie udało się wczytać gotowości do odzyskiwania.", + "state": { + "ok": "Gotowe", + "attention": "Wymaga uwagi", + "failing": "Zagrożone" + }, + "signal": { + "database_dump": "Zrzut bazy danych", + "weekly_copy": "Cotygodniowe kopie kont", + "offhost_copies": "Kopie poza hostem", + "restore_drill": "Próba przywracania", + "offhost_setup": "Kopia poza hostem", + "recovery_kit": "Zestaw odzyskiwania" + }, + "at": { + "database_dump": "Najnowszy zrzut: {date}", + "weekly_copy": "Najstarsza kopia konta: {date}", + "offhost_copies": "Najstarsza kopia poza hostem: {date}", + "restore_drill": "Ostatnia próba: {date}", + "recovery_kit": "Potwierdzono: {date}" + }, + "dueBy": "Termin: {date}", + "fix": { + "database_dump": "Otwórz kopie zapasowe", + "weekly_copy": "Otwórz kopie zapasowe", + "offhost_copies": "Otwórz kopie zapasowe", + "restore_drill": "Otwórz kopie zapasowe", + "offhost_setup": "Otwórz kopie zapasowe", + "recovery_kit": "Otwórz zestaw" + }, + "reason": { + "database_dump": { + "not_available": "Ta wersja nie planuje zrzutu bazy danych.", + "disabled": "Zaplanowany zrzut bazy danych jest wyłączony.", + "unreadable": "Nie udało się odczytać stanu zrzutu bazy danych.", + "never_succeeded": "Żaden zrzut bazy danych jeszcze się nie udał, a ostatnia próba zakończyła się błędem.", + "never_ran": "Od włączenia zrzutów minęło {hours} h, a nie zapisano żadnego zrzutu bazy danych.", + "first_pending": "Pierwszy zrzut bazy danych jeszcze się nie odbył.", + "stale": "Najnowszy zrzut bazy danych ma {hours} h.", + "last_failed": "Ostatni zrzut bazy danych nie powiódł się. Poprzedni jest jeszcze aktualny.", + "fresh": "Najnowszy zrzut bazy danych jest aktualny.", + "nowhere_to_write": "Katalog zrzutów {directory} nie jest zapisywalny i nie skonfigurowano zewnętrznego bucketu, więc żaden zrzut nie może zostać zachowany." + }, + "weekly_copy": { + "no_accounts": "Nie ma jeszcze kont do skopiowania.", + "stale": "Najstarsza cotygodniowa kopia ma {days} dni. Zaległe konta: {accounts}.", + "never": "Jedno z kont od {days} dni nie ma cotygodniowej kopii. Zaległe konta: {accounts}.", + "first_pending": "Pierwsza cotygodniowa kopia powstaje wcześnie rano w niedzielę.", + "fresh": "Każde konto ma kopię z ostatniego tygodnia." + }, + "offhost_copies": { + "not_configured": "Nie skonfigurowano bucketu poza hostem.", + "stale": "Kopie poza hostem są opóźnione o ponad dwie noce. Zaległe konta: {accounts}.", + "first_pending": "Nocny przebieg kopii poza hostem nie objął jeszcze żadnego konta.", + "fresh": "Każde konto ma aktualną kopię w buckecie poza hostem." + }, + "restore_drill": { + "not_configured": "Próba odczytuje bucket poza hostem, który nie jest skonfigurowany.", + "not_run_yet": "Comiesięczna próba przywracania jeszcze się nie odbyła.", + "failed": "Ostatnia próba przywracania nie powiodła się: nie udało się odczytać {failed} z {checked} kopii.", + "failed_run": "Ostatnia próba przywracania nie dobiegła końca.", + "passed": "Ostatnia próba przywracania odczytała kopie: {checked}." + }, + "offhost_setup": { + "configured": "Kopie opuszczają ten serwer każdej nocy.", + "no_admin": "Nie ma jeszcze konta administratora.", + "not_configured": "Nie skonfigurowano bucketu poza hostem, więc każda kopia jest tylko na tym serwerze.", + "not_configured_grace": "Nie skonfigurowano jeszcze bucketu poza hostem, więc każda kopia jest tylko na tym serwerze." + }, + "recovery_kit": { + "confirmed": "Istnieje kopia każdego klucza potrzebnego do przywrócenia.", + "out_of_date": "Zestaw odzyskiwania wymienia klucze, których kopii nikt nie potwierdził: {keys}.", + "no_admin": "Nie ma jeszcze konta administratora.", + "not_confirmed": "Nikt nie potwierdził kopii kluczy potrzebnych do przywrócenia: {keys}.", + "not_confirmed_grace": "Zapisz kopię kluczy potrzebnych do przywrócenia: {keys}." + } + }, + "recovered": { + "database_dump": "Zrzut bazy danych znów jest aktualny.", + "weekly_copy": "Każde konto znów ma aktualną cotygodniową kopię.", + "offhost_copies": "Każde konto znów ma aktualną kopię poza hostem.", + "restore_drill": "Próba przywracania znów się powiodła.", + "offhost_setup": "Kopie poza hostem są już skonfigurowane.", + "recovery_kit": "Zestaw odzyskiwania jest znów potwierdzony." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Przypomnienia o badaniach kontrolnych", "description": "Włączaj lub wyłączaj pojedyncze przypomnienia o badaniach." + }, + "eventBackupHealth": "Stan kopii zapasowych", + "eventBackupHealthDesc": "Gdy kopia zapasowa przestaje powstawać, zostaje w tyle lub wraca do normy. Wysyłane tylko do administratorów.", + "recoveryReadiness": { + "titleAttention": "Kopie zapasowe wymagają uwagi", + "titleFailing": "Kopie zapasowe są zagrożone", + "titleReminder": "Kopie zapasowe są nadal zagrożone", + "titleRecovered": "Kopie zapasowe znów działają normalnie" } }, "i18n": { diff --git a/messages/pt-BR.json b/messages/pt-BR.json index 30697ec52..0ade6a831 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -8606,6 +8606,96 @@ "confirmedAt": "Confirmado em {date}.", "confirmedAtBy": "Confirmado em {date} por {who}.", "dueAgain": "O kit foi confirmado em {date}, mas agora lista chaves que essa confirmação não cobre: {keys}." + }, + "recoveryReadiness": { + "title": "Prontidão para recuperação", + "description": "Se este servidor poderia ser reconstruído após uma perda, avaliado a partir de cada backup que ele faz.", + "loading": "Verificando os backups…", + "loadError": "Não foi possível carregar a prontidão para recuperação.", + "state": { + "ok": "Pronto", + "attention": "Precisa de atenção", + "failing": "Em risco" + }, + "signal": { + "database_dump": "Dump do banco de dados", + "weekly_copy": "Cópias semanais das contas", + "offhost_copies": "Cópias fora do host", + "restore_drill": "Teste de restauração", + "offhost_setup": "Backup fora do host", + "recovery_kit": "Kit de recuperação" + }, + "at": { + "database_dump": "Dump mais recente: {date}", + "weekly_copy": "Cópia de conta mais antiga: {date}", + "offhost_copies": "Cópia fora do host mais antiga: {date}", + "restore_drill": "Último teste: {date}", + "recovery_kit": "Confirmado: {date}" + }, + "dueBy": "Prazo: {date}", + "fix": { + "database_dump": "Abrir backups", + "weekly_copy": "Abrir backups", + "offhost_copies": "Abrir backups", + "restore_drill": "Abrir backups", + "offhost_setup": "Abrir backups", + "recovery_kit": "Abrir o kit" + }, + "reason": { + "database_dump": { + "not_available": "Esta versão não agenda um dump do banco de dados.", + "disabled": "O dump agendado do banco de dados está desativado.", + "unreadable": "Não foi possível ler o status do dump do banco de dados.", + "never_succeeded": "Nenhum dump do banco de dados foi concluído com sucesso ainda, e a última tentativa falhou.", + "never_ran": "Nenhum dump do banco de dados foi gravado nas {hours} horas desde que os dumps foram ativados.", + "first_pending": "O primeiro dump do banco de dados ainda não foi executado.", + "stale": "O dump do banco de dados mais recente tem {hours} horas.", + "last_failed": "O último dump do banco de dados falhou. O anterior ainda é recente.", + "fresh": "O dump do banco de dados mais recente está em dia.", + "nowhere_to_write": "Não é possível gravar no diretório de dumps {directory} e nenhum bucket externo está configurado, então nenhum dump pode ser guardado." + }, + "weekly_copy": { + "no_accounts": "Ainda não há contas para copiar.", + "stale": "A cópia semanal mais antiga tem {days} dias. Contas atrasadas: {accounts}.", + "never": "Uma conta está sem cópia semanal há {days} dias. Contas atrasadas: {accounts}.", + "first_pending": "A primeira cópia semanal é feita na madrugada de domingo.", + "fresh": "Todas as contas têm uma cópia da última semana." + }, + "offhost_copies": { + "not_configured": "Nenhum bucket fora do host está configurado.", + "stale": "As cópias fora do host estão mais de duas noites atrasadas. Contas atrasadas: {accounts}.", + "first_pending": "A execução noturna fora do host ainda não chegou a nenhuma conta.", + "fresh": "Todas as contas têm uma cópia recente no bucket fora do host." + }, + "restore_drill": { + "not_configured": "O teste lê o bucket fora do host, que não está configurado.", + "not_run_yet": "O teste mensal de restauração ainda não foi executado.", + "failed": "O último teste de restauração falhou: {failed} de {checked} cópias não puderam ser lidas de volta.", + "failed_run": "O último teste de restauração não pôde ser concluído.", + "passed": "O último teste de restauração leu de volta {checked} cópias." + }, + "offhost_setup": { + "configured": "As cópias saem deste servidor todas as noites.", + "no_admin": "Ainda não há uma conta de administrador.", + "not_configured": "Nenhum bucket fora do host está configurado, então todas as cópias ficam apenas neste servidor.", + "not_configured_grace": "Nenhum bucket fora do host foi configurado ainda, então todas as cópias ficam apenas neste servidor." + }, + "recovery_kit": { + "confirmed": "Há uma cópia guardada de cada chave de que uma restauração precisa.", + "out_of_date": "O kit de recuperação lista chaves das quais ninguém confirmou uma cópia: {keys}.", + "no_admin": "Ainda não há uma conta de administrador.", + "not_confirmed": "Ninguém confirmou uma cópia das chaves de que uma restauração precisa: {keys}.", + "not_confirmed_grace": "Guarde uma cópia das chaves de que uma restauração precisa: {keys}." + } + }, + "recovered": { + "database_dump": "O dump do banco de dados está em dia novamente.", + "weekly_copy": "Todas as contas têm novamente uma cópia semanal recente.", + "offhost_copies": "Todas as contas têm novamente uma cópia recente fora do host.", + "restore_drill": "O teste de restauração passou novamente.", + "offhost_setup": "As cópias fora do host estão configuradas agora.", + "recovery_kit": "O kit de recuperação está confirmado novamente." + } } }, "achievements": { @@ -9071,6 +9161,14 @@ "checkupReminders": { "title": "Lembretes de check-up", "description": "Ligue ou desligue lembretes de check-up individuais." + }, + "eventBackupHealth": "Saúde dos backups", + "eventBackupHealthDesc": "Enviado quando um backup para, fica atrasado ou se recupera. Apenas para administradores.", + "recoveryReadiness": { + "titleAttention": "Os backups precisam de atenção", + "titleFailing": "Os backups estão em risco", + "titleReminder": "Os backups continuam em risco", + "titleRecovered": "Os backups voltaram ao normal" } }, "i18n": { diff --git a/src/__tests__/success-affordance-guard.test.ts b/src/__tests__/success-affordance-guard.test.ts index 78eea0bbb..f271e4b2e 100644 --- a/src/__tests__/success-affordance-guard.test.ts +++ b/src/__tests__/success-affordance-guard.test.ts @@ -135,6 +135,13 @@ const PINNED_AFFORDANCES: Record< // off-host copy is, and the state it paints green came out of the database // rather than out of a request this card just made. "src/components/admin/offhost-backups-section.tsx": { "text-success": 1 }, + // Reads the server's verdict, writes nothing: the check and the green mark + // say a backup signal is ok as the readiness model judged it, not that a + // request this card made succeeded. + "src/components/admin/recovery-readiness-card.tsx": { + CheckCircle2: 2, + "text-success": 1, + }, "src/components/admin/recent-audit-preview.tsx": { CheckCircle2: 2, "text-success": 1, diff --git a/src/app/admin/[section]/renderer.tsx b/src/app/admin/[section]/renderer.tsx index fc35e8d16..97f3aaba9 100644 --- a/src/app/admin/[section]/renderer.tsx +++ b/src/app/admin/[section]/renderer.tsx @@ -28,6 +28,7 @@ import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-s import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section"; import { BackupsSection } from "@/components/admin/backups-section"; import { OffhostBackupsSection } from "@/components/admin/offhost-backups-section"; +import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card"; import { DangerZoneSection } from "@/components/admin/danger-zone-section"; import { EncryptionSection } from "@/components/admin/encryption-section"; import { GeneralSettingsSection } from "@/components/admin/general-settings-section"; @@ -166,10 +167,12 @@ export function AdminSectionRenderer({ ); case "backups": - // Two cards, two independent failure modes: the copies held in this - // database, then the copies that reached the operator's bucket. + // The verdict across every backup first, then two cards, two + // independent failure modes: the copies held in this database, then + // the copies that reached the operator's bucket. return ( + diff --git a/src/app/admin/page.tsx b/src/app/admin/page.tsx index cb2e9567c..7bcc5f60d 100644 --- a/src/app/admin/page.tsx +++ b/src/app/admin/page.tsx @@ -4,6 +4,7 @@ import { useAuth } from "@/hooks/use-auth"; import { useTranslations } from "@/lib/i18n/context"; import { QueryErrorRow } from "@/components/ui/query-error-row"; import { AdminShell } from "@/components/admin/admin-shell"; +import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card"; import { SystemStatusSummary } from "@/components/admin/system-status-summary"; import { VersionTileSection } from "@/components/admin/version-tile-section"; import { RecentAuditPreview } from "@/components/admin/recent-audit-preview"; @@ -49,6 +50,9 @@ export default function AdminOverviewPage() { + {/* v1.46 — every backup signal, the recovery kit included. */} + +
diff --git a/src/app/api/admin/recovery-readiness/route.ts b/src/app/api/admin/recovery-readiness/route.ts new file mode 100644 index 000000000..7c34286f9 --- /dev/null +++ b/src/app/api/admin/recovery-readiness/route.ts @@ -0,0 +1,24 @@ +/** + * GET /api/admin/recovery-readiness + * + * Every backup signal with its verdict, its time and where to fix it: the + * database dump, the weekly copy, the off-host copies, the last restore drill + * (no time window: a failed drill stays failed until a later one passes), the + * off-host setup and the recovery kit. The same judgement the hourly + * evaluation pushes to the admins (`src/lib/recovery/readiness-model.ts`). + * Cookie-only admin. + */ +import { apiHandler, requireAdmin } from "@/lib/api-handler"; +import { apiSuccess } from "@/lib/api-response"; +import { annotate } from "@/lib/logging/context"; +import { readRecoveryReadiness } from "@/lib/recovery/readiness"; + +export const dynamic = "force-dynamic"; + +export const GET = apiHandler(async () => { + await requireAdmin(); + annotate({ action: { name: "admin.recoveryReadiness.status" } }); + const readiness = await readRecoveryReadiness(); + annotate({ meta: { recovery_readiness_state: readiness.state } }); + return apiSuccess(readiness); +}); diff --git a/src/app/api/notifications/preferences/route.ts b/src/app/api/notifications/preferences/route.ts index ad39a9675..2a05c0a13 100644 --- a/src/app/api/notifications/preferences/route.ts +++ b/src/app/api/notifications/preferences/route.ts @@ -9,7 +9,10 @@ import { sanitiseZodIssues, } from "@/lib/api-response"; import { notificationPreferenceSchema } from "@/lib/validations/notifications"; -import { EVENT_TYPES, CHANNEL_TYPE_LABELS } from "@/lib/notifications/types"; +import { + CHANNEL_TYPE_LABELS, + eventTypesForRole, +} from "@/lib/notifications/types"; import type { ChannelType } from "@/lib/notifications/types"; import { decrypt, encrypt } from "@/lib/crypto"; import { resolveEventDefaults } from "@/lib/notifications/event-defaults"; @@ -92,7 +95,9 @@ export const GET = apiHandler(async () => { return apiSuccess({ channels: channelData, preferences, - eventTypes: EVENT_TYPES as unknown as string[], + // Admin-only events (backup health) are listed for admins only: nobody + // else is ever sent one, so a switch for it would switch nothing. + eventTypes: eventTypesForRole(user.role), // What an event does on a channel with no stored row, resolved for this // account by the same function the dispatcher asks. A client renders a // switch without a row from this map, never from its own copy of the diff --git a/src/components/admin/__tests__/backups-restore-preview.test.tsx b/src/components/admin/__tests__/backups-restore-preview.test.tsx index 4107fe523..ad65daf0e 100644 --- a/src/components/admin/__tests__/backups-restore-preview.test.tsx +++ b/src/components/admin/__tests__/backups-restore-preview.test.tsx @@ -53,7 +53,7 @@ vi.mock("@tanstack/react-query", () => ({ schedule: { lastSuccessAt: "2026-09-01T03:00:00.000Z", lastSuccessAgeDays: 2, - staleAfterDays: 10, + staleAfterDays: 9, stale: false, lastRun: null, lastRunFailed: false, diff --git a/src/components/admin/__tests__/backups-section-empty.test.tsx b/src/components/admin/__tests__/backups-section-empty.test.tsx index b1d44b41d..a4ab51b2c 100644 --- a/src/components/admin/__tests__/backups-section-empty.test.tsx +++ b/src/components/admin/__tests__/backups-section-empty.test.tsx @@ -6,7 +6,7 @@ import en from "../../../../messages/en.json"; const healthySchedule = { lastSuccessAt: "2026-09-01T03:00:00.000Z", lastSuccessAgeDays: 2, - staleAfterDays: 10, + staleAfterDays: 9, stale: false, lastRun: { at: "2026-09-01T03:00:12.000Z", diff --git a/src/components/admin/__tests__/recovery-readiness-card.test.tsx b/src/components/admin/__tests__/recovery-readiness-card.test.tsx new file mode 100644 index 000000000..3d130ed3b --- /dev/null +++ b/src/components/admin/__tests__/recovery-readiness-card.test.tsx @@ -0,0 +1,154 @@ +/** + * The recovery readiness card. + * + * 1. every signal renders as its own row with the server's state, its + * sentence and its time; + * 2. a row that needs work links to where it is fixed, unless that is the + * page already open; an ok or inactive row carries no link; + * 3. a grace period shows its due date rather than a time of an event; + * 4. a failed drill shows its reason with no time window: a failure months + * old is still on the card; + * 5. the overall badge carries the worst state; the copy resolves in German. + * + * Mutation check: show the fix link on every row and case 2 goes red; drop + * the `detail` line and case 4 goes red. + */ +import { describe, expect, it, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +import { I18nProvider } from "@/lib/i18n/context"; +import { queryKeys } from "@/lib/query-keys"; +import { RecoveryReadinessCard } from "@/components/admin/recovery-readiness-card"; +import type { + ReadinessSignal, + RecoveryReadiness, +} from "@/components/admin/use-recovery-readiness"; +import { READINESS_FIX_PATHS } from "@/lib/recovery/readiness-model"; +import { primeLocales } from "@/__tests__/helpers/prime-locales"; + +primeLocales("de"); + +const pathname = vi.hoisted(() => ({ current: "/admin" })); +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: vi.fn(), replace: vi.fn() }), + usePathname: () => pathname.current, +})); + +function sig( + id: ReadinessSignal["id"], + state: ReadinessSignal["state"], + reason: string, + over: Partial = {}, +): ReadinessSignal { + return { + id, + state, + reason, + at: null, + params: {}, + detail: null, + fixPath: READINESS_FIX_PATHS[id], + ...over, + }; +} + +const READINESS: RecoveryReadiness = { + state: "failing", + evaluatedAt: "2026-10-25T12:00:00.000Z", + signals: [ + sig("database_dump", "inactive", "not_available"), + sig("weekly_copy", "attention", "stale", { + at: "2026-10-14T03:00:00.000Z", + params: { days: 11, accounts: 2 }, + }), + sig("offhost_copies", "ok", "fresh", { at: "2026-10-25T02:40:00.000Z" }), + sig("restore_drill", "failing", "failed", { + at: "2026-07-01T04:20:00.000Z", + params: { failed: 1, checked: 4 }, + detail: "1 of 4 backup objects failed. Restore drill: bad tag", + }), + sig("offhost_setup", "ok", "configured"), + sig("recovery_kit", "ok", "not_confirmed_grace", { + at: "2026-10-30T09:00:00.000Z", + params: { keys: "v2, API_TOKEN_HMAC_KEY", count: 2 }, + }), + ], +}; + +function render(readiness: RecoveryReadiness, locale: "en" | "de" = "en") { + const client = new QueryClient(); + client.setQueryData(queryKeys.adminRecoveryReadiness(), readiness); + return renderToStaticMarkup( + + + + + , + ); +} + +/** The markup of one signal's row. */ +function row(html: string, id: string): string { + const start = html.indexOf(`data-testid="recovery-signal-${id}"`); + expect(start).toBeGreaterThan(-1); + const end = html.indexOf("", start); + return html.slice(start, end); +} + +describe("RecoveryReadinessCard", () => { + it("renders every signal with its state and sentence", () => { + pathname.current = "/admin"; + const html = render(READINESS); + for (const s of READINESS.signals) { + expect(row(html, s.id)).toContain(`data-state="${s.state}"`); + } + expect(row(html, "weekly_copy")).toContain( + "The oldest weekly copy is 11 days old. Accounts behind: 2.", + ); + expect(row(html, "database_dump")).toContain( + "This version does not schedule a database dump.", + ); + }); + + it("links a row that needs work to where it is fixed, and nothing else", () => { + pathname.current = "/admin"; + const html = render(READINESS); + expect(row(html, "weekly_copy")).toContain('href="/admin/backups"'); + expect(row(html, "restore_drill")).toContain('href="/admin/backups"'); + expect(row(html, "recovery_kit")).toContain('href="/admin/encryption"'); + expect(row(html, "offhost_copies")).not.toContain("href="); + expect(row(html, "database_dump")).not.toContain("href="); + }); + + it("does not link to the page already open", () => { + pathname.current = "/admin/backups"; + const html = render(READINESS); + expect(row(html, "weekly_copy")).not.toContain("href="); + expect(row(html, "recovery_kit")).toContain('href="/admin/encryption"'); + }); + + it("shows a grace period's due date", () => { + pathname.current = "/admin"; + expect(row(render(READINESS), "recovery_kit")).toContain("Due by "); + }); + + it("keeps a months-old drill failure on the card with its reason", () => { + pathname.current = "/admin"; + const drill = row(render(READINESS), "restore_drill"); + expect(drill).toContain( + "The last restore drill failed: 1 of 4 copies could not be read back.", + ); + expect(drill).toContain("Restore drill: bad tag"); + expect(drill).toContain("Last drill: "); + }); + + it("carries the worst state on the badge, and speaks German", () => { + pathname.current = "/admin"; + const html = render(READINESS); + expect(html).toContain('data-testid="recovery-readiness-state"'); + expect(html).toContain('data-state="failing"'); + expect(render({ ...READINESS, state: "ok" })).toContain(">Ready<"); + expect(render(READINESS, "de")).toContain("Wiederherstellbarkeit"); + }); +}); diff --git a/src/components/admin/recovery-readiness-card.tsx b/src/components/admin/recovery-readiness-card.tsx new file mode 100644 index 000000000..19cf4e5db --- /dev/null +++ b/src/components/admin/recovery-readiness-card.tsx @@ -0,0 +1,208 @@ +"use client"; + +/** + * `` — every backup signal on one card: the database + * dump, the weekly account copies, the off-host copies, the last restore + * drill, the off-host setup and the recovery kit. Each row says its verdict, + * the time it is about and where to fix it. + * + * No time window anywhere: a failed drill stays on this card until a later + * drill passes, where the failing-jobs card forgot it after 72 hours. The + * verdicts are the server's (`src/lib/recovery/readiness-model.ts`), the same + * ones the hourly evaluation pushes to every admin, so the card and the push + * cannot disagree. + * + * Shown on the admin overview and on the Backups page. A row whose fix lives + * on the page already open carries no link to itself. + */ + +import Link from "next/link"; +import { usePathname } from "next/navigation"; +import { + AlertTriangle, + CheckCircle2, + Clock, + LifeBuoy, + Loader2, + MinusCircle, + XCircle, +} from "lucide-react"; + +import { SettingsCardHeader } from "@/components/settings/_card-header"; +import { SettingsCard } from "@/components/settings/settings-card"; +import { Badge } from "@/components/ui/badge"; +import { InlineAction } from "@/components/ui/inline-action"; +import { QueryErrorRow } from "@/components/ui/query-error-row"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import { cn } from "@/lib/utils"; + +import { + useRecoveryReadiness, + type ReadinessSignal, + type ReadinessState, + type RecoveryReadiness, +} from "./use-recovery-readiness"; + +const STATE_ICON: Record< + ReadinessState, + React.ComponentType<{ className?: string; "aria-hidden"?: boolean }> +> = { + ok: CheckCircle2, + attention: AlertTriangle, + failing: XCircle, + inactive: MinusCircle, +}; + +const STATE_TONE: Record = { + ok: "text-success", + attention: "text-warning", + failing: "text-destructive", + inactive: "text-muted-foreground", +}; + +/** A grace period is ok for the verdict, but still a step for the admin. */ +const isGrace = (signal: ReadinessSignal) => signal.reason.endsWith("_grace"); + +function OverallBadge({ + state, +}: { + state: Exclude; +}) { + const { t } = useTranslations(); + const label = { + ok: t("admin.recoveryReadiness.state.ok"), + attention: t("admin.recoveryReadiness.state.attention"), + failing: t("admin.recoveryReadiness.state.failing"), + }[state]; + return ( + + {label} + + ); +} + +function SignalRow({ + signal, + currentPath, +}: { + signal: ReadinessSignal; + currentPath: string | null; +}) { + const { t } = useTranslations(); + const fmt = useFormatters(); + const grace = isGrace(signal); + const Icon = grace ? Clock : STATE_ICON[signal.state]; + const tone = grace ? "text-info" : STATE_TONE[signal.state]; + const date = signal.at ? fmt.dateTime(new Date(signal.at)) : null; + const showFix = + signal.state !== "inactive" && + (signal.state !== "ok" || grace) && + signal.fixPath !== currentPath; + + return ( +
  • + +
    +

    + {t(`admin.recoveryReadiness.signal.${signal.id}`)} +

    +

    + {t( + `admin.recoveryReadiness.reason.${signal.id}.${signal.reason}`, + signal.params, + )} +

    + {date ? ( +

    + {grace + ? t("admin.recoveryReadiness.dueBy", { date }) + : t(`admin.recoveryReadiness.at.${signal.id}`, { date })} +

    + ) : null} + {signal.detail && signal.state !== "ok" ? ( +

    + {signal.detail} +

    + ) : null} +
    + {showFix ? ( + + + {t(`admin.recoveryReadiness.fix.${signal.id}`)} + + + ) : null} +
    • + ); +} + +/** The list itself, for a readiness that has loaded. */ +export function RecoveryReadinessList({ + readiness, + currentPath, +}: { + readiness: RecoveryReadiness; + currentPath: string | null; +}) { + return ( +
      + {readiness.signals.map((signal) => ( + + ))} +
    + ); +} + +export function RecoveryReadinessCard() { + const { t } = useTranslations(); + const currentPath = usePathname(); + const { data, isError, error, refetch } = useRecoveryReadiness(); + + return ( + + : null} + /> + {data ? ( + + ) : isError ? ( + void refetch()} + /> + ) : ( +
    +
    + )} +     + ); +} diff --git a/src/components/admin/use-recovery-readiness.ts b/src/components/admin/use-recovery-readiness.ts new file mode 100644 index 000000000..74b6ed119 --- /dev/null +++ b/src/components/admin/use-recovery-readiness.ts @@ -0,0 +1,24 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { apiGet } from "@/lib/api/api-fetch"; +import { queryKeys } from "@/lib/query-keys"; +import type { RecoveryReadiness } from "@/lib/recovery/readiness-model"; + +export type { + ReadinessSignal, + ReadinessState, + RecoveryReadiness, +} from "@/lib/recovery/readiness-model"; + +/** `GET /api/admin/recovery-readiness`; admin-only and cookie-only. */ +export function useRecoveryReadiness() { + return useQuery({ + queryKey: queryKeys.adminRecoveryReadiness(), + queryFn: () => + apiGet("/api/admin/recovery-readiness", { + credentials: "include", + }), + staleTime: 60_000, + }); +} diff --git a/src/lib/jobs/__tests__/backup-schedule-status.test.ts b/src/lib/jobs/__tests__/backup-schedule-status.test.ts index 2a45bee31..6beaad532 100644 --- a/src/lib/jobs/__tests__/backup-schedule-status.test.ts +++ b/src/lib/jobs/__tests__/backup-schedule-status.test.ts @@ -50,6 +50,19 @@ describe("summariseBackupSchedule", () => { expect(status.lastRunFailed).toBe(false); }); + it("draws the line where the readiness model does: past nine days", () => { + const at = (ms: number) => + summariseBackupSchedule({ + scheduledCreatedAt: [new Date(NOW.getTime() - ms)], + lastRun: null, + now: NOW, + }).stale; + const DAY = 24 * 60 * 60 * 1000; + expect(BACKUP_STALE_AFTER_DAYS).toBe(9); + expect(at(9 * DAY)).toBe(false); + expect(at(9 * DAY + 1)).toBe(true); + }); + it("surfaces a failed last run even while the copy is still in window", () => { const status = summariseBackupSchedule({ scheduledCreatedAt: [daysAgo(3)], diff --git a/src/lib/jobs/__tests__/restore-drill.test.ts b/src/lib/jobs/__tests__/restore-drill.test.ts index 07136af8d..e6fe03ea3 100644 --- a/src/lib/jobs/__tests__/restore-drill.test.ts +++ b/src/lib/jobs/__tests__/restore-drill.test.ts @@ -14,7 +14,12 @@ vi.mock("../report-worker-error", () => ({ reportWorkerError: vi.fn().mockResolvedValue(undefined), })); +vi.mock("../restore-drill-outcome", () => ({ + recordRestoreDrillOutcome: vi.fn().mockResolvedValue(undefined), +})); + import { reportWorkerError } from "../report-worker-error"; +import { recordRestoreDrillOutcome } from "../restore-drill-outcome"; const ENC_KEY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; @@ -257,6 +262,8 @@ describe("handleRestoreDrill", () => { vi.stubEnv("BACKUP_S3_ENDPOINT", ""); await handleRestoreDrill([]); expect(reportWorkerError).not.toHaveBeenCalled(); + // Nothing was checked, so there is no verdict to keep. + expect(recordRestoreDrillOutcome).not.toHaveBeenCalled(); }); it("pages via reportWorkerError when the drill fails", async () => { @@ -270,5 +277,22 @@ describe("handleRestoreDrill", () => { expect(vi.mocked(reportWorkerError).mock.calls[0][0]).toBe( RESTORE_DRILL_QUEUE, ); + // The failure is kept for the readiness tile, past pg-boss's retention. + expect(recordRestoreDrillOutcome).toHaveBeenCalledTimes(1); + expect(vi.mocked(recordRestoreDrillOutcome).mock.calls[0][0]).toMatchObject( + { ok: false, accountsChecked: 0, accountsFailed: 0 }, + ); + }); + + it("keeps a failed drill's verdict even when keeping it fails", async () => { + stubConfigEnv(); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", "not-a-key"); + vi.mocked(recordRestoreDrillOutcome).mockRejectedValueOnce( + new Error("database away"), + ); + const outcome = await handleRestoreDrill([]); + // The drill's own failure is still what the run reports. + expect(outcome.ok).toBe(false); + expect(reportWorkerError).toHaveBeenCalledTimes(1); }); }); diff --git a/src/lib/jobs/backup-schedule-status.ts b/src/lib/jobs/backup-schedule-status.ts index d7a5ac20d..38264c08a 100644 --- a/src/lib/jobs/backup-schedule-status.ts +++ b/src/lib/jobs/backup-schedule-status.ts @@ -23,13 +23,15 @@ * alive. */ import type { LastQueueRun } from "@/lib/jobs/job-failures"; +import { WEEKLY_COPY_ATTENTION_AFTER_DAYS } from "@/lib/recovery/readiness-model"; /** * How old the newest scheduled copy may get before the page says so. The cron - * is weekly, so seven days is normal and eight is a rounding error; ten means - * at least one Sunday produced nothing. + * is weekly, so seven days is normal and eight is a long week; past nine at + * least one Sunday produced nothing. The recovery readiness model draws its + * attention line at the same instant, so this page and the push agree. */ -export const BACKUP_STALE_AFTER_DAYS = 10; +export const BACKUP_STALE_AFTER_DAYS = WEEKLY_COPY_ATTENTION_AFTER_DAYS; const DAY_MS = 24 * 60 * 60 * 1000; @@ -71,7 +73,9 @@ export function summariseBackupSchedule(args: { // new — the page's empty state already says there is nothing there, and // calling a fresh install "overdue" would train the operator to ignore the // warning that matters. - stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS, + stale: + newest !== null && + args.now.getTime() - newest.getTime() > BACKUP_STALE_AFTER_DAYS * DAY_MS, lastRun: args.lastRun, lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed", }; diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 250838b7d..007d7c6f7 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -234,6 +234,13 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "restore_drill_plaintext_bytes", "restore_drill_unchecked", "restore_drill_stale", + // The hourly recovery readiness evaluation: the overall verdict, how many + // signals need attention or are failing, notices sent, admins reached. + "recovery_readiness_admins", + "recovery_readiness_attention", + "recovery_readiness_failing", + "recovery_readiness_notices", + "recovery_readiness_state", "restore_fail_cause", "restore_failed", "restore_measurements", diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 28ed26810..0f0c273d3 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -368,6 +368,9 @@ export const QUEUE_RUNTIME: Readonly> = { "geo-backfill": short("At most 500 audit rows per pass."), "geolite2-fetch": short("One database download."), "tls-pin-monitor": short("One TLS handshake."), + "recovery-readiness": short( + "A few indexed reads, one row per account from two backup ledgers, and at most one notice per signal.", + ), "integration-watchdog-hourly": short( "A few indexed reads per account with a connection and no provider call; a 14-day history count only for a silent source.", ), diff --git a/src/lib/jobs/recovery-readiness-queue.ts b/src/lib/jobs/recovery-readiness-queue.ts new file mode 100644 index 000000000..77e1f3b74 --- /dev/null +++ b/src/lib/jobs/recovery-readiness-queue.ts @@ -0,0 +1,12 @@ +/** + * The queue facts of `recovery-readiness.ts`: its name and schedule. They live + * apart from the job so the worker registry can read them without loading it. + */ +export const RECOVERY_READINESS_QUEUE = "recovery-readiness"; + +/** + * Hourly at :33, a minute no other schedule uses. Hourly is far inside every + * threshold the evaluation judges (36 hours for the dump, two nights for the + * off-host copies), and a notice is sent once per change whatever the cadence. + */ +export const RECOVERY_READINESS_CRON = "33 * * * *"; diff --git a/src/lib/jobs/recovery-readiness.ts b/src/lib/jobs/recovery-readiness.ts new file mode 100644 index 000000000..18518d330 --- /dev/null +++ b/src/lib/jobs/recovery-readiness.ts @@ -0,0 +1,42 @@ +/** + * The hourly recovery readiness evaluation: read every backup signal, judge + * it, and push each change to the admins (`readiness-notify.ts`). The tile on + * the admin pages reads the same verdict live; this job is what makes it a + * push rather than a page somebody has to open. + * + * Cheap and bounded: a handful of indexed reads, one row per account for the + * two backup ledgers, and at most one notice per signal per run. The next + * tick is the retry. + */ +import type { Job } from "pg-boss"; + +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { withBackgroundEvent } from "@/lib/logging/background"; +import { readRecoveryReadiness } from "@/lib/recovery/readiness"; +import { notifyReadinessChanges } from "@/lib/recovery/readiness-notify"; + +export async function handleRecoveryReadiness( + jobs: Job[], +): Promise { + void jobs; + return withBackgroundEvent("job.recovery_readiness", async (evt) => { + try { + const readiness = await readRecoveryReadiness(); + const outcome = await notifyReadinessChanges(readiness); + const counts = (state: string) => + readiness.signals.filter((s) => s.state === state).length; + const did = { + recovery_readiness_state: readiness.state, + recovery_readiness_attention: counts("attention"), + recovery_readiness_failing: counts("failing"), + recovery_readiness_notices: outcome.sent.length, + recovery_readiness_admins: outcome.admins, + }; + for (const [key, value] of Object.entries(did)) evt.addMeta(key, value); + return jobDone(did); + } catch (err) { + evt.addWarning(`recovery-readiness failed: ${err}`); + return jobFailed("recovery readiness evaluation failed", err); + } + }); +} diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts index 70429c156..a13bb2dd2 100644 --- a/src/lib/jobs/registry/maintenance.ts +++ b/src/lib/jobs/registry/maintenance.ts @@ -169,6 +169,10 @@ import { } from "@/lib/jobs/pr-detection"; import { PULSE_DAY_MEAN_QUEUE } from "@/lib/jobs/pulse-day-mean"; import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + RECOVERY_READINESS_CRON, + RECOVERY_READINESS_QUEUE, +} from "@/lib/jobs/recovery-readiness-queue"; import { RESTORE_DRILL_CRON, RESTORE_DRILL_QUEUE, @@ -343,6 +347,17 @@ const jobs: readonly JobDefinition[] = [ load: async () => (await import("@/lib/jobs/restore-drill")).handleRestoreDrill, }), + // Hourly: judge every backup signal and push each change to the admins. + // Single-flight and no retries; the next tick is the retry, and the notice + // claim keeps two evaluations from sending the same notice. + defineJob({ + queue: RECOVERY_READINESS_QUEUE, + cron: RECOVERY_READINESS_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/recovery-readiness")).handleRecoveryReadiness, + }), // The nightly backstop for the purge an account deletion kicks at once. prune( OFFHOST_PURGE_QUEUE, diff --git a/src/lib/jobs/restore-drill-outcome.ts b/src/lib/jobs/restore-drill-outcome.ts new file mode 100644 index 000000000..c6ab0bb69 --- /dev/null +++ b/src/lib/jobs/restore-drill-outcome.ts @@ -0,0 +1,54 @@ +/** + * The monthly restore drill's verdict, kept until the next drill replaces it. + * + * The drill used to say what it found only through `reportWorkerError` + * (stderr, and GlitchTip where configured) and through the pg-boss row of its + * run. The row is gone after seven days and the admin failing-jobs card reads + * 72 hours of it, so a drill that failed on the first of the month was on + * screen until the fourth and invisible for the rest of it. One row, written + * at the end of every drill that had something to check, is what the recovery + * readiness tile and its push read instead, with no window at all. + * + * A failure stays the verdict until a later drill passes: `lastFailureAt` + * newer than `lastSuccessAt` is the rule (`evaluateRestoreDrill`). + */ +import { prisma } from "@/lib/db"; +import { redactSecrets } from "@/lib/logging/redact"; + +/** The reason is a line on a status tile, not a stack trace. */ +const MAX_REASON_CHARS = 300; + +export interface RestoreDrillResult { + ok: boolean; + accountsChecked: number; + accountsFailed: number; + /** Why it failed; ignored when `ok`. */ + reason?: string | null; +} + +export async function recordRestoreDrillOutcome( + result: RestoreDrillResult, + at: Date = new Date(), +): Promise { + const reason = result.ok + ? null + : (() => { + const text = redactSecrets((result.reason ?? "").trim()); + return text.length > MAX_REASON_CHARS + ? `${text.slice(0, MAX_REASON_CHARS)}…` + : text || null; + })(); + const verdict = result.ok + ? { lastSuccessAt: at } + : { lastFailureAt: at, lastFailureReason: reason }; + const counts = { + lastRunAt: at, + accountsChecked: result.accountsChecked, + accountsFailed: result.accountsFailed, + }; + await prisma.restoreDrillOutcome.upsert({ + where: { id: "singleton" }, + create: { id: "singleton", ...counts, ...verdict }, + update: { ...counts, ...verdict }, + }); +} diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index a2d38c476..deb8d1353 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -15,8 +15,10 @@ * application key it needs. Each account is reported on its own. It deliberately performs NO database restore — the * drill validates the artefact, not the import path. The result is * surfaced through the wide-event meta on success and through - * `reportWorkerError` (stderr + GlitchTip) on any failure, so a broken - * backup chain pages the operator eleven months before it matters. + * `reportWorkerError` (stderr + GlitchTip) on any failure, and the verdict is + * kept (`restore-drill-outcome.ts`) for the recovery readiness tile and its + * push to the admins, so a broken backup chain reaches the operator eleven + * months before it matters. * * Schedule: 04:11 on the 1st of each month (`11 4 1 * *`), after the * 02:30 nightly upload and the 03:xx cleanup window, on a minute slot @@ -45,6 +47,10 @@ import { type S3Like, } from "@/lib/jobs/offhost-backup"; import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { + recordRestoreDrillOutcome, + type RestoreDrillResult, +} from "@/lib/jobs/restore-drill-outcome"; import { jobBudget } from "@/lib/jobs/job-budget"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; @@ -254,6 +260,21 @@ export async function runRestoreDrill( }; } +/** + * Keep the verdict for the readiness tile. A failed write is a warning on the + * run, never a failed drill: the drill's own reports already went out. + */ +async function keepOutcome( + evt: { addWarning: (message: string) => void }, + result: RestoreDrillResult, +): Promise { + try { + await recordRestoreDrillOutcome(result); + } catch (err) { + evt.addWarning(`restore-drill outcome not recorded: ${err}`); + } +} + /** * pg-boss handler. Mirrors the off-host uploader's posture: a * not-configured deployment skips with a wide-event warning (most @@ -323,6 +344,17 @@ export async function handleRestoreDrill( { dateKey: report.dateKey, unchecked: report.unchecked }, ); } + await keepOutcome(evt, { + ok: report.failed.length === 0, + accountsChecked: report.accounts.length, + accountsFailed: report.failed.length, + reason: + report.failed.length === 0 + ? null + : `${report.failed.length} of ${report.accounts.length} backup objects failed. ${ + report.failed[0].error ?? report.failed[0].objectKey + }`, + }); if (report.failed.length > 0) { // Already paged above, per account. The queue does not retry: a // retry reads the same objects and would page the same accounts again. @@ -346,6 +378,12 @@ export async function handleRestoreDrill( } evt.addWarning(`restore-drill failed: ${err}`); await reportWorkerError(RESTORE_DRILL_QUEUE, err); + await keepOutcome(evt, { + ok: false, + accountsChecked: 0, + accountsFailed: 0, + reason: err instanceof Error ? err.message : String(err), + }); return jobFailed("restore drill failed", err); } }); diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index d90560c84..a2cd5a468 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -100,6 +100,7 @@ const subsystemSurface = { "step-up-elevation-cleanup": { audience: "system" }, "data-backup-offhost": { audience: "system" }, "data-restore-drill": { audience: "system" }, + "recovery-readiness": { audience: "system" }, "offhost-backup-purge": { audience: "system" }, "backup-restore": { audience: "system" }, "host-metric-sample": { audience: "system" }, diff --git a/src/lib/notifications/tap-target.ts b/src/lib/notifications/tap-target.ts index 95fd59c40..60e1ac7c1 100644 --- a/src/lib/notifications/tap-target.ts +++ b/src/lib/notifications/tap-target.ts @@ -51,6 +51,7 @@ export const NOTIFICATION_TAP_PATHS: Record = { SECURITY_ALERT: "/settings/security", DAILY_BRIEFING: "/", SHARED_RECORD_INTAKE: "/medications", + BACKUP_HEALTH: "/admin/backups", }; const MAX_PATH_LENGTH = 512; diff --git a/src/lib/notifications/types.ts b/src/lib/notifications/types.ts index a84a01395..ea05bf42a 100644 --- a/src/lib/notifications/types.ts +++ b/src/lib/notifications/types.ts @@ -125,6 +125,13 @@ export const EVENT_TYPES = [ // dispatcher addresses a userId, and under record substitution that userId // is the owner by design). Default ON, see EVENT_DEFAULT_ENABLED. "SHARED_RECORD_INTAKE", + // Recovery readiness, for admins only: a backup signal turned to + // attention or failing, is still failing a day later, or is back to normal + // (`src/lib/recovery/readiness-notify.ts`). Default ON: the whole point is + // that a stopped backup is pushed rather than found on a page weeks later. + // An admin routes it per channel in the /notifications matrix, where only + // admins see the row (`eventTypesForRole`). + "BACKUP_HEALTH", ] as const; export type EventType = (typeof EVENT_TYPES)[number]; @@ -214,8 +221,21 @@ export const EVENT_DEFAULT_ENABLED: Record = { // sits in the /notifications matrix from day one, so an owner who finds it // noisy silences it per channel with one switch. SHARED_RECORD_INTAKE: true, + // ON by default; see the note on `EVENT_TYPES`. Only admins are ever + // addressed, and at most once per state change per signal. + BACKUP_HEALTH: true, }; +/** Event types only an admin is ever sent, and only an admin is shown. */ +export const ADMIN_ONLY_EVENT_TYPES: readonly EventType[] = ["BACKUP_HEALTH"]; + +/** The event types a person with `role` can receive, in matrix order. */ +export function eventTypesForRole(role: string): EventType[] { + return role === "ADMIN" + ? [...EVENT_TYPES] + : EVENT_TYPES.filter((type) => !ADMIN_ONLY_EVENT_TYPES.includes(type)); +} + export const CHANNEL_TYPE_LABELS: Record = { TELEGRAM: "Telegram", NTFY: "ntfy", diff --git a/src/lib/recovery/__tests__/readiness-model.test.ts b/src/lib/recovery/__tests__/readiness-model.test.ts new file mode 100644 index 000000000..dc15c3bf0 --- /dev/null +++ b/src/lib/recovery/__tests__/readiness-model.test.ts @@ -0,0 +1,424 @@ +/** + * The recovery readiness thresholds, one signal at a time, on both sides of + * every line the model draws. + * + * Mutation check: move any threshold by one unit (35 h for the dump, 8 or 13 + * days for the weekly copy, 6 days of setup grace) and the matching + * "just inside" or "just past" case goes red. + */ +import { describe, expect, it } from "vitest"; + +import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health"; +import { + DUMP_FAILING_AFTER_HOURS, + evaluateDatabaseDump, + evaluateOffhostCopies, + evaluateOffhostSetup, + evaluateRecoveryKit, + evaluateRecoveryReadiness, + evaluateRestoreDrill, + evaluateWeeklyCopy, + READINESS_FIX_PATHS, + READINESS_SIGNALS, + SETUP_GRACE_DAYS, + WEEKLY_COPY_ATTENTION_AFTER_DAYS, + WEEKLY_COPY_FAILING_AFTER_DAYS, + type ReadinessInputs, +} from "@/lib/recovery/readiness-model"; + +const NOW = new Date("2026-10-25T12:00:00Z"); +const HOUR = 3_600_000; +const DAY = 24 * HOUR; +const ago = (ms: number) => new Date(NOW.getTime() - ms); + +/** A dump report in the dump module's own shape. */ +function dump( + over: Partial & { + lastGoodAt?: Date | null; + lastFailureAt?: Date | null; + lastFailureReason?: string; + } = {}, +): DatabaseBackupHealth { + const { + lastGoodAt = ago(2 * HOUR), + lastFailureAt = null, + lastFailureReason, + ...rest + } = over; + return { + enabled: true, + directory: "/data/db-backups", + localDirWritable: true, + offhostConfigured: false, + lastGood: lastGoodAt + ? { + at: lastGoodAt.toISOString(), + fileName: "healthlog.dump.enc", + sizeBytes: 1024, + local: true, + offhost: false, + } + : null, + lastFailure: lastFailureAt + ? { + at: lastFailureAt.toISOString(), + error: lastFailureReason ?? "failed", + } + : null, + ...rest, + }; +} + +function inputs(over: Partial = {}): ReadinessInputs { + return { + now: NOW, + firstAdminAt: ago(30 * DAY), + dump: null, + dumpWatchedSince: null, + weekly: { accounts: [{ createdAt: ago(60 * DAY), lastCopyAt: ago(DAY) }] }, + offhost: { + configured: true, + accounts: [ + { lastAttemptAt: ago(10 * HOUR), lastSuccessAt: ago(10 * HOUR) }, + ], + }, + drill: null, + kit: { confirmedAt: ago(10 * DAY), unconfirmed: [] }, + ...over, + }; +} + +describe("database dump", () => { + const evalDump = (d: ReadinessInputs["dump"], watched: Date | null = null) => + evaluateDatabaseDump({ now: NOW, dump: d, dumpWatchedSince: watched }); + + it("does not apply while no dump is scheduled, or while it is switched off", () => { + expect(evalDump(null)).toMatchObject({ + state: "inactive", + reason: "not_available", + }); + expect(evalDump(dump({ enabled: false }))).toMatchObject({ + state: "inactive", + reason: "disabled", + }); + }); + + it("fails when its status cannot be read", () => { + expect(evalDump("unreadable")).toMatchObject({ + state: "failing", + reason: "unreadable", + }); + }); + + it("is ok up to 36 hours and failing past them", () => { + const inside = evalDump( + dump({ lastGoodAt: ago(DUMP_FAILING_AFTER_HOURS * HOUR - 1) }), + ); + expect(inside).toMatchObject({ state: "ok", reason: "fresh" }); + const past = evalDump( + dump({ lastGoodAt: ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1) }), + ); + expect(past).toMatchObject({ + state: "failing", + reason: "stale", + params: { hours: 36 }, + }); + expect(past.at).toBe( + ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1).toISOString(), + ); + }); + + it("calls a recent failure after a recent dump attention, with the reason", () => { + const s = evalDump( + dump({ lastFailureAt: ago(HOUR), lastFailureReason: "pg_dump exited 1" }), + ); + expect(s).toMatchObject({ + state: "attention", + reason: "last_failed", + detail: "pg_dump exited 1", + }); + }); + + it("fails when a dump can be kept nowhere, and accepts a bucket-only setup", () => { + expect(evalDump(dump({ localDirWritable: false }))).toMatchObject({ + state: "failing", + reason: "nowhere_to_write", + params: { directory: "/data/db-backups" }, + }); + expect( + evalDump(dump({ localDirWritable: false, offhostConfigured: true })) + .state, + ).toBe("ok"); + }); + + it("gives a newly switched-on dump 36 hours before calling it failing", () => { + const fresh = dump({ lastGoodAt: null }); + expect(evalDump(fresh, ago(HOUR))).toMatchObject({ + state: "ok", + reason: "first_pending", + }); + expect(evalDump(fresh, null)).toMatchObject({ + state: "ok", + reason: "first_pending", + }); + expect(evalDump(fresh, ago(37 * HOUR))).toMatchObject({ + state: "failing", + reason: "never_ran", + }); + expect( + evalDump(dump({ lastGoodAt: null, lastFailureAt: ago(HOUR) }), ago(HOUR)), + ).toMatchObject({ state: "failing", reason: "never_succeeded" }); + }); +}); + +describe("weekly copy", () => { + const weekly = ( + ...accounts: Array<{ createdAt: Date; lastCopyAt: Date | null }> + ) => evaluateWeeklyCopy({ now: NOW, weekly: { accounts } }); + + it("is attention past 9 days and failing past 14, judged by the worst account", () => { + const fresh = { createdAt: ago(90 * DAY), lastCopyAt: ago(DAY) }; + expect( + weekly(fresh, { createdAt: ago(90 * DAY), lastCopyAt: ago(9 * DAY - 1) }) + .state, + ).toBe("ok"); + expect( + weekly(fresh, { + createdAt: ago(90 * DAY), + lastCopyAt: ago(WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY + 1), + }), + ).toMatchObject({ + state: "attention", + reason: "stale", + params: { days: 9, accounts: 1 }, + }); + expect( + weekly(fresh, { + createdAt: ago(90 * DAY), + lastCopyAt: ago(WEEKLY_COPY_FAILING_AFTER_DAYS * DAY - 1), + }).state, + ).toBe("attention"); + expect( + weekly(fresh, { + createdAt: ago(90 * DAY), + lastCopyAt: ago(WEEKLY_COPY_FAILING_AFTER_DAYS * DAY + 1), + }), + ).toMatchObject({ state: "failing", params: { days: 14 } }); + }); + + it("ages an account without any copy from its creation", () => { + expect(weekly({ createdAt: ago(3 * DAY), lastCopyAt: null })).toMatchObject( + { + state: "ok", + reason: "first_pending", + }, + ); + expect( + weekly({ createdAt: ago(20 * DAY), lastCopyAt: null }), + ).toMatchObject({ + state: "failing", + reason: "never", + }); + }); + + it("does not apply without accounts", () => { + expect(weekly().state).toBe("inactive"); + }); +}); + +describe("off-host copies", () => { + const offhost = ( + configured: boolean, + ...accounts: Array<{ + lastAttemptAt: Date | null; + lastSuccessAt: Date | null; + }> + ) => evaluateOffhostCopies({ now: NOW, offhost: { configured, accounts } }); + + it("does not apply while no bucket is configured", () => { + expect(offhost(false).state).toBe("inactive"); + }); + + it("calls any account more than two nights behind attention", () => { + const fresh = { + lastAttemptAt: ago(5 * HOUR), + lastSuccessAt: ago(5 * HOUR), + }; + // One missed night (inside two periods plus the grace) is not yet news. + expect( + offhost(true, fresh, { + lastAttemptAt: ago(HOUR), + lastSuccessAt: ago(40 * HOUR), + }).state, + ).toBe("ok"); + expect( + offhost(true, fresh, { + lastAttemptAt: ago(HOUR), + lastSuccessAt: ago(2 * DAY + 7 * HOUR), + }), + ).toMatchObject({ + state: "attention", + reason: "stale", + params: { accounts: 1 }, + }); + }); + + it("counts an account a run reached and never wrote as behind", () => { + expect( + offhost(true, { lastAttemptAt: ago(HOUR), lastSuccessAt: null }), + ).toMatchObject({ state: "attention", params: { accounts: 1 } }); + }); + + it("waits for the first run rather than calling the cohort behind", () => { + expect( + offhost(true, { lastAttemptAt: null, lastSuccessAt: null }), + ).toMatchObject({ + state: "ok", + reason: "first_pending", + }); + }); +}); + +describe("restore drill", () => { + const drill = (d: ReadinessInputs["drill"], configured = true) => + evaluateRestoreDrill({ offhost: { configured, accounts: [] }, drill: d }); + const run = { + lastRunAt: ago(5 * DAY), + lastSuccessAt: null as Date | null, + lastFailureAt: null as Date | null, + lastFailureReason: null as string | null, + accountsChecked: 3, + accountsFailed: 0, + }; + + it("stays failing after a failure, with no time window, until a later success", () => { + const failed = { + ...run, + lastFailureAt: ago(60 * DAY), + accountsFailed: 1, + lastFailureReason: "bad tag", + }; + expect(drill(failed)).toMatchObject({ + state: "failing", + reason: "failed", + params: { failed: 1, checked: 3 }, + detail: "bad tag", + }); + expect(drill({ ...failed, lastSuccessAt: ago(61 * DAY) }).state).toBe( + "failing", + ); + expect(drill({ ...failed, lastSuccessAt: ago(DAY) })).toMatchObject({ + state: "ok", + reason: "passed", + }); + }); + + it("names a run that checked nothing", () => { + expect( + drill({ ...run, accountsChecked: 0, lastFailureAt: ago(DAY) }).reason, + ).toBe("failed_run"); + }); + + it("does not apply without a bucket, and is ok before its first run", () => { + expect(drill(null, false).state).toBe("inactive"); + expect(drill(null)).toMatchObject({ state: "ok", reason: "not_run_yet" }); + }); +}); + +describe("off-host setup", () => { + const setup = (configured: boolean, firstAdminAt: Date | null) => + evaluateOffhostSetup({ + now: NOW, + firstAdminAt, + offhost: { configured, accounts: [] }, + }); + + it("asks for a bucket 7 days after the first admin exists", () => { + expect(setup(false, ago(SETUP_GRACE_DAYS * DAY - 1))).toMatchObject({ + state: "ok", + reason: "not_configured_grace", + }); + expect(setup(false, ago(SETUP_GRACE_DAYS * DAY + 1))).toMatchObject({ + state: "attention", + reason: "not_configured", + }); + expect(setup(true, ago(90 * DAY)).state).toBe("ok"); + expect(setup(false, null).state).toBe("inactive"); + }); +}); + +describe("recovery kit", () => { + const kit = ( + confirmedAt: Date | null, + unconfirmed: Array<{ + kind: "application" | "backup" | "token"; + id: string; + }>, + firstAdminAt: Date | null = ago(30 * DAY), + ) => + evaluateRecoveryKit({ + now: NOW, + firstAdminAt, + kit: { confirmedAt, unconfirmed }, + }); + + it("is ok while the confirmation covers every key", () => { + expect(kit(ago(DAY), [])).toMatchObject({ + state: "ok", + reason: "confirmed", + }); + }); + + it("names a needed key nobody confirmed, at once once a kit was confirmed", () => { + expect(kit(ago(DAY), [{ kind: "application", id: "v1" }])).toMatchObject({ + state: "attention", + reason: "out_of_date", + params: { keys: "v1", count: 1 }, + }); + }); + + it("gives a never-confirmed kit 7 days after setup", () => { + const keys = [ + { kind: "application" as const, id: "v1" }, + { kind: "token" as const, id: "API_TOKEN_HMAC_KEY" }, + ]; + expect(kit(null, keys, ago(SETUP_GRACE_DAYS * DAY - 1))).toMatchObject({ + state: "ok", + reason: "not_confirmed_grace", + }); + expect(kit(null, keys, ago(SETUP_GRACE_DAYS * DAY + 1))).toMatchObject({ + state: "attention", + reason: "not_confirmed", + params: { keys: "v1, API_TOKEN_HMAC_KEY", count: 2 }, + }); + }); +}); + +describe("the whole verdict", () => { + it("is the worst applying signal, and inactive ones never count", () => { + expect(evaluateRecoveryReadiness(inputs()).state).toBe("ok"); + expect( + evaluateRecoveryReadiness( + inputs({ dump: dump({ lastGoodAt: ago(40 * HOUR) }) }), + ).state, + ).toBe("failing"); + expect( + evaluateRecoveryReadiness( + inputs({ + kit: { + confirmedAt: ago(DAY), + unconfirmed: [{ kind: "backup", id: "abc" }], + }, + }), + ).state, + ).toBe("attention"); + }); + + it("reports every signal once, in order, each with a fix path", () => { + const r = evaluateRecoveryReadiness(inputs()); + expect(r.signals.map((s) => s.id)).toEqual([...READINESS_SIGNALS]); + for (const s of r.signals) + expect(s.fixPath).toBe(READINESS_FIX_PATHS[s.id]); + expect(r.evaluatedAt).toBe(NOW.toISOString()); + }); +}); diff --git a/src/lib/recovery/__tests__/readiness-notify.test.ts b/src/lib/recovery/__tests__/readiness-notify.test.ts new file mode 100644 index 000000000..95e0bc949 --- /dev/null +++ b/src/lib/recovery/__tests__/readiness-notify.test.ts @@ -0,0 +1,287 @@ +/** + * The notice state machine: once per state change per signal, a recovery + * notice once, a reminder at most once a day while failing, nothing else. + * + * Two layers. `decideNotice` is pure and pinned case by case. + * `notifyReadinessChanges` runs against an in-memory stand-in for its one + * table that honours the compare-and-set exactly as Postgres would (a row + * moves only from the values the caller read), so the hourly sequence and the + * two-workers race are proven on the real claim logic rather than assumed. + * + * Mutation check: drop the `last !== current` comparison and the "same state + * twice" case sends twice; drop the CAS `where` on `notifiedState` and the + * race sends two notices. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const store = vi.hoisted(() => ({ + notices: new Map< + string, + { + signal: string; + notifiedState: string | null; + notifiedAt: Date | null; + watchedSince: Date | null; + } + >(), + admins: [] as Array<{ id: string }>, +})); + +const same = (a: Date | null, b: Date | null | undefined) => + (a?.getTime() ?? null) === (b?.getTime() ?? null); + +vi.mock("@/lib/db", () => ({ + prisma: { + user: { findMany: vi.fn(async () => store.admins) }, + recoveryReadinessNotice: { + createMany: vi.fn( + async ({ data }: { data: Array<{ signal: string }> }) => { + for (const { signal } of data) { + if (!store.notices.has(signal)) { + store.notices.set(signal, { + signal, + notifiedState: null, + notifiedAt: null, + watchedSince: null, + }); + } + } + return { count: data.length }; + }, + ), + findMany: vi.fn(async () => + [...store.notices.values()].map((row) => ({ ...row })), + ), + updateMany: vi.fn( + async ({ + where, + data, + }: { + where: { + signal: string; + notifiedState?: string | null; + notifiedAt?: Date | null; + watchedSince?: Date | null; + }; + data: Record; + }) => { + const row = store.notices.get(where.signal); + if (!row) return { count: 0 }; + if ( + "notifiedState" in where && + row.notifiedState !== where.notifiedState + ) + return { count: 0 }; + if ("notifiedAt" in where && !same(row.notifiedAt, where.notifiedAt)) + return { count: 0 }; + if ( + "watchedSince" in where && + !same(row.watchedSince, where.watchedSince) + ) + return { count: 0 }; + Object.assign(row, data); + return { count: 1 }; + }, + ), + }, + }, +})); + +const dispatch = vi.hoisted(() => + vi.fn(async (opts: Record) => { + void opts; + }), +); +vi.mock("@/lib/notifications/dispatch-localised", () => ({ + dispatchLocalisedNotification: dispatch, +})); + +import { + decideNotice, + FAILING_REMINDER_HOURS, + notifyReadinessChanges, +} from "@/lib/recovery/readiness-notify"; +import { + READINESS_FIX_PATHS, + READINESS_SIGNALS, + type ReadinessSignal, + type ReadinessState, + type RecoveryReadiness, +} from "@/lib/recovery/readiness-model"; + +const T0 = new Date("2026-10-25T12:00:00Z"); +const HOUR = 3_600_000; +const at = (hours: number) => new Date(T0.getTime() + hours * HOUR); + +describe("decideNotice", () => { + it("announces a change to attention or failing once", () => { + expect(decideNotice(null, "attention", T0)).toEqual({ + send: "problem", + next: { notifiedState: "attention", notifiedAt: T0 }, + }); + const told = { notifiedState: "attention" as const, notifiedAt: T0 }; + expect(decideNotice(told, "attention", at(30)).send).toBeNull(); + expect(decideNotice(told, "failing", at(1)).send).toBe("problem"); + }); + + it("reminds a failing signal once a day, not sooner", () => { + const told = { notifiedState: "failing" as const, notifiedAt: T0 }; + expect( + decideNotice(told, "failing", at(FAILING_REMINDER_HOURS - 1)).send, + ).toBeNull(); + expect(decideNotice(told, "failing", at(FAILING_REMINDER_HOURS))).toEqual({ + send: "reminder", + next: { + notifiedState: "failing", + notifiedAt: at(FAILING_REMINDER_HOURS), + }, + }); + }); + + it("announces a recovery once, and says nothing about an ok that was never broken", () => { + const told = { notifiedState: "failing" as const, notifiedAt: T0 }; + expect(decideNotice(told, "ok", at(2)).send).toBe("recovered"); + expect( + decideNotice({ notifiedState: "ok", notifiedAt: at(2) }, "ok", at(3)), + ).toEqual({ + send: null, + next: null, + }); + expect(decideNotice(null, "ok", T0)).toEqual({ send: null, next: null }); + }); + + it("forgets silently when a signal stops applying", () => { + expect( + decideNotice( + { notifiedState: "failing", notifiedAt: T0 }, + "inactive", + at(1), + ), + ).toEqual({ send: null, next: { notifiedState: null, notifiedAt: null } }); + expect(decideNotice(null, "inactive", T0)).toEqual({ + send: null, + next: null, + }); + }); +}); + +function readiness( + states: Partial>, + now: Date, +): RecoveryReadiness { + return { + state: "ok", + evaluatedAt: now.toISOString(), + signals: READINESS_SIGNALS.map((id) => { + const [state, reason] = states[id] ?? ["ok", "fresh"]; + return { + id, + state, + reason, + at: null, + params: { days: 12, accounts: 1 }, + detail: null, + fixPath: READINESS_FIX_PATHS[id], + }; + }), + }; +} + +const sent = () => + dispatch.mock.calls.map(([opts]) => { + const o = opts as unknown as { + userId: string; + titleKey: string; + messageKey: string; + }; + return `${o.userId} ${o.titleKey.split(".").at(-1)} ${o.messageKey}`; + }); + +describe("notifyReadinessChanges", () => { + beforeEach(() => { + store.notices.clear(); + store.admins = [{ id: "admin-a" }, { id: "admin-b" }]; + dispatch.mockClear(); + }); + + it("sends each notice to every admin, through the backup event", async () => { + await notifyReadinessChanges( + readiness({ weekly_copy: ["attention", "stale"] }, T0), + ); + expect(sent()).toEqual([ + "admin-a titleAttention admin.recoveryReadiness.reason.weekly_copy.stale", + "admin-b titleAttention admin.recoveryReadiness.reason.weekly_copy.stale", + ]); + const first = dispatch.mock.calls[0][0] as unknown as { + eventType: string; + params: Record; + metadata: Record; + }; + expect(first.eventType).toBe("BACKUP_HEALTH"); + expect(first.params).toEqual({ days: 12, accounts: 1 }); + expect(first.metadata).toMatchObject({ + url: "/admin/backups", + signal: "weekly_copy", + }); + }); + + it("runs an hourly sequence without nagging: change, silence, escalation, daily reminder, recovery", async () => { + store.admins = [{ id: "admin-a" }]; + const run = (h: number, state: ReadinessState, reason: string) => + notifyReadinessChanges( + readiness({ restore_drill: [state, reason] }, at(h)), + ); + + await run(0, "attention", "failed"); + for (let h = 1; h < 5; h++) await run(h, "attention", "failed"); + expect(sent()).toHaveLength(1); + + await run(5, "failing", "failed"); + for (let h = 6; h < 5 + FAILING_REMINDER_HOURS; h++) + await run(h, "failing", "failed"); + expect(sent()).toHaveLength(2); + + await run(5 + FAILING_REMINDER_HOURS, "failing", "failed"); + await run(6 + FAILING_REMINDER_HOURS, "failing", "failed"); + expect(sent()).toHaveLength(3); + expect(sent()[2]).toContain("titleReminder"); + + await run(40, "ok", "passed"); + await run(41, "ok", "passed"); + expect(sent()).toHaveLength(4); + expect(sent()[3]).toBe( + "admin-a titleRecovered admin.recoveryReadiness.recovered.restore_drill", + ); + }); + + it("sends one notice when two evaluations race on the same row", async () => { + const r = readiness({ recovery_kit: ["attention", "out_of_date"] }, T0); + await Promise.all([notifyReadinessChanges(r), notifyReadinessChanges(r)]); + expect(sent()).toHaveLength(2); // one notice, two admins + }); + + it("claims nothing while there is no admin to tell", async () => { + store.admins = []; + const r = readiness({ offhost_setup: ["attention", "not_configured"] }, T0); + await notifyReadinessChanges(r); + expect(store.notices.get("offhost_setup")?.notifiedState).toBeNull(); + store.admins = [{ id: "admin-a" }]; + await notifyReadinessChanges(r); + expect(sent()).toHaveLength(1); + }); + + it("remembers since when a signal applies, and forgets it when it stops", async () => { + await notifyReadinessChanges( + readiness({ database_dump: ["ok", "fresh"] }, T0), + ); + expect(store.notices.get("database_dump")?.watchedSince).toEqual(T0); + await notifyReadinessChanges( + readiness({ database_dump: ["ok", "fresh"] }, at(5)), + ); + expect(store.notices.get("database_dump")?.watchedSince).toEqual(T0); + await notifyReadinessChanges( + readiness({ database_dump: ["inactive", "disabled"] }, at(6)), + ); + expect(store.notices.get("database_dump")?.watchedSince).toBeNull(); + }); +}); diff --git a/src/lib/recovery/database-dump-health.ts b/src/lib/recovery/database-dump-health.ts new file mode 100644 index 000000000..76bb908c8 --- /dev/null +++ b/src/lib/recovery/database-dump-health.ts @@ -0,0 +1,50 @@ +/** + * The scheduled database dump, as the recovery readiness model sees it. + * + * The dump is the one artifact that rebuilds a lost host: it holds every + * account, every grant and every credential, and a schema change cannot leave + * anything out of it. Its health is a signal of its own in the readiness + * model, read through a provider so the model does not reach into the dump's + * module and a test can hand in any state it wants. + * + * A provider that answers `null` means "this build does not schedule a dump", + * which the model reports as not available rather than as healthy or broken. + */ + +/** + * What the dump module reports about its own schedule. Mirrors + * `DatabaseBackupHealth` in `src/lib/database-backup/health.ts` field for + * field, so the provider below becomes that module's + * `getDatabaseBackupHealth` without a translation. + */ +export interface DatabaseBackupHealth { + /** `DB_BACKUP_ENABLED` is not off. */ + enabled: boolean; + /** The directory dumps are kept in (`DB_BACKUP_DIR`). */ + directory: string; + /** This process can write there. False means dumps only go off-host. */ + localDirWritable: boolean; + /** The off-host bucket is configured, or a dump has been uploaded to it. */ + offhostConfigured: boolean; + /** The newest dump that was kept on the server or reached the bucket. */ + lastGood: { + at: string; + fileName: string; + sizeBytes: number | null; + local: boolean; + offhost: boolean; + } | null; + /** The newest run that failed, or that a dying process left unfinished. */ + lastFailure: { at: string; error: string } | null; +} + +export type DatabaseBackupHealthProvider = ( + now?: Date, +) => Promise; + +/** + * The provider every caller uses unless it passes its own: the dump's health, + * or `null` while no dump is scheduled by this build. + */ +export const readDatabaseBackupHealth: DatabaseBackupHealthProvider = + async () => null; diff --git a/src/lib/recovery/readiness-model.ts b/src/lib/recovery/readiness-model.ts new file mode 100644 index 000000000..962bb05f5 --- /dev/null +++ b/src/lib/recovery/readiness-model.ts @@ -0,0 +1,422 @@ +/** + * Recovery readiness: can this instance be put back after a loss, judged from + * every fact the server holds about its own copies, in one place. + * + * Every one of these facts used to be a pull, on a page nobody opens until the + * day it matters. The weekly copy stopped for 46 days and showed a perfectly + * ordinary row. A failed monthly drill was on the failing-jobs card three days + * in thirty. An off-host bucket that was never configured showed only on the + * backups page, and the key step confirmed one key of the several a restore + * needs. This module turns them into six signals with one verdict each, which + * the readiness tile shows and the hourly evaluation pushes to every admin. + * + * Pure: the inputs are read elsewhere (`readiness.ts`), so each threshold is a + * unit test away and the tile and the push cannot disagree about a verdict. + * + * States: + * ok — nothing to do. A grace period (`*_grace` reasons) is ok too: + * the admin is shown the step, nobody is pushed about it yet. + * attention — something will hurt on the day of a restore; worth a push. + * failing — the instance cannot be restored the way it is meant to be. + * inactive — the signal does not apply here (no dump in this build, no + * off-host bucket for the drill). Never pushed, never counts + * towards the overall verdict. + */ +import { + classifyOffhostBackup, + type OffhostBackupFreshness, +} from "@/lib/jobs/offhost-backup-freshness"; +import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health"; +import type { KitEntry } from "@/lib/recovery/kit-rules"; + +const HOUR_MS = 60 * 60 * 1000; +const DAY_MS = 24 * HOUR_MS; + +/** A dump older than this, while dumps are on, means no usable dump. */ +export const DUMP_FAILING_AFTER_HOURS = 36; +/** The weekly copy runs every Sunday; past nine days one Sunday was missed. */ +export const WEEKLY_COPY_ATTENTION_AFTER_DAYS = 9; +/** Past fourteen days two Sundays produced nothing for some account. */ +export const WEEKLY_COPY_FAILING_AFTER_DAYS = 14; +/** + * How long after the first admin exists the off-host bucket and the recovery + * kit may stay unset before the admins are pushed about it. + */ +export const SETUP_GRACE_DAYS = 7; + +export const READINESS_SIGNALS = [ + "database_dump", + "weekly_copy", + "offhost_copies", + "restore_drill", + "offhost_setup", + "recovery_kit", +] as const; +export type ReadinessSignalId = (typeof READINESS_SIGNALS)[number]; + +export const READINESS_STATES = [ + "ok", + "attention", + "failing", + "inactive", +] as const; +export type ReadinessState = (typeof READINESS_STATES)[number]; + +/** Where each signal is fixed. Every path is an admin page. */ +export const READINESS_FIX_PATHS: Record = { + database_dump: "/admin/backups", + weekly_copy: "/admin/backups", + offhost_copies: "/admin/backups", + restore_drill: "/admin/backups", + offhost_setup: "/admin/backups", + recovery_kit: "/admin/encryption", +}; + +export interface ReadinessSignal { + id: ReadinessSignalId; + state: ReadinessState; + /** + * Why, as a closed code per signal. The tile and the push render it through + * `admin.recoveryReadiness..`, so it never carries prose. + */ + reason: string; + /** The instant the signal is about (the newest dump, the last drill). */ + at: string | null; + /** Values the sentence for `reason` interpolates. Never key material. */ + params: Record; + /** + * An operator-facing line from the source (a dump's or a drill's failure + * text), already redacted where it was written. Shown in the tile only. + */ + detail: string | null; + fixPath: string; +} + +export interface RecoveryReadiness { + /** The worst state of the signals that apply; `ok` when none do. */ + state: Exclude; + evaluatedAt: string; + signals: ReadinessSignal[]; +} + +/** What the evaluation reads, gathered by `readiness.ts`. */ +export interface ReadinessInputs { + now: Date; + /** When the first admin account was created; null on an empty install. */ + firstAdminAt: Date | null; + /** + * `null` while no dump is scheduled by this build; `"unreadable"` when the + * dump module threw while reporting, which is itself a failure to report. + */ + dump: DatabaseBackupHealth | null | "unreadable"; + /** + * Since when the evaluation has seen dumps switched on. A dump that has + * never been attempted is judged against this, so an upgrade that turns + * dumps on is not failing before the first run had a chance. + */ + dumpWatchedSince: Date | null; + weekly: { + /** One entry per account: when it was created and its newest copy. */ + accounts: ReadonlyArray<{ createdAt: Date; lastCopyAt: Date | null }>; + }; + offhost: { + configured: boolean; + accounts: ReadonlyArray<{ + lastAttemptAt: Date | null; + lastSuccessAt: Date | null; + }>; + }; + drill: { + lastRunAt: Date; + lastSuccessAt: Date | null; + lastFailureAt: Date | null; + lastFailureReason: string | null; + accountsChecked: number; + accountsFailed: number; + } | null; + kit: { + /** A confirmation exists at all. */ + confirmedAt: Date | null; + /** Keys the kit lists that the confirmation does not cover. */ + unconfirmed: ReadonlyArray>; + }; +} + +function signal( + id: ReadinessSignalId, + state: ReadinessState, + reason: string, + at: Date | null, + params: Record = {}, + detail: string | null = null, +): ReadinessSignal { + return { + id, + state, + reason, + at: at?.toISOString() ?? null, + params, + detail, + fixPath: READINESS_FIX_PATHS[id], + }; +} + +const wholeHours = (ms: number) => Math.max(0, Math.floor(ms / HOUR_MS)); +const wholeDays = (ms: number) => Math.max(0, Math.floor(ms / DAY_MS)); + +function setupGraceEndsAt(firstAdminAt: Date | null): Date | null { + return firstAdminAt + ? new Date(firstAdminAt.getTime() + SETUP_GRACE_DAYS * DAY_MS) + : null; +} + +export function evaluateDatabaseDump( + input: Pick, +): ReadinessSignal { + const { dump, now } = input; + const id = "database_dump"; + if (dump === null) return signal(id, "inactive", "not_available", null); + if (dump === "unreadable") return signal(id, "failing", "unreadable", null); + if (!dump.enabled) return signal(id, "inactive", "disabled", null); + const lastGoodAt = dump.lastGood ? new Date(dump.lastGood.at) : null; + const lastFailureAt = dump.lastFailure ? new Date(dump.lastFailure.at) : null; + const failureReason = dump.lastFailure?.error ?? null; + // A directory this process cannot write is a valid setup when the dumps go + // to the bucket; with no bucket either, no dump can be kept anywhere. + if (!dump.localDirWritable && !dump.offhostConfigured) { + return signal(id, "failing", "nowhere_to_write", lastGoodAt, { + directory: dump.directory, + }); + } + const limitMs = DUMP_FAILING_AFTER_HOURS * HOUR_MS; + const failedSinceGood = + lastFailureAt !== null && + (lastGoodAt === null || lastFailureAt > lastGoodAt); + if (lastGoodAt === null) { + if (failedSinceGood) { + return signal( + id, + "failing", + "never_succeeded", + lastFailureAt, + {}, + failureReason, + ); + } + const watched = input.dumpWatchedSince ?? now; + return now.getTime() - watched.getTime() > limitMs + ? signal(id, "failing", "never_ran", null, { + hours: DUMP_FAILING_AFTER_HOURS, + }) + : signal(id, "ok", "first_pending", null); + } + const ageMs = now.getTime() - lastGoodAt.getTime(); + if (ageMs > limitMs) { + return signal( + id, + "failing", + "stale", + lastGoodAt, + { hours: wholeHours(ageMs) }, + failedSinceGood ? failureReason : null, + ); + } + if (failedSinceGood) { + return signal( + id, + "attention", + "last_failed", + lastGoodAt, + {}, + failureReason, + ); + } + return signal(id, "ok", "fresh", lastGoodAt); +} + +/** + * The worst account decides. An account without any copy ages from its + * creation, so a new account is not overdue before its first Sunday, and one + * the pass keeps skipping becomes overdue like a stale copy would. + */ +export function evaluateWeeklyCopy( + input: Pick, +): ReadinessSignal { + const id = "weekly_copy"; + const { now } = input; + if (input.weekly.accounts.length === 0) { + return signal(id, "inactive", "no_accounts", null); + } + let worstMs = -Infinity; + let worst: { createdAt: Date; lastCopyAt: Date | null } | null = null; + let overdue = 0; + for (const account of input.weekly.accounts) { + const ref = account.lastCopyAt ?? account.createdAt; + const ageMs = now.getTime() - ref.getTime(); + if (ageMs > WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY_MS) overdue += 1; + if (ageMs > worstMs) { + worstMs = ageMs; + worst = account; + } + } + const days = wholeDays(worstMs); + const at = worst?.lastCopyAt ?? null; + const params = { days, accounts: overdue }; + if (worstMs > WEEKLY_COPY_FAILING_AFTER_DAYS * DAY_MS) { + return signal(id, "failing", at ? "stale" : "never", at, params); + } + if (worstMs > WEEKLY_COPY_ATTENTION_AFTER_DAYS * DAY_MS) { + return signal(id, "attention", at ? "stale" : "never", at, params); + } + // The oldest of the newest copies: the one a restore would be furthest + // behind with. Null only while no account has been copied yet. + const oldestCopy = input.weekly.accounts + .map((a) => a.lastCopyAt) + .filter((d): d is Date => d !== null) + .reduce((min, d) => (min === null || d < min ? d : min), null); + return signal(id, "ok", oldestCopy ? "fresh" : "first_pending", oldestCopy); +} + +/** + * Any account whose newest off-host object is two schedule periods old (the + * freshness module's `stale`, which carries the DST and long-night grace), or + * that a run reached and never put an object there. + */ +export function evaluateOffhostCopies( + input: Pick, +): ReadinessSignal { + const id = "offhost_copies"; + if (!input.offhost.configured) { + return signal(id, "inactive", "not_configured", null); + } + const verdicts: Array<{ + freshness: OffhostBackupFreshness; + lastSuccessAt: Date | null; + }> = input.offhost.accounts.map((a) => ({ + freshness: classifyOffhostBackup({ + lastAttemptAt: a.lastAttemptAt, + lastSuccessAt: a.lastSuccessAt, + now: input.now, + }).freshness, + lastSuccessAt: a.lastSuccessAt, + })); + const oldest = (rows: typeof verdicts) => + rows + .map((r) => r.lastSuccessAt) + .filter((d): d is Date => d !== null) + .reduce( + (min, d) => (min === null || d < min ? d : min), + null, + ); + const behind = verdicts.filter( + (v) => v.freshness === "stale" || v.freshness === "never", + ); + if (behind.length > 0) { + return signal(id, "attention", "stale", oldest(behind), { + accounts: behind.length, + }); + } + const seen = verdicts.filter((v) => v.freshness !== "unknown"); + if (seen.length === 0) return signal(id, "ok", "first_pending", null); + return signal(id, "ok", "fresh", oldest(seen)); +} + +/** A failed drill stays the verdict until a later drill passes. */ +export function evaluateRestoreDrill( + input: Pick, +): ReadinessSignal { + const id = "restore_drill"; + const { drill } = input; + if (!input.offhost.configured) { + return signal(id, "inactive", "not_configured", null); + } + if (drill === null) return signal(id, "ok", "not_run_yet", null); + const failing = + drill.lastFailureAt !== null && + (drill.lastSuccessAt === null || drill.lastFailureAt > drill.lastSuccessAt); + if (failing) { + return signal( + id, + "failing", + // A drill that checked nothing stopped before any object: the bucket + // could not be listed, or held nothing to check. + drill.accountsChecked === 0 ? "failed_run" : "failed", + drill.lastFailureAt, + { failed: drill.accountsFailed, checked: drill.accountsChecked }, + drill.lastFailureReason, + ); + } + return signal(id, "ok", "passed", drill.lastSuccessAt ?? drill.lastRunAt, { + checked: drill.accountsChecked, + }); +} + +export function evaluateOffhostSetup( + input: Pick, +): ReadinessSignal { + const id = "offhost_setup"; + if (input.offhost.configured) return signal(id, "ok", "configured", null); + const graceEnds = setupGraceEndsAt(input.firstAdminAt); + if (graceEnds === null) return signal(id, "inactive", "no_admin", null); + return input.now > graceEnds + ? signal(id, "attention", "not_configured", null) + : signal(id, "ok", "not_configured_grace", graceEnds); +} + +export function evaluateRecoveryKit( + input: Pick, +): ReadinessSignal { + const id = "recovery_kit"; + const { kit } = input; + const keys = kit.unconfirmed.map((k) => k.id).join(", "); + if (kit.unconfirmed.length === 0 && kit.confirmedAt !== null) { + return signal(id, "ok", "confirmed", kit.confirmedAt); + } + if (kit.confirmedAt !== null) { + // Confirmed once, and a key it needs is not in that confirmation: a + // rotation, a new backup key, a re-keyed install. No grace: the keys are + // already in use. + return signal(id, "attention", "out_of_date", kit.confirmedAt, { + keys, + count: kit.unconfirmed.length, + }); + } + const graceEnds = setupGraceEndsAt(input.firstAdminAt); + if (graceEnds === null) return signal(id, "inactive", "no_admin", null); + return input.now > graceEnds + ? signal(id, "attention", "not_confirmed", null, { + keys, + count: kit.unconfirmed.length, + }) + : signal(id, "ok", "not_confirmed_grace", graceEnds, { + keys, + count: kit.unconfirmed.length, + }); +} + +const SEVERITY: Record, number> = { + ok: 0, + attention: 1, + failing: 2, +}; + +export function evaluateRecoveryReadiness( + inputs: ReadinessInputs, +): RecoveryReadiness { + const signals = [ + evaluateDatabaseDump(inputs), + evaluateWeeklyCopy(inputs), + evaluateOffhostCopies(inputs), + evaluateRestoreDrill(inputs), + evaluateOffhostSetup(inputs), + evaluateRecoveryKit(inputs), + ]; + const state = signals.reduce>( + (worst, s) => + s.state !== "inactive" && SEVERITY[s.state] > SEVERITY[worst] + ? s.state + : worst, + "ok", + ); + return { state, evaluatedAt: inputs.now.toISOString(), signals }; +} diff --git a/src/lib/recovery/readiness-notify.ts b/src/lib/recovery/readiness-notify.ts new file mode 100644 index 000000000..4b491c40b --- /dev/null +++ b/src/lib/recovery/readiness-notify.ts @@ -0,0 +1,205 @@ +/** + * Push recovery readiness to every admin: once when a signal turns to + * attention or failing, once when it is back to ok, and a reminder at most + * once a day while it stays failing. Nothing else. + * + * The memory is one row per signal (`RecoveryReadinessNotice`): the state the + * admins were last told about and when. A notice is claimed with a + * compare-and-set on both columns before it is sent, so two workers that + * evaluate in the same minute send it once, and a crash between the claim and + * the send loses a notice rather than repeating it every hour. + * + * Delivery goes through the ordinary localised path, as the deploy-failure + * alert does: every ADMIN, in their own locale, on the channels they enabled + * for `BACKUP_HEALTH`, with the dispatcher's cooldowns and hard-reject rules. + * An admin who switched the event off on a channel is not reached there. + */ +import { prisma } from "@/lib/db"; +import { dispatchLocalisedNotification } from "@/lib/notifications/dispatch-localised"; +import { + READINESS_SIGNALS, + type ReadinessSignal, + type ReadinessSignalId, + type ReadinessState, + type RecoveryReadiness, +} from "@/lib/recovery/readiness-model"; + +/** A failing signal is reminded at most this often. */ +export const FAILING_REMINDER_HOURS = 24; + +type NotifiedState = Exclude; + +export interface StoredNotice { + notifiedState: NotifiedState | null; + notifiedAt: Date | null; +} + +export type NoticeKind = "problem" | "reminder" | "recovered"; + +export interface NoticeDecision { + /** What to send, or null for nothing. */ + send: NoticeKind | null; + /** What to store, or null to leave the row as it is. */ + next: StoredNotice | null; +} + +const NOTHING: NoticeDecision = { send: null, next: null }; + +/** + * Pure: what one evaluation of one signal sends and stores. + * + * - attention / failing, different from what was last announced: send it. + * - failing, as announced, and the last notice is a day old: remind. + * - ok after attention or failing: announce the recovery. + * - inactive: forget silently. A signal that stops applying (the bucket was + * unset, so there is no drill to judge) did not recover, and saying so + * would be untrue; if it applies again and is still broken, it is news. + */ +export function decideNotice( + stored: StoredNotice | null, + current: ReadinessState, + now: Date, +): NoticeDecision { + const last = stored?.notifiedState ?? null; + if (current === "inactive") { + return last === null + ? NOTHING + : { send: null, next: { notifiedState: null, notifiedAt: null } }; + } + if (current === "ok") { + return last === "attention" || last === "failing" + ? { send: "recovered", next: { notifiedState: "ok", notifiedAt: now } } + : NOTHING; + } + if (last !== current) { + return { + send: "problem", + next: { notifiedState: current, notifiedAt: now }, + }; + } + const lastAt = stored?.notifiedAt ?? null; + if ( + current === "failing" && + (lastAt === null || + now.getTime() - lastAt.getTime() >= FAILING_REMINDER_HOURS * 3_600_000) + ) { + return { + send: "reminder", + next: { notifiedState: current, notifiedAt: now }, + }; + } + return NOTHING; +} + +/** The title of a notice, by what it says. */ +function titleKey(kind: NoticeKind, state: ReadinessState): string { + if (kind === "recovered") { + return "notifications.recoveryReadiness.titleRecovered"; + } + if (kind === "reminder") { + return "notifications.recoveryReadiness.titleReminder"; + } + return state === "failing" + ? "notifications.recoveryReadiness.titleFailing" + : "notifications.recoveryReadiness.titleAttention"; +} + +/** + * The body: the same sentence the tile shows for the signal's reason, or the + * signal's own recovery line. + */ +function messageKey(kind: NoticeKind, signal: ReadinessSignal): string { + return kind === "recovered" + ? `admin.recoveryReadiness.recovered.${signal.id}` + : `admin.recoveryReadiness.reason.${signal.id}.${signal.reason}`; +} + +export interface NotifyOutcome { + /** Notices claimed and sent, by signal. */ + sent: Array<{ signal: ReadinessSignalId; kind: NoticeKind }>; + /** Admins each notice went to. */ + admins: number; +} + +function isNotifiedState(value: string | null): value is NotifiedState { + return value === "ok" || value === "attention" || value === "failing"; +} + +/** + * Compare a fresh evaluation against what the admins were last told, claim + * every notice that is due, and send it. + */ +export async function notifyReadinessChanges( + readiness: RecoveryReadiness, + now: Date = new Date(readiness.evaluatedAt), +): Promise { + await prisma.recoveryReadinessNotice.createMany({ + data: READINESS_SIGNALS.map((signal) => ({ signal })), + skipDuplicates: true, + }); + const rows = await prisma.recoveryReadinessNotice.findMany(); + const byId = new Map(rows.map((row) => [row.signal, row])); + const admins = await prisma.user.findMany({ + where: { role: "ADMIN" }, + select: { id: true }, + }); + + const outcome: NotifyOutcome = { sent: [], admins: admins.length }; + for (const signal of readiness.signals) { + const row = byId.get(signal.id); + const applies = signal.state !== "inactive"; + // Since when the signal applies: set once, cleared when it stops. + if (row && applies !== (row.watchedSince !== null)) { + await prisma.recoveryReadinessNotice.updateMany({ + where: { signal: signal.id, watchedSince: row.watchedSince }, + data: { watchedSince: applies ? now : null }, + }); + } + + const stored: StoredNotice | null = row + ? { + notifiedState: isNotifiedState(row.notifiedState) + ? row.notifiedState + : null, + notifiedAt: row.notifiedAt, + } + : null; + const decision = decideNotice(stored, signal.state, now); + // With no admin yet there is nobody to tell, so nothing is claimed: the + // first admin hears about a problem that is still standing. + if (!decision.next || admins.length === 0) continue; + + // Claim: only the evaluation that moves the row from what it read sends. + const claimed = await prisma.recoveryReadinessNotice.updateMany({ + where: { + signal: signal.id, + notifiedState: row?.notifiedState ?? null, + notifiedAt: row?.notifiedAt ?? null, + }, + data: { + notifiedState: decision.next.notifiedState, + notifiedAt: decision.next.notifiedAt, + }, + }); + if (claimed.count !== 1 || decision.send === null) continue; + + for (const admin of admins) { + await dispatchLocalisedNotification({ + userId: admin.id, + eventType: "BACKUP_HEALTH", + titleKey: titleKey(decision.send, signal.state), + messageKey: messageKey(decision.send, signal), + params: signal.params, + metadata: { + source: "recovery-readiness", + url: signal.fixPath, + signal: signal.id, + state: signal.state, + reason: signal.reason, + }, + }); + } + outcome.sent.push({ signal: signal.id, kind: decision.send }); + } + return outcome; +} diff --git a/src/lib/recovery/readiness.ts b/src/lib/recovery/readiness.ts new file mode 100644 index 000000000..3e0c1553d --- /dev/null +++ b/src/lib/recovery/readiness.ts @@ -0,0 +1,117 @@ +/** + * Read every fact the recovery readiness model judges, and judge it. + * + * One reader for both consumers: the admin tile (`GET + * /api/admin/recovery-readiness`) and the hourly evaluation that pushes state + * changes to the admins. Each fact comes from where its own surface already + * reads it, so the tile can never disagree with the backups page about a copy: + * + * - the weekly copy: the `WEEKLY_AUTO` row per account, which never ages out + * of any retention window (pg-boss forgets a run after seven days); + * - the off-host copies: the per-account ledger the nightly run writes, and + * the same "configured" rule the backups page uses (the environment, or a + * ledger row as proof that an upload happened); + * - the drill: its persisted last outcome, with no time window; + * - the kit: the same assessment the kit card shows; + * - the dump: whatever the dump module reports, through a provider. + */ +import { prisma } from "@/lib/db"; +import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; +import { + readDatabaseBackupHealth, + type DatabaseBackupHealthProvider, +} from "@/lib/recovery/database-dump-health"; +import { readRecoveryKit } from "@/lib/recovery/kit"; +import { + evaluateRecoveryReadiness, + type ReadinessInputs, + type RecoveryReadiness, +} from "@/lib/recovery/readiness-model"; + +/** The type the weekly worker writes; anything else was uploaded by hand. */ +const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO"; + +export interface ReadRecoveryReadinessOptions { + now?: Date; + dumpHealth?: DatabaseBackupHealthProvider; +} + +export async function readReadinessInputs( + options: ReadRecoveryReadinessOptions = {}, +): Promise { + const now = options.now ?? new Date(); + const dumpHealth = options.dumpHealth ?? readDatabaseBackupHealth; + const [firstAdmin, accounts, drill, dumpNotice, dump, kit] = + await Promise.all([ + prisma.user.findFirst({ + where: { role: "ADMIN" }, + orderBy: { createdAt: "asc" }, + select: { createdAt: true }, + }), + prisma.user.findMany({ + select: { + createdAt: true, + dataBackups: { + where: { type: SCHEDULED_BACKUP_TYPE }, + select: { createdAt: true }, + }, + offhostBackupState: { + select: { lastAttemptAt: true, lastSuccessAt: true }, + }, + }, + }), + prisma.restoreDrillOutcome.findUnique({ where: { id: "singleton" } }), + prisma.recoveryReadinessNotice.findUnique({ + where: { signal: "database_dump" }, + select: { watchedSince: true }, + }), + dumpHealth(now).catch(() => "unreadable" as const), + readRecoveryKit(now), + ]); + + // Evidence beats this process's environment, as on the backups page: a + // ledger row is proof an upload happened, whichever process reads it. + const configured = + offhostBackupConfigured() || + accounts.some((account) => account.offhostBackupState !== null); + + return { + now, + firstAdminAt: firstAdmin?.createdAt ?? null, + dump, + dumpWatchedSince: dumpNotice?.watchedSince ?? null, + weekly: { + accounts: accounts.map((account) => ({ + createdAt: account.createdAt, + lastCopyAt: account.dataBackups[0]?.createdAt ?? null, + })), + }, + offhost: { + configured, + accounts: accounts.map((account) => ({ + lastAttemptAt: account.offhostBackupState?.lastAttemptAt ?? null, + lastSuccessAt: account.offhostBackupState?.lastSuccessAt ?? null, + })), + }, + drill: drill + ? { + lastRunAt: drill.lastRunAt, + lastSuccessAt: drill.lastSuccessAt, + lastFailureAt: drill.lastFailureAt, + lastFailureReason: drill.lastFailureReason, + accountsChecked: drill.accountsChecked, + accountsFailed: drill.accountsFailed, + } + : null, + kit: { + confirmedAt: kit.confirmedAt ? new Date(kit.confirmedAt) : null, + unconfirmed: kit.unconfirmed, + }, + }; +} + +export async function readRecoveryReadiness( + options: ReadRecoveryReadinessOptions = {}, +): Promise { + return evaluateRecoveryReadiness(await readReadinessInputs(options)); +} diff --git a/tests/integration/recovery-readiness-notify.test.ts b/tests/integration/recovery-readiness-notify.test.ts new file mode 100644 index 000000000..2f199f482 --- /dev/null +++ b/tests/integration/recovery-readiness-notify.test.ts @@ -0,0 +1,295 @@ +/** + * Recovery readiness, end to end against Postgres: a weekly copy that stopped + * reaches every admin exactly once per state change, through the real + * readiness reader, the real notice ledger, the real dispatcher with its + * preference lookup and ledger, and the real APNs sender (vendor client + * stubbed, the shape `delegated-intake-notification.test.ts` uses). + * + * The facts: + * + * 1. A stale copy becomes one `BACKUP_HEALTH` delivery per admin; the next + * hourly runs at the same state add none. + * 2. Escalating to failing adds one; a reminder comes a day later and not + * sooner; the recovery adds one; ok after that adds none. + * 3. An admin who switched the event off on the channel gets nothing, and + * the notice is still spent, so switching it back on later does not + * replay an old alarm. + * 4. The hourly job runs the same path from its handler. + * + * The ledger write is fire-and-forget in `recordPushAttempt`, so counts are + * polled up to the expected number and then asserted exactly. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +const apnsSendMock = vi.fn(); + +vi.mock("@parse/node-apn", () => { + class Provider { + constructor(_opts: unknown) { + void _opts; + } + send(...args: unknown[]) { + return apnsSendMock(...args); + } + shutdown() {} + } + class Notification { + public topic = ""; + public alert: unknown; + public sound: string | undefined; + public badge: number | undefined; + public threadId: string | undefined; + public collapseId: string | undefined; + public payload: unknown; + } + return { default: { Provider, Notification }, Provider, Notification }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const ADMIN_ID = "user-readiness-admin"; +const SECOND_ADMIN_ID = "user-readiness-admin-2"; +const MEMBER_ID = "user-readiness-member"; +const DAY = 86_400_000; +const HOUR = 3_600_000; + +const TEST_EC_PEM_LINES = [ + "-----BEGIN PRIVATE KEY-----", + "MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQgLOXP3Exjr5L5tamN", + "pTxck85Iaum80PdRlWDpc/ezviOgCgYIKoZIzj0DAQehRANCAAT1x8nKRb8KshQU", + "1aPieSCqOY6ilgC959umaFSlhfav8eZ91UHP/xond9aMoZcuQ7lJG/Rsj70SWMvZ", + "bw81BG89", + "-----END PRIVATE KEY-----", +]; + +const APNS_ENV = { + APNS_KEY_ID: "ABCDE12345", + APNS_TEAM_ID: "TEAM123456", + APNS_BUNDLE_ID: "test.healthlog.ios", + APNS_KEY: TEST_EC_PEM_LINES.join("\\n"), +}; + +async function seedApnsChannel(userId: string): Promise { + const { encrypt } = await import("@/lib/crypto"); + const channel = await getPrismaClient().notificationChannel.create({ + data: { userId, type: "APNS", enabled: true, config: encrypt("{}") }, + }); + await getPrismaClient().device.create({ + data: { + userId, + platform: "ios", + token: `device-${userId}`, + bundleId: APNS_ENV.APNS_BUNDLE_ID, + apnsToken: `apns-${userId}`, + apnsEnvironment: "sandbox", + }, + }); + return channel.id; +} + +async function backupAttempts( + userId: string, + expected: number, +): Promise { + const count = () => + getPrismaClient().pushAttempt.count({ + where: { userId, eventType: "BACKUP_HEALTH" }, + }); + for (let attempt = 0; attempt < 40; attempt += 1) { + if ((await count()) >= expected) break; + await new Promise((r) => setTimeout(r, 25)); + } + // A short settle, so a delivery that should NOT have happened has the same + // chance to land as one that should. + await new Promise((r) => setTimeout(r, 75)); + return count(); +} + +/** Put the member's weekly copy `ageDays` before `now`. */ +async function weeklyCopyAged(ageDays: number, now: Date): Promise { + await getPrismaClient().dataBackup.upsert({ + where: { userId_type: { userId: MEMBER_ID, type: "WEEKLY_AUTO" } }, + create: { + userId: MEMBER_ID, + type: "WEEKLY_AUTO", + data: "sealed", + createdAt: new Date(now.getTime() - ageDays * DAY), + }, + update: { createdAt: new Date(now.getTime() - ageDays * DAY) }, + }); +} + +/** One hourly evaluation at `now`, through the reader and the notifier. */ +async function evaluateAt(now: Date) { + const { readRecoveryReadiness } = await import("@/lib/recovery/readiness"); + const { notifyReadinessChanges } = + await import("@/lib/recovery/readiness-notify"); + const readiness = await readRecoveryReadiness({ now }); + await notifyReadinessChanges(readiness, now); + return readiness; +} + +beforeEach(async () => { + vi.clearAllMocks(); + for (const [k, v] of Object.entries(APNS_ENV)) process.env[k] = v; + delete process.env.APNS_PRODUCTION; + for (const name of [ + "BACKUP_S3_ENDPOINT", + "BACKUP_S3_BUCKET", + "BACKUP_S3_ACCESS_KEY", + "BACKUP_S3_SECRET_KEY", + "BACKUP_ENCRYPTION_KEY", + ]) { + delete process.env[name]; + } + const apns = await import("@/lib/notifications/senders/apns"); + apns.resetApnsForTesting(); + const { __resetDispatchLocaleCacheForTests } = + await import("@/lib/notifications/dispatch-localised"); + __resetDispatchLocaleCacheForTests(); + + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + // Admins created "now": the setup grace keeps the off-host and kit signals + // quiet, so the weekly copy is the only signal that moves. + await prisma.user.createMany({ + data: [ + { + id: ADMIN_ID, + username: "readiness-admin", + email: "readiness-admin@example.test", + role: "ADMIN", + locale: "en", + }, + { + id: SECOND_ADMIN_ID, + username: "readiness-admin-2", + email: "readiness-admin-2@example.test", + role: "ADMIN", + locale: "de", + }, + { + id: MEMBER_ID, + username: "readiness-member", + email: "readiness-member@example.test", + createdAt: new Date(Date.now() - 90 * DAY), + }, + ], + }); + // An admin's own account needs a fresh copy too, or it would age from its + // creation and move the verdict on its own. + const now = new Date(); + for (const userId of [ADMIN_ID, SECOND_ADMIN_ID]) { + await prisma.dataBackup.create({ + data: { userId, type: "WEEKLY_AUTO", data: "sealed", createdAt: now }, + }); + } + + apnsSendMock.mockImplementation(async (note: unknown) => { + void note; + return { sent: [{ device: "x" }], failed: [] }; + }); +}); + +describe("recovery readiness push", () => { + it("notifies every admin once per state change, reminds daily while failing, and announces the recovery once", async () => { + await seedApnsChannel(ADMIN_ID); + await seedApnsChannel(SECOND_ADMIN_ID); + const t0 = new Date(); + const at = (hours: number) => new Date(t0.getTime() + hours * HOUR); + + // Ten days: attention. + await weeklyCopyAged(10, t0); + const first = await evaluateAt(t0); + expect(first.signals.find((s) => s.id === "weekly_copy")).toMatchObject({ + state: "attention", + reason: "stale", + }); + expect(first.signals.find((s) => s.id === "recovery_kit")?.state).toBe( + "ok", + ); + await evaluateAt(at(1)); + await evaluateAt(at(2)); + expect(await backupAttempts(ADMIN_ID, 1)).toBe(1); + expect(await backupAttempts(SECOND_ADMIN_ID, 1)).toBe(1); + + // Each admin in their own language. + const titles = apnsSendMock.mock.calls.map( + ([note]) => (note as { alert: { title: string } }).alert.title, + ); + expect(titles).toContain("Backups need attention"); + expect(titles).toContain("Sicherungen brauchen Aufmerksamkeit"); + + // Fifteen days: failing, once. + await weeklyCopyAged(15, at(3)); + await evaluateAt(at(3)); + await evaluateAt(at(4)); + expect(await backupAttempts(ADMIN_ID, 2)).toBe(2); + + // Still failing: nothing until a day after the last notice, then one. + await evaluateAt(at(3 + 23)); + expect(await backupAttempts(ADMIN_ID, 3)).toBe(2); + await evaluateAt(at(3 + 24)); + await evaluateAt(at(3 + 25)); + expect(await backupAttempts(ADMIN_ID, 3)).toBe(3); + + // A fresh copy: one recovery notice, then silence. + await weeklyCopyAged(0, at(30)); + await evaluateAt(at(30)); + await evaluateAt(at(31)); + expect(await backupAttempts(ADMIN_ID, 4)).toBe(4); + expect(await backupAttempts(SECOND_ADMIN_ID, 4)).toBe(4); + + const notice = await getPrismaClient().recoveryReadinessNotice.findUnique({ + where: { signal: "weekly_copy" }, + }); + expect(notice?.notifiedState).toBe("ok"); + }); + + it("respects an admin's per-channel opt-out and still spends the notice", async () => { + const channelId = await seedApnsChannel(ADMIN_ID); + await getPrismaClient().notificationPreference.create({ + data: { channelId, eventType: "BACKUP_HEALTH", enabled: false }, + }); + const t0 = new Date(); + await weeklyCopyAged(10, t0); + await evaluateAt(t0); + expect(await backupAttempts(ADMIN_ID, 1)).toBe(0); + + await getPrismaClient().notificationPreference.updateMany({ + where: { channelId }, + data: { enabled: true }, + }); + await evaluateAt(new Date(t0.getTime() + HOUR)); + expect(await backupAttempts(ADMIN_ID, 1)).toBe(0); + const notice = await getPrismaClient().recoveryReadinessNotice.findUnique({ + where: { signal: "weekly_copy" }, + }); + expect(notice?.notifiedState).toBe("attention"); + }); + + it("runs the same path from the hourly job", async () => { + await seedApnsChannel(ADMIN_ID); + await weeklyCopyAged(10, new Date()); + const { handleRecoveryReadiness } = + await import("@/lib/jobs/recovery-readiness"); + const outcome = await handleRecoveryReadiness([]); + expect(outcome).toMatchObject({ + ok: true, + did: { + recovery_readiness_state: "attention", + recovery_readiness_notices: 1, + recovery_readiness_admins: 2, + }, + }); + await handleRecoveryReadiness([]); + expect(await backupAttempts(ADMIN_ID, 1)).toBe(1); + }); +}); From ed51211e820e819f5fa0b2207dbb43d769725400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 13:32:43 +0100 Subject: [PATCH 18/44] docs(self-hosting): describe the recovery kit and the readiness push The key backup sections become "Save the recovery kit": which keys the kit lists and why, the fingerprint of the token key, and the readiness card with its Backup health notification. --- docs/self-hosting/getting-started.md | 40 +++++++++++++++++++++------- docs/self-hosting/truenas.md | 19 ++++++------- docs/self-hosting/unraid.md | 14 +++++----- 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/docs/self-hosting/getting-started.md b/docs/self-hosting/getting-started.md index f4fb5d208..c4c337468 100644 --- a/docs/self-hosting/getting-started.md +++ b/docs/self-hosting/getting-started.md @@ -270,14 +270,35 @@ ps` — the `app` healthcheck polls `/api/health` every 30 seconds `pull_policy: always` on the `app` service, and verify the running build with `GET /api/version` (it returns `version` + `buildSha`). -## Back up your encryption key +## Save the recovery kit -`ENCRYPTION_KEY` (or the `ENCRYPTION_KEYS` map) is not stored in the database. -Without it, the encrypted parts of the database and every in-database backup -cannot be read. Keep a copy outside the server, then confirm it under Admin, -Encryption: the step shows the key id and a fingerprint (the first 12 hex -characters of SHA-256 over the key) and can check a pasted copy without -storing it. Admins see a dashboard reminder until the active key is confirmed. +No key HealthLog encrypts with is stored in the database. Without them the +encrypted parts of the database, a database dump and every backup copy cannot +be read. The recovery kit under Admin, Encryption lists every key a restore of +this server needs: + +- `ENCRYPTION_KEY` (or each id in the `ENCRYPTION_KEYS` map), including a + retired id for as long as a stored or off-host backup copy still needs it; +- `BACKUP_ENCRYPTION_KEY` and any `BACKUP_ENCRYPTION_PREVIOUS_KEYS`, when + off-host copies are set up; +- `API_TOKEN_HMAC_KEY`, without which every session, token and share link ends + (no data is lost). + +Each key is shown by id and a fingerprint (the first 12 hex characters of +SHA-256 over the key), never the key itself, and **Check my copy** compares a +pasted copy without storing it. Keep a copy of each outside the server, then +confirm the kit. Admins see a dashboard reminder until it is confirmed, and the +kit is due again whenever a key it needs changes: a rotation, a new backup +key, a re-keyed install. The kit also links the +[disaster recovery guide](../ops/disaster-recovery.md). + +The **Recovery readiness** card on the admin overview and the Backups page +judges every backup signal in one place: the database dump, the weekly account +copies, the off-host copies, the last restore drill, whether off-host copies +are set up, and the recovery kit. An hourly check sends each admin a +**Backup health** notification when a signal turns to "needs attention" or "at +risk", once a day while it stays at risk, and once when it recovers. Route or +switch it off per channel under Notifications. | Platform | Where the key is set | | --------- | -------------------------------------------------------------------------------------------------- | @@ -288,12 +309,13 @@ storing it. Admins see a dashboard reminder until the active key is confirmed. | Portainer | the stack's Environment variables | Set `HEALTHLOG_PLATFORM` (`compose`, `truenas`, `coolify`, `unraid`, -`portainer`) to open the step on your platform's instructions. +`portainer`) to open the kit on your platform's instructions. -To compute the fingerprint of your copy: +To compute the fingerprint of your copy of a hex key, and of the token key: ```bash echo -n "" | xxd -r -p | sha256sum | cut -c1-12 +printf %s "" | sha256sum | cut -c1-12 ``` **HealthLog refuses to serve: encryption key does not match this database.** diff --git a/docs/self-hosting/truenas.md b/docs/self-hosting/truenas.md index d312e4fb7..422efd68a 100644 --- a/docs/self-hosting/truenas.md +++ b/docs/self-hosting/truenas.md @@ -20,7 +20,7 @@ database dataset and in one value you must keep yourself: the encryption key. ``` Each run prints 64 hex characters. Put both in your password manager - before you continue; see [Back up your encryption key](#back-up-your-encryption-key). + before you continue; see [Save the recovery kit](#save-the-recovery-kit). 2. Open Apps, Discover Apps, search for **HealthLog** and choose Install. 3. Fill in the form: @@ -61,7 +61,7 @@ and the login page says so before you type a password: "Signing in cannot complete on this page". Turn it on only when HealthLog is served over `https://`, and set App URL to that `https://` address. -## Back up your encryption key +## Save the recovery kit HealthLog encrypts tokens, notes, documents and in-database backups with the Encryption Key. The key is not stored in the database. If the app is deleted @@ -75,9 +75,10 @@ password manager or another place outside the NAS. 1. Open Apps, select HealthLog and choose Edit. The key is the Encryption Key field under HealthLog Configuration. -2. Sign in as the admin and open Admin, Encryption. The **Back up your - encryption key** step shows the active key id and a fingerprint, the first - 12 hex characters of SHA-256 over the key. Compare it with your copy: +2. Sign in as the admin and open Admin, Encryption. The **Recovery kit** lists + every key a restore needs (the Encryption Key, the token key and, with + off-host copies, the backup key), each with an id and a fingerprint, the + first 12 hex characters of SHA-256 over the key. Compare it with your copy: ```bash echo -n "" | xxd -r -p | sha256sum | cut -c1-12 @@ -86,10 +87,10 @@ password manager or another place outside the NAS. or paste the copy into **Check my copy**. It is compared and dropped, never stored or logged. -3. Press **I have backed up this key**. Until you do, admins see a reminder on - the dashboard. After a key rotation the step comes back for the new key. +3. Press **I have a copy of every key**. Until you do, admins see a reminder on + the dashboard. After a key rotation or a new backup key the kit is due again. -The catalog app sets `HEALTHLOG_PLATFORM=truenas`, so the step opens on these +The catalog app sets `HEALTHLOG_PLATFORM=truenas`, so the kit opens on these instructions. ## Backups and restore @@ -224,7 +225,7 @@ the menu next to Custom App, Install via YAML, using these values: | `NEXT_PUBLIC_APP_URL` | yes | the address people open, for example `http://truenas.local:30180` | | `APP_URL` | yes | the same address | | `SESSION_COOKIE_SECURE` | no | `false` for a LAN install served over plain `http://`; leave unset behind an HTTPS proxy | -| `HEALTHLOG_PLATFORM` | no | `truenas`: the admin's key backup step opens on the TrueNAS instructions | +| `HEALTHLOG_PLATFORM` | no | `truenas`: the admin's recovery kit opens on the TrueNAS instructions | `HEALTHLOG_PROCESS_TYPE` stays unset (`all`): one container serves the web app and runs the background jobs. In a YAML install the key sits in the diff --git a/docs/self-hosting/unraid.md b/docs/self-hosting/unraid.md index 87c0e91df..9f861bcba 100644 --- a/docs/self-hosting/unraid.md +++ b/docs/self-hosting/unraid.md @@ -123,13 +123,15 @@ Set `SESSION_COOKIE_SECURE=true` (or leave it unset) only when a TLS reverse proxy serves HealthLog over HTTPS. In that case also point `NEXT_PUBLIC_APP_URL` and `APP_URL` at the public `https://` URL. -## Back up your encryption key +## Save the recovery kit -Copy `ENCRYPTION_KEY` from the template into your password manager, then -confirm it under Admin, Encryption (set `HEALTHLOG_PLATFORM=unraid` so the step -opens on the Unraid instructions). A reinstall with a new key over the old -database makes the data unreadable; HealthLog then refuses to serve and says -so. See [Getting started](getting-started.md#back-up-your-encryption-key). +Copy `ENCRYPTION_KEY`, `API_TOKEN_HMAC_KEY` and, with off-host copies, +`BACKUP_ENCRYPTION_KEY` from the template into your password manager, then +confirm the recovery kit under Admin, Encryption (set +`HEALTHLOG_PLATFORM=unraid` so the kit opens on the Unraid instructions). A +reinstall with a new key over the old database makes the data unreadable; +HealthLog then refuses to serve and says so. See +[Getting started](getting-started.md#save-the-recovery-kit). ## Updates From 82db3de30a41093a1dc4d6babce3b789e5d78a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 13:24:49 +0100 Subject: [PATCH 19/44] feat(backup): restore a dump straight from the off-host bucket A lost host takes its local dumps with it, and the copy left is the one in the bucket. `docker compose run --rm db-restore latest --from-bucket` now lists `database/` in the bucket the BACKUP_S3_* variables name, takes the newest dump whose manifest is there too (or the one named), downloads both with the S3 client the image already ships, and continues as for any off-host copy: opened with BACKUP_ENCRYPTION_KEY, checked against the manifest, restored into an empty database. `list --from-bucket` shows what is there. The sealed download is removed once it is opened. The image build now proves the S3 client resolves from the shared module's place in the tree, so the restore cannot lose it unnoticed. --- Dockerfile | 6 +- docker-compose.yml | 11 +- scripts/__tests__/db-restore.test.ts | 6 + scripts/db-restore.mjs | 92 +++++++++++- .../__tests__/dump-core.test.ts | 59 ++++++++ src/lib/database-backup/dump-core.mjs | 134 ++++++++++++++++++ 6 files changed, 298 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 033b08005..7c07c594d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -265,9 +265,11 @@ RUN node scripts/reset-password.mjs 2>&1 | grep -q "reset-password: usage:" RUN env -u DATABASE_URL node scripts/pending-migrations.mjs && \ node --input-type=module -e "await import('pg')" && \ node --input-type=module -e "await import('./scripts/migrate-deploy.mjs')" -# The dump and restore scripts load, and the client tools they run are the -# major the bundled database runs. +# The dump and restore scripts load, the client tools they run are the +# major the bundled database runs, and the S3 client `db-restore --from-bucket` +# loads resolves from the shared module's place in the tree. RUN node scripts/db-dump.mjs 2>&1 | grep -q "db-dump: usage:" && \ + node --input-type=module -e "import { createRequire } from 'node:module'; createRequire('/app/src/lib/database-backup/dump-core.mjs').resolve('@aws-sdk/client-s3')" && \ node scripts/db-restore.mjs 2>&1 | grep -q "db-restore: usage:" && \ pg_dump --version | grep -q "(PostgreSQL) 18\." && \ pg_restore --version | grep -q "(PostgreSQL) 18\." diff --git a/docker-compose.yml b/docker-compose.yml index 676522118..19ce9acec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -806,15 +806,18 @@ services: # docker compose run --rm db-restore list # docker compose run --rm db-restore latest # docker compose run --rm db-restore healthlog-