From 28340e7c66210f2a47b29f53238d472c83904152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:42:37 +0100 Subject: [PATCH 01/19] fix(coach): restore every step a turn records on reload The turn records up to 64 steps while the read path kept only the first 48, so the tail of a long answer's step list vanished on reload. Both ends now read one MAX_TURN_STEPS constant, and a unit plus an integration test round-trip a full 64-step answer through the stored provenance. --- .../baselines/version-stamp-comments.json | 4 +- .../ai/coach/__tests__/persistence.test.ts | 43 ++++++++- src/lib/ai/coach/activity/contract.ts | 7 ++ src/lib/ai/coach/persistence.ts | 15 ++- src/lib/ai/coach/turn/__tests__/steps.test.ts | 3 +- src/lib/ai/coach/turn/steps.ts | 8 +- .../coach-steps-round-trip.test.ts | 96 +++++++++++++++++++ 7 files changed, 157 insertions(+), 19 deletions(-) create mode 100644 tests/integration/coach-steps-round-trip.test.ts diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 81a01b793..5a3506c2e 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2897, - "total": 9249, + "total": 9247, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -1737,7 +1737,7 @@ "lib/ai/coach/turn/reply-guards.ts": 18, "lib/ai/coach/turn/reuse-turn.ts": 1, "lib/ai/coach/turn/sse.ts": 10, - "lib/ai/coach/turn/steps.ts": 3, + "lib/ai/coach/turn/steps.ts": 1, "lib/ai/coach/turn/types.ts": 17, "lib/ai/coach/types.ts": 58, "lib/ai/coach/visits-snapshot.ts": 1, diff --git a/src/lib/ai/coach/__tests__/persistence.test.ts b/src/lib/ai/coach/__tests__/persistence.test.ts index 5e555e918..0075ef193 100644 --- a/src/lib/ai/coach/__tests__/persistence.test.ts +++ b/src/lib/ai/coach/__tests__/persistence.test.ts @@ -27,7 +27,8 @@ import { summariseTitle, } from "../persistence"; import { encryptToBytes } from "../bytes-codec"; -import type { CoachProvenance } from "../types"; +import { MAX_TURN_STEPS } from "../activity/contract"; +import type { CoachProvenance, CoachStep } from "../types"; describe("summariseTitle", () => { it("returns the input unchanged when below 80 chars", () => { @@ -212,6 +213,46 @@ describe("appendMessage — provenance round-trip", () => { expect(out.metricSource).toEqual(envelope); }); + it("restores every step the turn may record, and none past the cap", async () => { + const step = (n: number): CoachStep => ({ + id: `s${n}`, + tool: "get_metric_series", + labelKey: "coach.step.readWindow", + label: "Read the series", + domain: "bp", + window: "last30days", + status: "done", + count: n, + }); + const recorded = Array.from({ length: MAX_TURN_STEPS }, (_, i) => + step(i + 1), + ); + stubEchoCreate(); + const full = await appendMessage({ + conversationId: "conv_1", + role: "assistant", + content: "A long answer.", + metricSource: { windows: [], metrics: [], steps: recorded }, + }); + expect(full.metricSource?.steps).toEqual(recorded); + + // A stored blob longer than the cap is cut at the cap, not before it. + stubEchoCreate( + JSON.stringify({ + windows: [], + metrics: [], + steps: [...recorded, step(MAX_TURN_STEPS + 1)], + }), + ); + const over = await appendMessage({ + conversationId: "conv_1", + role: "assistant", + content: "An over-long blob.", + metricSource: { windows: [], metrics: [] }, + }); + expect(over.metricSource?.steps).toEqual(recorded); + }); + it("tolerates a legacy blob without the new fields (no throw, fields absent)", async () => { // A pre-feature row: only the original windows/metrics envelope on disk. stubEchoCreate( diff --git a/src/lib/ai/coach/activity/contract.ts b/src/lib/ai/coach/activity/contract.ts index cca931799..d4b4d65e2 100644 --- a/src/lib/ai/coach/activity/contract.ts +++ b/src/lib/ai/coach/activity/contract.ts @@ -41,6 +41,13 @@ export const ACTIVITY_TEXT_MAX_CHARS = 400; /** `a1`..`a99`: the 100th entry of a turn is not recorded. */ export const ACTIVITY_MAX_ENTRIES = 99; +/** + * `s1`..`s64`: the live steps of one turn (four per round of the longest + * turn). Later tool calls still run but show no step. The turn records up to + * this many and a reload restores up to this many, from this one constant. + */ +export const MAX_TURN_STEPS = 64; + export interface ActivityStart { phase: CoachActivityPhase; round: number; diff --git a/src/lib/ai/coach/persistence.ts b/src/lib/ai/coach/persistence.ts index 787d4e09f..a40ffd939 100644 --- a/src/lib/ai/coach/persistence.ts +++ b/src/lib/ai/coach/persistence.ts @@ -28,7 +28,11 @@ import { coachStopSchema, coachTrailSchema, } from "./stream-events"; -import { ACTIVITY_MAX_ENTRIES, TRAIL_MAX_BYTES } from "./activity/contract"; +import { + ACTIVITY_MAX_ENTRIES, + MAX_TURN_STEPS, + TRAIL_MAX_BYTES, +} from "./activity/contract"; import { commitTurnWrites } from "./memory/turn-writes"; import { COACH_CONVERSATION_TITLE_MAX } from "./types"; import { isRedundantViewChip } from "./follow-ups/view-chip"; @@ -56,7 +60,6 @@ import type { CoachResultEntry, CoachResultMeta, CoachResultTable, - CoachStep, CoachStepDomain, CoachSuggestion, CoachTrail, @@ -225,11 +228,6 @@ function restoreOne(raw: unknown, schema: z.ZodType): T | undefined { return parsed.success ? parsed.data : undefined; } -/** The live steps a turn persisted (at most 48 per turn since v1.41). */ -function restoreSteps(raw: unknown): CoachStep[] | undefined { - return restoreEach(raw, coachStepSchema, 48); -} - /** The method line; dropped whole when it does not parse. */ function restoreMethod(raw: unknown): CoachMethod | undefined { const parsed = coachMethodSchema.safeParse(raw); @@ -341,7 +339,8 @@ function provenanceFromJson(raw: string | null): CoachProvenance | null { const toolCalls = restoreToolCalls(parsed.toolCalls); // v1.39.4 — the dialog fields: steps, method, table metadata, chips, the // clarification choices, and the forced-answer marker. - const steps = restoreSteps(parsed.steps); + // Up to the same cap the turn records steps under. + const steps = restoreEach(parsed.steps, coachStepSchema, MAX_TURN_STEPS); const method = restoreMethod(parsed.method); const results = restoreResultMetas(parsed.results); // A reply stored before view chips were withheld for a table the answer diff --git a/src/lib/ai/coach/turn/__tests__/steps.test.ts b/src/lib/ai/coach/turn/__tests__/steps.test.ts index b5453ed75..06a297fbf 100644 --- a/src/lib/ai/coach/turn/__tests__/steps.test.ts +++ b/src/lib/ai/coach/turn/__tests__/steps.test.ts @@ -34,7 +34,8 @@ import { } from "@/lib/ai/coach/tools/definitions"; import type { CoachToolResult } from "@/lib/ai/coach/tools/run/types"; -import { MAX_TURN_STEPS, snapshotStep, toStep } from "../steps"; +import { MAX_TURN_STEPS } from "@/lib/ai/coach/activity/contract"; +import { snapshotStep, toStep } from "../steps"; const SOURCES = coachScopeSourceSchema.options; const WINDOWS = coachScopeWindowSchema.options; diff --git a/src/lib/ai/coach/turn/steps.ts b/src/lib/ai/coach/turn/steps.ts index 87aa40787..09b6153df 100644 --- a/src/lib/ai/coach/turn/steps.ts +++ b/src/lib/ai/coach/turn/steps.ts @@ -38,13 +38,7 @@ import { } from "@/lib/ai/coach/tools/definitions"; import { isTurnResultRef } from "@/lib/ai/coach/results/refs"; import { COMPARE_SERIES_TOOL_NAME } from "@/lib/ai/coach/tools/compare-series"; - -/** - * At most this many steps per turn; later calls run but show no step. - * v1.41 — four per round of the longest turn: 48 for twelve rounds, 64 since - * v1.41.2 raised the cap to sixteen. - */ -export const MAX_TURN_STEPS = 64; +import { MAX_TURN_STEPS } from "@/lib/ai/coach/activity/contract"; const SCOPE_SOURCES: ReadonlySet = new Set( coachScopeSourceSchema.options, diff --git a/tests/integration/coach-steps-round-trip.test.ts b/tests/integration/coach-steps-round-trip.test.ts new file mode 100644 index 000000000..0abdf9906 --- /dev/null +++ b/tests/integration/coach-steps-round-trip.test.ts @@ -0,0 +1,96 @@ +/** + * The steps of a long Coach answer survive a reload against a real Postgres. + * + * The turn records up to `MAX_TURN_STEPS` steps on `metricSource.steps`; the + * read path restores them from `coach_messages.metric_source_json`. Both ends + * read the one constant, so a full turn's worth comes back whole and an + * over-long stored blob is cut at that cap, not before it. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { MAX_TURN_STEPS } from "@/lib/ai/coach/activity/contract"; +import { + appendMessage, + createConversation, + fetchConversationWithMessages, +} from "@/lib/ai/coach/persistence"; +import type { CoachStep } from "@/lib/ai/coach/types"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); +}); + +function step(n: number): CoachStep { + return { + id: `s${n}`, + tool: "get_metric_series", + labelKey: "coach.step.readWindow", + label: "Read the last 30 days", + domain: "bp", + window: "last30days", + status: "done", + count: n, + }; +} + +const RECORDED = Array.from({ length: MAX_TURN_STEPS }, (_, i) => step(i + 1)); + +async function seedConversation(): Promise<{ + userId: string; + conversationId: string; +}> { + const user = await getPrismaClient().user.create({ + data: { username: "steps-owner", role: "USER" }, + }); + const convo = await createConversation({ userId: user.id, title: "long" }); + return { userId: user.id, conversationId: convo.id }; +} + +describe("Coach steps across a reload", () => { + it("restores every step of a full turn", async () => { + const { userId, conversationId } = await seedConversation(); + const reply = await appendMessage({ + conversationId, + role: "assistant", + content: "A long answer.", + metricSource: { windows: [], metrics: [], steps: RECORDED }, + }); + + const reloaded = await fetchConversationWithMessages( + userId, + conversationId, + ); + const message = reloaded?.messages.find((m) => m.id === reply.id); + expect(message?.metricSource?.steps).toHaveLength(MAX_TURN_STEPS); + expect(message?.metricSource?.steps).toEqual(RECORDED); + }); + + it("cuts an over-long stored blob at the cap, not before it", async () => { + const { userId, conversationId } = await seedConversation(); + const reply = await appendMessage({ + conversationId, + role: "assistant", + content: "An over-long blob.", + metricSource: { windows: [], metrics: [] }, + }); + await getPrismaClient().coachMessage.update({ + where: { id: reply.id }, + data: { + metricSourceJson: JSON.stringify({ + windows: [], + metrics: [], + steps: [...RECORDED, step(MAX_TURN_STEPS + 1)], + }), + }, + }); + + const reloaded = await fetchConversationWithMessages( + userId, + conversationId, + ); + const message = reloaded?.messages.find((m) => m.id === reply.id); + expect(message?.metricSource?.steps).toEqual(RECORDED); + }); +}); From 856eb036f83a284e515aafbc23e7bf40de95bb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:44:42 +0100 Subject: [PATCH 02/19] fix(documents): point the on-device scan hint at the Labs switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hint shown when a text-only provider needs on-device text recognition sent people to the AI settings, where no such switch exists. It now names the switch by its label under Settings → Modules → Labs in every locale and links straight to that card. Inside somebody else's record the link is withheld, since Settings there is the caller's own and closed; the sentence stays. --- messages/de.json | 5 +- messages/en.json | 5 +- messages/es.json | 5 +- messages/fr.json | 5 +- messages/it.json | 5 +- messages/ko.json | 5 +- messages/nl.json | 5 +- messages/pl.json | 5 +- messages/pt-BR.json | 5 +- .../__tests__/document-ai-panels.test.tsx | 29 ++++++++++-- .../__tests__/document-ai-section.test.tsx | 7 ++- .../documents/document-ai-panels.tsx | 47 +++++++++++++------ 12 files changed, 90 insertions(+), 38 deletions(-) diff --git a/messages/de.json b/messages/de.json index cfe7abcc4..60821114e 100644 --- a/messages/de.json +++ b/messages/de.json @@ -11769,8 +11769,9 @@ "dismiss": "Verwerfen", "empty": "Aus diesem Dokument konnte nichts gelesen werden.", "unavailableBody": "Richte einen KI-Anbieter ein, um Titel, Typ und Datum vorzuschlagen.", - "unavailableLocalOcr": "Aktiviere lokale Texterkennung in den KI-Einstellungen, um Bilder auf diesem Gerät zu lesen.", + "unavailableLocalOcr": "Schalte „Scans auf diesem Gerät lesen“ unter Einstellungen → Module → Labor ein, um Bilder auf diesem Gerät zu lesen.", "unavailableAction": "KI-Einstellungen öffnen", + "unavailableLocalOcrAction": "Labor-Einstellungen öffnen", "errorGeneric": "Dokument konnte nicht gelesen werden. Bitte erneut versuchen.", "errorOcr": "Aus diesem Bild konnte kein Text gelesen werden.", "errorRateLimited": "Das stündliche KI-Leselimit ist erreicht. Bitte später erneut versuchen.", @@ -11782,7 +11783,7 @@ "errorFileType": "Dieser Dateityp kann nicht gelesen werden.", "errorNotIndexed": "Lies das Dokument zuerst aus, dann extrahiere.", "errorOutputLimit": "Der KI-Anbieter hat sein Ausgabelimit erreicht, bevor er einen einzigen Wert geliefert hat. Versuche ein Modell, das längere Antworten erlaubt.", - "errorLocalOcrDisabled": "Aktiviere zuerst die lokale Texterkennung in den KI-Einstellungen.", + "errorLocalOcrDisabled": "Schalte zuerst „Scans auf diesem Gerät lesen“ unter Einstellungen → Module → Labor ein.", "errorProvider": "Dafür ist kein KI-Anbieter eingerichtet.", "errorTextImageOnly": "Lokale Texterkennung liest nur Bilder, nicht diese Datei.", "errorConsent": "Für das Lesen von Dokumenten mit KI brauchst du zuerst deine Einwilligung.", diff --git a/messages/en.json b/messages/en.json index 9e7fa4317..53b89951f 100644 --- a/messages/en.json +++ b/messages/en.json @@ -11769,8 +11769,9 @@ "dismiss": "Dismiss", "empty": "Nothing could be read from this document.", "unavailableBody": "Set up an AI provider to suggest a title, type, and date.", - "unavailableLocalOcr": "Turn on local OCR in the AI settings to read images on this device.", + "unavailableLocalOcr": "Turn on “Read scans on this device” under Settings → Modules → Labs to read images on this device.", "unavailableAction": "Open AI settings", + "unavailableLocalOcrAction": "Open lab settings", "errorGeneric": "Couldn't read the document. Try again.", "errorOcr": "Couldn't read text from this image.", "errorRateLimited": "The hourly AI reading limit is reached. Try again later.", @@ -11782,7 +11783,7 @@ "errorFileType": "This file type can't be read.", "errorNotIndexed": "Read the document first, then extract.", "errorOutputLimit": "The AI provider reached its output limit before it returned a single value. Try a model that allows longer answers.", - "errorLocalOcrDisabled": "Turn on local OCR in the AI settings first.", + "errorLocalOcrDisabled": "Turn on “Read scans on this device” under Settings → Modules → Labs first.", "errorProvider": "No AI provider is set up for this.", "errorTextImageOnly": "Local OCR reads images only, not this file.", "errorConsent": "Reading documents with AI needs your consent first.", diff --git a/messages/es.json b/messages/es.json index 55dd09b69..be842de69 100644 --- a/messages/es.json +++ b/messages/es.json @@ -11769,8 +11769,9 @@ "dismiss": "Descartar", "empty": "No se pudo leer nada de este documento.", "unavailableBody": "Configura un proveedor de IA para sugerir título, tipo y fecha.", - "unavailableLocalOcr": "Activa el OCR local en los ajustes de IA para leer imágenes en este dispositivo.", + "unavailableLocalOcr": "Activa «Leer escaneos en este dispositivo» en Ajustes → Módulos → Laboratorio para leer imágenes en este dispositivo.", "unavailableAction": "Abrir ajustes de IA", + "unavailableLocalOcrAction": "Abrir ajustes de laboratorio", "errorGeneric": "No se pudo leer el documento. Inténtalo de nuevo.", "errorOcr": "No se pudo leer texto de esta imagen.", "errorRateLimited": "Se alcanzó el límite de lecturas de IA por hora. Inténtalo más tarde.", @@ -11782,7 +11783,7 @@ "errorFileType": "Este tipo de archivo no se puede leer.", "errorNotIndexed": "Primero lee el documento y luego extrae.", "errorOutputLimit": "El proveedor de IA alcanzó su límite de salida antes de devolver un solo valor. Prueba con un modelo que permita respuestas más largas.", - "errorLocalOcrDisabled": "Activa primero el OCR local en los ajustes de IA.", + "errorLocalOcrDisabled": "Activa primero «Leer escaneos en este dispositivo» en Ajustes → Módulos → Laboratorio.", "errorProvider": "No hay ningún proveedor de IA configurado para esto.", "errorTextImageOnly": "El OCR local solo lee imágenes, no este archivo.", "errorConsent": "Leer documentos con IA necesita primero tu consentimiento.", diff --git a/messages/fr.json b/messages/fr.json index d74a6043b..e009d81f5 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -11769,8 +11769,9 @@ "dismiss": "Ignorer", "empty": "Impossible de lire ce document.", "unavailableBody": "Configurez un fournisseur d'IA pour suggérer un titre, un type et une date.", - "unavailableLocalOcr": "Activez l'OCR local dans les paramètres d'IA pour lire les images sur cet appareil.", + "unavailableLocalOcr": "Activez « Lire les scans sur cet appareil » dans Paramètres → Modules → Analyses pour lire les images sur cet appareil.", "unavailableAction": "Ouvrir les paramètres d'IA", + "unavailableLocalOcrAction": "Ouvrir les paramètres des analyses", "errorGeneric": "Impossible de lire le document. Réessayez.", "errorOcr": "Impossible de lire le texte de cette image.", "errorRateLimited": "La limite horaire de lectures IA est atteinte. Réessayez plus tard.", @@ -11782,7 +11783,7 @@ "errorFileType": "Ce type de fichier ne peut pas être lu.", "errorNotIndexed": "Lisez d’abord le document, puis extrayez.", "errorOutputLimit": "Le fournisseur d’IA a atteint sa limite de sortie avant de renvoyer la moindre valeur. Essayez un modèle qui accepte des réponses plus longues.", - "errorLocalOcrDisabled": "Activez d'abord l'OCR local dans les paramètres d'IA.", + "errorLocalOcrDisabled": "Activez d'abord « Lire les scans sur cet appareil » dans Paramètres → Modules → Analyses.", "errorProvider": "Aucun fournisseur d'IA n'est configuré pour cela.", "errorTextImageOnly": "L'OCR local ne lit que les images, pas ce fichier.", "errorConsent": "La lecture de documents par IA nécessite d'abord votre consentement.", diff --git a/messages/it.json b/messages/it.json index b1de9b105..47004b65b 100644 --- a/messages/it.json +++ b/messages/it.json @@ -11769,8 +11769,9 @@ "dismiss": "Ignora", "empty": "Impossibile leggere questo documento.", "unavailableBody": "Configura un provider IA per suggerire titolo, tipo e data.", - "unavailableLocalOcr": "Attiva l'OCR locale nelle impostazioni IA per leggere le immagini su questo dispositivo.", + "unavailableLocalOcr": "Attiva «Leggi le scansioni su questo dispositivo» in Impostazioni → Moduli → Esami per leggere le immagini su questo dispositivo.", "unavailableAction": "Apri impostazioni IA", + "unavailableLocalOcrAction": "Apri le impostazioni degli esami", "errorGeneric": "Impossibile leggere il documento. Riprova.", "errorOcr": "Impossibile leggere il testo da questa immagine.", "errorRateLimited": "Il limite orario di letture IA è stato raggiunto. Riprova più tardi.", @@ -11782,7 +11783,7 @@ "errorFileType": "Questo tipo di file non può essere letto.", "errorNotIndexed": "Leggi prima il documento, poi estrai.", "errorOutputLimit": "Il fornitore di IA ha raggiunto il suo limite di output prima di restituire un solo valore. Prova con un modello che consente risposte più lunghe.", - "errorLocalOcrDisabled": "Attiva prima l'OCR locale nelle impostazioni IA.", + "errorLocalOcrDisabled": "Attiva prima «Leggi le scansioni su questo dispositivo» in Impostazioni → Moduli → Esami.", "errorProvider": "Nessun provider IA è configurato per questo.", "errorTextImageOnly": "L'OCR locale legge solo immagini, non questo file.", "errorConsent": "La lettura dei documenti con IA richiede prima il tuo consenso.", diff --git a/messages/ko.json b/messages/ko.json index 3fbdc541b..7e743c844 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -11769,8 +11769,9 @@ "dismiss": "닫기", "empty": "이 문서에서 읽어낼 내용이 없었어요.", "unavailableBody": "AI 제공자를 설정하면 제목·종류·날짜를 추천해요.", - "unavailableLocalOcr": "AI 설정에서 로컬 OCR을 켜면 이 기기에서 이미지를 읽어요.", + "unavailableLocalOcr": "설정 → 모듈 → 검사에서 “이 기기에서 스캔 읽기”를 켜면 이 기기에서 이미지를 읽어요.", "unavailableAction": "AI 설정 열기", + "unavailableLocalOcrAction": "검사 설정 열기", "errorGeneric": "문서를 읽지 못했어요. 다시 시도해 주세요.", "errorOcr": "이 이미지에서 텍스트를 읽지 못했어요.", "errorRateLimited": "시간당 AI 읽기 한도에 도달했어요. 나중에 다시 시도해 주세요.", @@ -11782,7 +11783,7 @@ "errorFileType": "이 파일 형식은 읽을 수 없어요.", "errorNotIndexed": "문서를 먼저 읽은 뒤에 추출해 주세요.", "errorOutputLimit": "AI 제공자가 값을 하나도 돌려주기 전에 출력 한도에 도달했어요. 더 긴 답변을 허용하는 모델로 다시 시도해 주세요.", - "errorLocalOcrDisabled": "AI 설정에서 로컬 OCR을 먼저 켜 주세요.", + "errorLocalOcrDisabled": "설정 → 모듈 → 검사에서 “이 기기에서 스캔 읽기”를 먼저 켜 주세요.", "errorProvider": "이 작업에 쓸 AI 제공자가 설정돼 있지 않아요.", "errorTextImageOnly": "로컬 OCR은 이미지만 읽어요. 이 파일은 읽을 수 없어요.", "errorConsent": "AI로 문서를 읽으려면 먼저 동의가 필요합니다.", diff --git a/messages/nl.json b/messages/nl.json index af2efc0a4..6a390f9eb 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -11769,8 +11769,9 @@ "dismiss": "Negeren", "empty": "Er kon niets uit dit document worden gelezen.", "unavailableBody": "Stel een AI-provider in om een titel, soort en datum voor te stellen.", - "unavailableLocalOcr": "Zet lokale OCR aan in de AI-instellingen om afbeeldingen op dit apparaat te lezen.", + "unavailableLocalOcr": "Zet “Scans op dit apparaat lezen” aan onder Instellingen → Modules → Labwaarden om afbeeldingen op dit apparaat te lezen.", "unavailableAction": "AI-instellingen openen", + "unavailableLocalOcrAction": "Instellingen voor labwaarden openen", "errorGeneric": "Het document kon niet worden gelezen. Probeer het opnieuw.", "errorOcr": "Er kon geen tekst uit deze afbeelding worden gelezen.", "errorRateLimited": "De limiet voor lezen met AI per uur is bereikt. Probeer het later opnieuw.", @@ -11782,7 +11783,7 @@ "errorFileType": "Dit bestandstype kan niet worden gelezen.", "errorNotIndexed": "Laat het document eerst lezen en extraheer daarna.", "errorOutputLimit": "De AI-provider bereikte zijn uitvoerlimiet voordat hij één waarde teruggaf. Probeer een model dat langere antwoorden toestaat.", - "errorLocalOcrDisabled": "Zet eerst lokale OCR aan in de AI-instellingen.", + "errorLocalOcrDisabled": "Zet eerst “Scans op dit apparaat lezen” aan onder Instellingen → Modules → Labwaarden.", "errorProvider": "Hiervoor is geen AI-provider ingesteld.", "errorTextImageOnly": "Lokale OCR leest alleen afbeeldingen, niet dit bestand.", "errorConsent": "Voor het lezen van documenten met AI is eerst je toestemming nodig.", diff --git a/messages/pl.json b/messages/pl.json index b0dec38cc..0d86a2800 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -11769,8 +11769,9 @@ "dismiss": "Odrzuć", "empty": "Nie udało się nic odczytać z tego dokumentu.", "unavailableBody": "Skonfiguruj dostawcę AI, aby zaproponować tytuł, typ i datę.", - "unavailableLocalOcr": "Włącz lokalne OCR w ustawieniach AI, aby odczytywać obrazy na tym urządzeniu.", + "unavailableLocalOcr": "Włącz „Odczytuj skany na tym urządzeniu” w Ustawienia → Moduły → Badania, aby odczytywać obrazy na tym urządzeniu.", "unavailableAction": "Otwórz ustawienia AI", + "unavailableLocalOcrAction": "Otwórz ustawienia badań", "errorGeneric": "Nie udało się odczytać dokumentu. Spróbuj ponownie.", "errorOcr": "Nie udało się odczytać tekstu z tego obrazu.", "errorRateLimited": "Godzinowy limit odczytów AI został osiągnięty. Spróbuj ponownie później.", @@ -11782,7 +11783,7 @@ "errorFileType": "Tego typu pliku nie można odczytać.", "errorNotIndexed": "Najpierw odczytaj dokument, potem wyodrębnij.", "errorOutputLimit": "Dostawca AI osiągnął limit odpowiedzi, zanim zwrócił choć jedną wartość. Spróbuj modelu, który pozwala na dłuższe odpowiedzi.", - "errorLocalOcrDisabled": "Najpierw włącz lokalne OCR w ustawieniach AI.", + "errorLocalOcrDisabled": "Najpierw włącz „Odczytuj skany na tym urządzeniu” w Ustawienia → Moduły → Badania.", "errorProvider": "Nie skonfigurowano dostawcy AI do tego celu.", "errorTextImageOnly": "Lokalne OCR odczytuje tylko obrazy, nie ten plik.", "errorConsent": "Odczyt dokumentów przez AI wymaga najpierw Twojej zgody.", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index 3137e637c..33a57ff8b 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -11769,8 +11769,9 @@ "dismiss": "Dispensar", "empty": "Nada pôde ser lido deste documento.", "unavailableBody": "Configure um provedor de IA para sugerir um título, tipo e data.", - "unavailableLocalOcr": "Ative o OCR local nas configurações de IA para ler imagens neste dispositivo.", + "unavailableLocalOcr": "Ative “Ler digitalizações neste dispositivo” em Configurações → Módulos → Exames para ler imagens neste dispositivo.", "unavailableAction": "Abrir configurações de IA", + "unavailableLocalOcrAction": "Abrir configurações de exames", "errorGeneric": "Não foi possível ler o documento. Tente novamente.", "errorOcr": "Não foi possível ler texto desta imagem.", "errorRateLimited": "O limite horário de leitura por IA foi atingido. Tente novamente mais tarde.", @@ -11782,7 +11783,7 @@ "errorFileType": "Este tipo de arquivo não pode ser lido.", "errorNotIndexed": "Leia o documento primeiro, depois extraia.", "errorOutputLimit": "O provedor de IA atingiu o limite de saída antes de devolver um único valor. Tente um modelo que permita respostas mais longas.", - "errorLocalOcrDisabled": "Ative o OCR local nas configurações de IA primeiro.", + "errorLocalOcrDisabled": "Ative primeiro “Ler digitalizações neste dispositivo” em Configurações → Módulos → Exames.", "errorProvider": "Nenhum provedor de IA está configurado para isso.", "errorTextImageOnly": "O OCR local lê apenas imagens, não este arquivo.", "errorConsent": "Ler documentos com IA precisa do seu consentimento primeiro.", diff --git a/src/components/documents/__tests__/document-ai-panels.test.tsx b/src/components/documents/__tests__/document-ai-panels.test.tsx index 53748f937..d6e2c20dc 100644 --- a/src/components/documents/__tests__/document-ai-panels.test.tsx +++ b/src/components/documents/__tests__/document-ai-panels.test.tsx @@ -1,7 +1,12 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { renderToStaticMarkup } from "react-dom/server"; import { I18nProvider } from "@/lib/i18n/context"; + +const record = vi.hoisted(() => ({ inSharedRecord: false })); +vi.mock("@/hooks/use-record-capabilities", () => ({ + useRecordCapabilities: () => ({ inSharedRecord: record.inSharedRecord }), +})); import type { DocumentSuggestionDto } from "@/lib/validations/inbound-documents"; import { AiUnavailableHint, @@ -57,10 +62,26 @@ describe("", () => { expect(html).not.toContain('role="alert"'); }); - it("nudges toward local OCR when a text-only provider needs it", () => { + it("points at the on-device scan switch when a text-only provider needs it", () => { const html = render(); - expect(html).toContain("local OCR"); - expect(html).toContain('href="/settings/ai"'); + // The switch lives on the Labs module page, under its own label. + expect(html).toContain("Read scans on this device"); + expect(html).toContain("Settings → Modules → Labs"); + expect(html).toContain('href="/settings/modules/labs#labs-local-ocr"'); + expect(html).toContain("Open lab settings"); + expect(html).not.toContain('href="/settings/ai"'); + }); + + it("keeps the sentence but withholds the Settings link inside somebody else's record", () => { + record.inSharedRecord = true; + try { + const html = render(); + expect(html).toContain("Read scans on this device"); + expect(html).not.toContain('data-slot="assist-settings-link"'); + expect(html).not.toContain('href="/settings'); + } finally { + record.inSharedRecord = false; + } }); }); diff --git a/src/components/documents/__tests__/document-ai-section.test.tsx b/src/components/documents/__tests__/document-ai-section.test.tsx index 742859335..c6b85142d 100644 --- a/src/components/documents/__tests__/document-ai-section.test.tsx +++ b/src/components/documents/__tests__/document-ai-section.test.tsx @@ -1,7 +1,12 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { renderToStaticMarkup } from "react-dom/server"; import { I18nProvider } from "@/lib/i18n/context"; + +const record = vi.hoisted(() => ({ inSharedRecord: false })); +vi.mock("@/hooks/use-record-capabilities", () => ({ + useRecordCapabilities: () => ({ inSharedRecord: record.inSharedRecord }), +})); import type { DocumentSuggestionDto } from "@/lib/validations/inbound-documents"; import { DocumentAiSection } from "../document-ai-section"; diff --git a/src/components/documents/document-ai-panels.tsx b/src/components/documents/document-ai-panels.tsx index b2c5e8d66..114f3d987 100644 --- a/src/components/documents/document-ai-panels.tsx +++ b/src/components/documents/document-ai-panels.tsx @@ -22,6 +22,7 @@ import Link from "next/link"; import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; import type { AiRunPhase } from "@/hooks/use-ai-run"; +import { useRecordCapabilities } from "@/hooks/use-record-capabilities"; import { useTranslations } from "@/lib/i18n/context"; import { cn } from "@/lib/utils"; import { @@ -64,9 +65,13 @@ export function AiRunPhaseNote({ } /** - * Calm pointer to the AI settings, shown when assist is unavailable. When the - * document is already searchable (auto-indexed locally) the copy stays honest — - * it is searchable, an AI provider only adds a richer read. + * Calm pointer to the setting that would make assist available: the AI + * settings when no provider is configured, the on-device scan switch on the + * Labs module page when the provider cannot read images. Both live under + * `/settings`, which is the caller's own and closed inside somebody else's + * record, so the link is withheld there and the sentence stands alone. When + * the document is already searchable (auto-indexed locally) the copy stays + * honest — it is searchable, an AI provider only adds a richer read. */ export function AiUnavailableHint({ reason, @@ -74,10 +79,11 @@ export function AiUnavailableHint({ reason: "no-provider" | "enable-local-ocr" | null; }) { const { t } = useTranslations(); - const body = - reason === "enable-local-ocr" - ? t("documents.assist.unavailableLocalOcr") - : t("documents.assist.unavailableBody"); + const { inSharedRecord } = useRecordCapabilities(); + const localOcr = reason === "enable-local-ocr"; + const body = localOcr + ? t("documents.assist.unavailableLocalOcr") + : t("documents.assist.unavailableBody"); return (

- {body}{" "} - - {t("documents.assist.unavailableAction")} - + {body} + {!inSharedRecord && ( + <> + {" "} + + {localOcr + ? t("documents.assist.unavailableLocalOcrAction") + : t("documents.assist.unavailableAction")} + + + )}

); From d6bb7bc905946f8e9ae7ed82d694bc35f2136059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:46:11 +0100 Subject: [PATCH 03/19] docs(api): describe the legacy integration-status fields as still served The published contract said the nine health-shaped fields that predate `health` were removed in v1.44, but every status read still sends them: older native builds decode them, the current one falls back to them against a server without `health`, and the web card's copied diagnostics read the failure streak. They stay deprecated, without a removal release. --- docs/api/openapi.yaml | 69 +++++++++---------- .../baselines/version-stamp-comments.json | 4 +- src/app/api/integrations/status/route.ts | 9 +-- src/lib/openapi/routes/integrations.ts | 12 ++-- 4 files changed, 48 insertions(+), 46 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 9255e81e3..aa48c04d7 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -82136,7 +82136,7 @@ components: - type: "null" syncHealth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Present on the GET read; omitted from the PATCH echo." + description: "Deprecated, still served: read `health`. Present on the GET read; omitted from the PATCH echo." $ref: "#/components/schemas/SyncHealth" metricFreshness: description: Per-metric-type freshness for the `APPLE_HEALTH` source, with the per-type arrival facts. Present on the @@ -82814,7 +82814,7 @@ components: type: boolean state: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." type: string enum: - connected @@ -82837,14 +82837,14 @@ components: - type: "null" lastError: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Last recorded failure message, decrypted from the ledger. - Null when the last attempt succeeded or none has run." + description: "Deprecated, still served: read `health`. Last recorded failure message, decrypted from the ledger. Null + when the last attempt succeeded or none has run." type: - string - "null" syncHealth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." $ref: "#/components/schemas/SyncHealth" health: description: The unified connection state. Present whenever the connection exists; null in the rare case it resolved to @@ -83033,9 +83033,9 @@ components: pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. The stored access token's expiry is in the past. Read - straight off the row — unlike the Withings status read, this endpoint does not refresh; the sync path - refreshes lazily instead." + description: "Deprecated, still served: read `health`. The stored access token's expiry is in the past. Read straight + off the row — unlike the Withings status read, this endpoint does not refresh; the sync path refreshes + lazily instead." type: boolean tokenExpiresAt: type: string @@ -83124,8 +83124,7 @@ components: pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. True after a refresh attempt that did not produce a live - token." + description: "Deprecated, still served: read `health`. True after a refresh attempt that did not produce a live token." type: boolean tokenRefreshFailed: description: The read tried to refresh an expired token and could not — either the per-user credentials are gone or @@ -83145,8 +83144,8 @@ components: - "null" hasActivityScope: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Pre-computed convenience flag the reconnect banner reads. A - null `scope` counts as missing activity." + description: "Deprecated, still served: read `health`. Pre-computed convenience flag the reconnect banner reads. A null + `scope` counts as missing activity." type: boolean health: description: The unified connection state. Present whenever the connection exists; null in the rare case it resolved to @@ -83455,8 +83454,8 @@ components: pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Read straight off the row — this endpoint does not refresh; - the sync path refreshes lazily." + description: "Deprecated, still served: read `health`. Read straight off the row — this endpoint does not refresh; the + sync path refreshes lazily." type: boolean tokenExpiresAt: type: string @@ -83626,8 +83625,8 @@ components: pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Read straight off the row — this endpoint does not refresh; - the sync path refreshes lazily." + description: "Deprecated, still served: read `health`. Read straight off the row — this endpoint does not refresh; the + sync path refreshes lazily." type: boolean tokenExpiresAt: type: string @@ -83677,7 +83676,7 @@ components: when every type was read to its start. Kept after the backfill completes; cleared on reconnect. needsReauth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. The stored refresh token was rejected with `invalid_grant` — + description: "Deprecated, still served: read `health`. The stored refresh token was rejected with `invalid_grant` — Google expires it after 7 days while the OAuth client is in Testing publishing mode, and a user-revoked grant lands here too. Distinct from `parked` and from `disconnected`: the connection still exists, it just cannot mint a token until the user reconnects. Only Google Health has this field." @@ -83881,7 +83880,7 @@ components: description: The user has stored their own BYO client id and secret. state: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." type: string enum: - connected @@ -83904,13 +83903,13 @@ components: - type: "null" lastError: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." type: - string - "null" syncHealth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." $ref: "#/components/schemas/SyncHealth" health: description: The unified connection state. Present whenever the connection exists; null in the rare case it resolved to @@ -84305,7 +84304,7 @@ components: - disconnected - parked - unknown - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." deprecated: true lastSuccessAt: anyOf: @@ -84321,7 +84320,7 @@ components: - type: "null" lastError: deprecated: true - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." type: - string - "null" @@ -84348,11 +84347,11 @@ components: additionalProperties: false - type: "null" deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Per-kind consecutive-failure counters, or null when the row - has never written a bucket payload. Compare against the top-level `threshold` to know how close the provider - is to the operator alert." + description: "Deprecated, still served: read `health`. Per-kind consecutive-failure counters, or null when the row has + never written a bucket payload. Compare against the top-level `threshold` to know how close the provider is + to the operator alert." syncHealth: - description: "Deprecated: removed in v1.44, read `health`." + description: "Deprecated, still served: read `health`." deprecated: true $ref: "#/components/schemas/SyncHealth" metricFreshness: @@ -84389,7 +84388,7 @@ components: - type: "null" tokenExpired: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Null — not false — when there is no connection to judge." + description: "Deprecated, still served: read `health`. Null — not false — when there is no connection to judge." type: - boolean - "null" @@ -84432,14 +84431,14 @@ components: - "null" hasActivityScope: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Withings only." + description: "Deprecated, still served: read `health`. Withings only." type: boolean planUnavailable: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Withings only. The readings this connection's Withings API - plan withholds (status 277 since the 2026-10-12 plan split), deduplicated and in display order. Empty when - the plan refuses nothing. The sync skips these and keeps importing everything else; the list refreshes when - the weekly re-probe asks again." + description: "Deprecated, still served: read `health`. Withings only. The readings this connection's Withings API plan + withholds (status 277 since the 2026-10-12 plan split), deduplicated and in display order. Empty when the + plan refuses nothing. The sync skips these and keeps importing everything else; the list refreshes when the + weekly re-probe asks again." type: array items: type: string @@ -84468,7 +84467,7 @@ components: - weight needsReauth: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Google Health only." + description: "Deprecated, still served: read `health`. Google Health only." type: boolean available: description: Oura / Polar / Strava only. @@ -84478,7 +84477,7 @@ components: type: boolean scopeReconnect: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Oura only. Collections the stored grant does not cover: Oura + description: "Deprecated, still served: read `health`. Oura only. Collections the stored grant does not cover: Oura refused them for a missing scope (named in its 401, or inferred when sibling collections succeeded with the same token), or the scopes Oura reported at connect leave theirs out. Not a failure and not counted toward the reconnect threshold; one reconnect asks for the missing scopes. Empty when the grant covers everything @@ -84534,7 +84533,7 @@ components: description: The unified connection state, null when the provider is not connected (and for Fitbit once `retirement.ended`). It supersedes `state`, `lastError`, `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, `hasActivityScope`, `planUnavailable` and `scopeReconnect` on this entry; - those nine are deprecated and removed in v1.44. + those nine are deprecated and still served for clients that predate `health`. required: - integration - state diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 5a3506c2e..95b97061a 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2897, - "total": 9247, + "total": 9246, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -360,7 +360,7 @@ "app/api/integrations/healthkit/__tests__/route.test.ts": 4, "app/api/integrations/healthkit/route.ts": 3, "app/api/integrations/status/__tests__/route.test.ts": 2, - "app/api/integrations/status/route.ts": 7, + "app/api/integrations/status/route.ts": 6, "app/api/integrations/withings/resume/__tests__/route.test.ts": 1, "app/api/integrations/withings/resume/route.ts": 1, "app/api/internal/deploy-webhook/__tests__/route.test.ts": 2, diff --git a/src/app/api/integrations/status/route.ts b/src/app/api/integrations/status/route.ts index 417037307..56539ee6e 100644 --- a/src/app/api/integrations/status/route.ts +++ b/src/app/api/integrations/status/route.ts @@ -502,8 +502,8 @@ export const GET = apiHandler(async () => { } satisfies IntegrationViewModel & NightscoutExtras, { ...publish(dawarichStatus), - // v1.43 — Dawarich, read by the card through `health`. The ledger core - // rides along like on every entry, and goes with the rest in v1.44. + // Dawarich, read by the card through `health`. The ledger core rides + // along like on every entry. connected: dawarichConfigured, configured: dawarichConfigured, health: health["dawarich"], @@ -524,8 +524,9 @@ interface IntegrationViewModel { * health-shaped field on this entry (`state`, `lastError`, * `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, * `hasActivityScope`, `planUnavailable`, `scopeReconnect`) is folded into - * it and goes once the settings cards render `health`; no other client - * reads this envelope. + * it. Those stay on the wire, deprecated in the published document; the + * cards render `health`, and only their copied diagnostics still read + * `consecutiveFailuresByKind`. No native client reads this envelope. */ health: ConnectionHealth | null; state: string; diff --git a/src/lib/openapi/routes/integrations.ts b/src/lib/openapi/routes/integrations.ts index c2acc5677..a50b645d7 100644 --- a/src/lib/openapi/routes/integrations.ts +++ b/src/lib/openapi/routes/integrations.ts @@ -75,10 +75,12 @@ const healthKitEntry = z "One resolved HealthKit metric mapping (defaults merged with the user's stored overrides).", }); -// The health-shaped fields that predate `health` (v1.42.4). They stay on the -// wire through v1.43 so App 1.2.0 keeps decoding, are marked deprecated in the -// document, and go in v1.44. -const LEGACY_HEALTH_FIELD = "Deprecated: removed in v1.44, read `health`."; +// The health-shaped fields that predate `health`. They are marked deprecated +// in the document and stay on the wire: native builds from before `health` +// still decode them, the current one falls back to them only against a server +// that sends no `health`, and the web card's copied diagnostics read the +// failure streak. No removal release is set. +const LEGACY_HEALTH_FIELD = "Deprecated, still served: read `health`."; function legacy(text?: string): { deprecated: true; description: string } { return { @@ -1184,7 +1186,7 @@ const integrationEnvelopeEntry = z health: connectionHealth .nullable() .describe( - "The unified connection state, null when the provider is not connected (and for Fitbit once `retirement.ended`). It supersedes `state`, `lastError`, `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, `hasActivityScope`, `planUnavailable` and `scopeReconnect` on this entry; those nine are deprecated and removed in v1.44.", + "The unified connection state, null when the provider is not connected (and for Fitbit once `retirement.ended`). It supersedes `state`, `lastError`, `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, `hasActivityScope`, `planUnavailable` and `scopeReconnect` on this entry; those nine are deprecated and still served for clients that predate `health`.", ), }) .meta({ From a57d2ffe83c992f9e9825df418a7ec612f8ebfae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:46:51 +0100 Subject: [PATCH 04/19] test(fhir): pin the search parameters the CapabilityStatement declares The statement already declares `date` and `code` on Observation beside paging on every search; the route test now asserts that list per resource, so a parameter the search honours cannot drop out of it unnoticed. --- tests/integration/routes/fhir.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/routes/fhir.test.ts b/tests/integration/routes/fhir.test.ts index a094784b8..8ff64fff2 100644 --- a/tests/integration/routes/fhir.test.ts +++ b/tests/integration/routes/fhir.test.ts @@ -313,6 +313,9 @@ describe("GET /api/fhir/metadata", () => { z.looseObject({ type: z.string(), interaction: z.array(z.object({ code: z.string() })), + searchParam: z.array( + z.object({ name: z.string(), type: z.string() }), + ), operation: z.array(z.object({ name: z.string() })).optional(), }), ), @@ -330,6 +333,21 @@ describe("GET /api/fhir/metadata", () => { expect(patient).toHaveLength(1); expect(patient[0].operation?.map((o) => o.name)).toEqual(["everything"]); + // Paging on every search; the Observation search alone honours the + // time-series range and the code filter, and says so. + for (const resource of resources) { + expect(resource.searchParam, resource.type).toEqual([ + { name: "_count", type: "number" }, + { name: "_offset", type: "number" }, + ...(resource.type === "Observation" + ? [ + { name: "date", type: "date" }, + { name: "code", type: "token" }, + ] + : []), + ]); + } + const root = path.join(process.cwd(), "src/app/api/fhir"); for (const resource of resources) { for (const { code } of resource.interaction) { From 72d90282d5cfc4efb824745765942e376b40ca50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:48:40 +0100 Subject: [PATCH 05/19] fix(settings): open the docs pages themselves, not their old addresses The integration runbooks, the Garmin note, the guardian note on a managed record's connections, the backups admin link and the README's self-hosting link all pointed at retired docs paths that only resolve through redirects (or not at all, on a host without them). They now go to the final guide pages, and the settings tests assert the new addresses. --- README.md | 2 +- .../baselines/version-stamp-comments.json | 4 ++-- src/components/admin/backups-section.tsx | 2 +- .../__tests__/integrations-section.test.tsx | 7 ++++++- .../managed-integration-status.test.tsx | 2 +- .../fitbit-retirement-notice.test.tsx | 2 +- .../settings/integrations/garmin-info-note.tsx | 4 ++-- .../settings/integrations/setup-guide-link.tsx | 18 ++++++++++-------- .../settings/managed-integration-status.tsx | 2 +- .../__tests__/setup-docs-parity.test.ts | 8 ++++---- 10 files changed, 29 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 854eb8cc3..65c2c13f3 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Verify the running version: curl -s http://localhost:3000/api/version # returns version + buildSha + builtAt ``` -`SESSION_COOKIE_SECURE=false` is the one setting plain-HTTP self-hosts must set: without it the session cookie carries the `Secure` flag, the browser drops it over HTTP, and login silently fails. Leave it unset (or `true`) only when a TLS reverse proxy serves HTTPS — and then set `NEXT_PUBLIC_APP_URL` and `APP_URL` to the public URL first (see the [self-hosting guide](https://docs.healthlog.dev/self-hosting/)). +`SESSION_COOKIE_SECURE=false` is the one setting plain-HTTP self-hosts must set: without it the session cookie carries the `Secure` flag, the browser drops it over HTTP, and login silently fails. Leave it unset (or `true`) only when a TLS reverse proxy serves HTTPS — and then set `NEXT_PUBLIC_APP_URL` and `APP_URL` to the public URL first (see the [self-hosting guide](https://docs.healthlog.dev/getting-started/self-host-in-10-minutes/)). Persistent state lives in exactly one named volume (`postgres-data`, the Postgres data); the app container is stateless, so backing up the database backs up everything. An install from v1.45 or earlier moves its Postgres 16 database into it on the first start ([details](docs/ops/postgres-major-upgrade.md)). diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 95b97061a..6902bda8e 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2897, - "total": 9246, + "total": 9243, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -1381,7 +1381,7 @@ "components/settings/integrations/connections-panel.tsx": 5, "components/settings/integrations/garmin-info-note.tsx": 1, "components/settings/integrations/notification-channels-panel.tsx": 1, - "components/settings/integrations/setup-guide-link.tsx": 5, + "components/settings/integrations/setup-guide-link.tsx": 2, "components/settings/integrations/shared.tsx": 3, "components/settings/labs-section.tsx": 1, "components/settings/layout-groups.ts": 2, diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index d8dcb62c2..0eb220c00 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -98,7 +98,7 @@ export function BackupsSection() { {/* External docs link — `noopener noreferrer` because this leaves the admin shell. */} { for (const provider of PROVIDERS) { expect(html).toContain(`data-testid="${provider}-setup-guide"`); expect(html).toContain( - `href="https://docs.healthlog.dev/integrations/${provider}"`, + `href="https://docs.healthlog.dev/guides/connect-${provider}/"`, ); } + // Garmin has no connector; its note goes straight to the section of the + // Google Health guide that explains the route, not through a redirect. + expect(html).toContain( + 'href="https://docs.healthlog.dev/guides/connect-google-health/#garmin-owners"', + ); }); }); diff --git a/src/components/settings/__tests__/managed-integration-status.test.tsx b/src/components/settings/__tests__/managed-integration-status.test.tsx index d7126b5dd..f805c891a 100644 --- a/src/components/settings/__tests__/managed-integration-status.test.tsx +++ b/src/components/settings/__tests__/managed-integration-status.test.tsx @@ -139,7 +139,7 @@ describe("", () => { "Services you connect here send their data to Managed record's record, not to yours.", ); expect(html).toContain( - 'href="https://docs.healthlog.dev/self-hosting/account-sharing#records-with-no-login"', + 'href="https://docs.healthlog.dev/guides/managed-profiles/#connected-services"', ); }); diff --git a/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx b/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx index 21fd77d23..753be99d2 100644 --- a/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx +++ b/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx @@ -31,7 +31,7 @@ describe("", () => { expect(html).toContain("Google Health reads the same Fitbit and Pixel"); expect(html).toContain("Everything already imported stays."); expect(html).toContain('href="#google-health"'); - expect(html).toContain("/google-health"); + expect(html).toContain("/guides/connect-google-health/"); }); it("says the connection has stopped once the day has come", () => { diff --git a/src/components/settings/integrations/garmin-info-note.tsx b/src/components/settings/integrations/garmin-info-note.tsx index 4d2986acc..10c3fd645 100644 --- a/src/components/settings/integrations/garmin-info-note.tsx +++ b/src/components/settings/integrations/garmin-info-note.tsx @@ -11,7 +11,7 @@ import { ExternalLink, Watch } from "lucide-react"; import { SettingsCard } from "@/components/settings/settings-card"; import { SettingsCardHeader } from "@/components/settings/_card-header"; -import { INTEGRATION_DOCS_BASE } from "@/components/settings/integrations/setup-guide-link"; +import { GARMIN_DOCS_HREF } from "@/components/settings/integrations/setup-guide-link"; import { useTranslations } from "@/lib/i18n/context"; export function GarminInfoNote() { @@ -25,7 +25,7 @@ export function GarminInfoNote() {

{t("settings.garminInfo.body")}{" "} /`, e.g. `https://docs.healthlog.dev/guides/connect-whoop/`. + * These are the pages themselves: the older `/integrations/` paths + * only resolve through redirects. Kept as a single constant so the host never + * drifts across cards. */ -export const INTEGRATION_DOCS_BASE = "https://docs.healthlog.dev/integrations"; +export const DOCS_GUIDES_BASE = "https://docs.healthlog.dev/guides"; + +/** Garmin has no connector; the Google Health guide explains the route. */ +export const GARMIN_DOCS_HREF = `${DOCS_GUIDES_BASE}/connect-google-health/#garmin-owners`; export type IntegrationDocsProvider = | "whoop" @@ -30,15 +35,12 @@ export type IntegrationDocsProvider = | "polar" | "oura" | "nightscout" - // v1.27.0 — Google Health runbook (docs.healthlog.dev/integrations/google-health). | "google-health" - // v1.28.x — Strava runbook (docs.healthlog.dev/integrations/strava). | "strava" - // v1.43 — Dawarich runbook (docs.healthlog.dev/integrations/dawarich). | "dawarich"; export function integrationDocsHref(provider: IntegrationDocsProvider): string { - return `${INTEGRATION_DOCS_BASE}/${provider}`; + return `${DOCS_GUIDES_BASE}/connect-${provider}/`; } /** diff --git a/src/components/settings/managed-integration-status.tsx b/src/components/settings/managed-integration-status.tsx index 6f84aa551..1498add4a 100644 --- a/src/components/settings/managed-integration-status.tsx +++ b/src/components/settings/managed-integration-status.tsx @@ -54,7 +54,7 @@ const INTEGRATION_DISPLAY_NAMES = { /** Where the guardian boundary is explained, connected services included. */ const SHARING_DOCS_HREF = - "https://docs.healthlog.dev/self-hosting/account-sharing#records-with-no-login"; + "https://docs.healthlog.dev/guides/managed-profiles/#connected-services"; function iconFor(key: ManagedIntegrationKey) { if (key === "nightscout") return Droplet; diff --git a/src/lib/google-health/__tests__/setup-docs-parity.test.ts b/src/lib/google-health/__tests__/setup-docs-parity.test.ts index 9e8a3a094..653b72cfe 100644 --- a/src/lib/google-health/__tests__/setup-docs-parity.test.ts +++ b/src/lib/google-health/__tests__/setup-docs-parity.test.ts @@ -2,8 +2,8 @@ * Doc ↔ code parity guard for the Google Health operator runbook. * * The setup-guide link on the Settings card sends a mid-setup user to - * `docs.healthlog.dev/integrations/google-health`, rendered from - * `docs/integrations/google-health.md`. If the client ever requests a scope the + * `docs.healthlog.dev/guides/connect-google-health/`; the operator runbook in + * this repository is `docs/integrations/google-health.md`. If the client ever requests a scope the * runbook doesn't list, or the walkthrough regresses back to a wrong provider * (the file was once a Fitbit-Web-API walkthrough), a self-hoster follows a * broken recipe and the OAuth consent screen rejects the connection. @@ -20,7 +20,7 @@ import { describe, expect, it } from "vitest"; import { GOOGLE_HEALTH_CORE_SCOPES } from "@/lib/google-health/oauth"; import { - INTEGRATION_DOCS_BASE, + DOCS_GUIDES_BASE, integrationDocsHref, } from "@/components/settings/integrations/setup-guide-link"; @@ -70,7 +70,7 @@ describe("google-health setup runbook parity", () => { // The literal argument also compile-checks that "google-health" is a member // of the IntegrationDocsProvider union. expect(integrationDocsHref("google-health")).toBe( - `${INTEGRATION_DOCS_BASE}/google-health`, + `${DOCS_GUIDES_BASE}/connect-google-health/`, ); }); }); From f85035a8ecd82ab9ec0d2b75bc2307c31ec2a106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:49:46 +0100 Subject: [PATCH 06/19] docs(coach): say what the labs block reads since it covers the whole record The labs block reads the whole record, or up to the lookback limit, not just recent readings, and a named read also comes back empty when no marker matches. The header of the labs snapshot and the note at its call site still described the earlier twelve-month window. --- src/lib/ai/coach/labs-snapshot.ts | 5 +++-- src/lib/ai/coach/snapshot.ts | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/ai/coach/labs-snapshot.ts b/src/lib/ai/coach/labs-snapshot.ts index bb030f516..5d606dd35 100644 --- a/src/lib/ai/coach/labs-snapshot.ts +++ b/src/lib/ai/coach/labs-snapshot.ts @@ -48,8 +48,9 @@ * `userId` is narrowed from the authenticated session by the caller and feeds * the Prisma `where` field-by-field; it is never an input. Unlike the illness * block, Labs is intentionally NOT module-gated (mirrors `/api/labs` — the data - * is always owner-scoped and safe to read), so the block is `null` only when the - * account has no recent readings. + * is always owner-scoped and safe to read). The block is `null` only when no + * reading falls inside the lookback limit (the whole record when there is + * none), or, for a named read, when no marker matches the analyte. */ import { sameLabUnit } from "@/lib/labs/unit-normalise"; import { diff --git a/src/lib/ai/coach/snapshot.ts b/src/lib/ai/coach/snapshot.ts index b572218a7..7e884a756 100644 --- a/src/lib/ai/coach/snapshot.ts +++ b/src/lib/ai/coach/snapshot.ts @@ -668,8 +668,8 @@ async function buildCoachSnapshotImpl( // scope-gated metric: attached without a `registerBlock` so the budget // degrader never sheds it, and Labs is intentionally not module-gated (the // helper reads owner-scoped rows directly, mirroring `/api/labs`). The block - // carries the most-recent resolved reading per biomarker (last 12 months, - // capped) so the Coach can answer "what was my LDL" without re-deriving. + // carries the latest resolved reading per biomarker (whole record or the + // lookback limit, capped) so the Coach can answer "what was my LDL". const labsBlockPromise = buildLabsSnapshotBlock(userId, now, reach); // v1.38 — doctor-visit context. Like illness/labs it is attempted always (a From 12f8265fdba7c275a07b434e5b9486f9b952ce97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:53:19 +0100 Subject: [PATCH 07/19] docs: correct claims and anchors the code no longer backs Fix two dead anchors (the MCP trust model, the README quick start) and the statements that no longer match the app: a guardian can connect a managed profile's services, the environment module is on by default and reaches the air-quality host too, its readable location columns were dropped in 0409, settings sections carry their current names, the Withings card lives on /settings/integrations, the TrueNAS start period matches the image, the deploy check names HEALTHLOG_PROD_URL, and the docs index describes what self-hosting/ and integrations/ hold. --- README.md | 2 +- docs/README.md | 20 +++++++++------ docs/integrations/apple-health.md | 2 +- docs/integrations/dawarich.md | 2 +- docs/integrations/fitbit.md | 2 +- docs/integrations/google-health.md | 2 +- docs/integrations/strava.md | 2 +- docs/integrations/withings.md | 8 +++--- docs/ops/deploy.md | 7 ++++-- docs/ops/measurement-maintenance.md | 5 ++-- docs/self-hosting/account-sharing.md | 14 +++++++---- docs/self-hosting/environment.md | 9 ++++--- docs/self-hosting/getting-started.md | 14 +++++------ docs/self-hosting/mcp.md | 8 +++--- docs/self-hosting/reproductive-health-data.md | 2 +- docs/self-hosting/truenas.md | 25 ++++++++++--------- docs/self-hosting/unraid.md | 4 +-- docs/ui-guidelines.md | 2 +- 18 files changed, 73 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 65c2c13f3..edf748bbb 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Try the [live demo](https://demo.healthlog.dev) to see a working install — sig - **Clinical signals, opt-in and additive.** Validated self-checks for mental wellbeing and sleep (WHO-5, PHQ-9, GAD-7, Sleep Condition Indicator — official questionnaire wording, calm crisis-resource handling), grip strength, a 0–10 pain score, waist and body measurements, and a longevity lab panel — each added through one signal registry, so a new signal wires into charts, correlations, the coach, the FHIR map, and the API at once. - **Records, structured.** Allergies and family history as structured records that flow into the doctor report (reactions and notes are encrypted, the substance and condition names are not), plus lab values you key in or scan from a printed report — OCR extracts the stated values, units, and dates for you to review and confirm before anything is saved. HealthLog transcribes what a report says; it never interprets or diagnoses from it. A vaccination record keeps a lifetime immunization log, doses grouped under the disease they protect against, and prints in the doctor report. - **A document vault for the paper trail.** An opt-in module stores every letter, report, and scan encrypted at rest — PDFs and photos preview inline, Office and text files stay download-only, and nothing user-uploaded can ever execute in the app. One searchable timeline with type, condition, and year filters, drag-and-drop and paste-to-upload, bulk filing, undo-able delete with a 30-day grace, and per-file/per-account limits the operator tunes. -- **Context that explains the numbers.** An opt-in environmental module folds weather, daylight, and temperature in as influence signals (winter darkness against mood, heat against blood pressure) using a home location you set, so a pattern has somewhere to point. +- **Context that explains the numbers.** An environmental module folds weather, daylight, temperature, air quality, and pollen in as influence signals (winter darkness against mood, heat against blood pressure) using a home location you set, so a pattern has somewhere to point. It is on by default, and nothing leaves the host until someone sets that location. - **Your devices, one server.** Withings, WHOOP, Oura and Polar sync over OAuth2, Google Health connects Fitbit and Pixel Watch through your own Google Cloud client, Nightscout streams CGM glucose, an Apple Health `export.zip` folds your full history in, and the native iOS app streams HealthKit live. A per-metric source priority decides which reading is canonical when wearables overlap. - **Medication tracking that tells the truth.** Flexible schedules (weekly injections, weekday-only, intervals, PRN, cyclic), a configurable intake window per dose, and a traceable dose history where every slot reads taken, late, skipped, or missed — the same ledger the compliance rate is computed from, so the percentage can never disagree with the timeline. - **AI insights you own.** Daily briefing, health scores, correlations, and a conversational coach grounded in your own measurements and interpreted against cited guideline reference bands — where a value sits and what that means in plain words, never a diagnosis. The coach proposes concrete if-then plans you accept or decline, and suppresses proactive check-ins when there's nothing genuine to say. Bring your own OpenAI or Anthropic key, or point at a local endpoint (Ollama, LM Studio, vLLM) so AI requests stay on your network. diff --git a/docs/README.md b/docs/README.md index 7b99ebc09..225e5a1f8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,9 +17,13 @@ notes, and machine-readable specs that ship alongside the source: encryption-key rotation, password reset and account recovery, env checks, TLS pinning, data repair. These are what an operator reaches for during an incident. -- [`self-hosting/`](./self-hosting/) — horizontal-scaling notes - (`HEALTHLOG_PROCESS_TYPE=web|worker|all`) and the deploy-pipeline - recipe. Routine install steps stay on `docs.healthlog.dev`. Includes +- [`self-hosting/`](./self-hosting/) — operator notes next to the code: + [`getting-started.md`](./self-hosting/getting-started.md) (install and + updates, the page the boot log points at), the TrueNAS, Unraid and + Portainer installs with their catalog templates, Tailscale, the reverse + proxy, horizontal scaling (`HEALTHLOG_PROCESS_TYPE=web|worker|all`), + notification channels, the environment module, document sources and + reproductive-health data. Includes [`self-hosting/mcp.md`](./self-hosting/mcp.md) — enabling the off-by-default MCP connector and pointing Claude.ai / ChatGPT / Claude Desktop at it, and [`self-hosting/sso.md`](./self-hosting/sso.md) @@ -44,9 +48,11 @@ notes, and machine-readable specs that ship alongside the source: - [`migration/`](./migration/) — migration notes. Currently only v1.3 to v1.4; later releases are covered by `CHANGELOG.md` and . -- [`integrations/`](./integrations/) — per-integration reference - (Withings, Fitbit, Google Health, WHOOP, Apple Health, AI providers, - data import). +- [`integrations/`](./integrations/) — per-integration reference: + Withings, WHOOP, Fitbit, Google Health, Strava, Nightscout, Dawarich and + the Garmin route; the Apple Health, Health Connect, document and data + imports; AI providers; and Home Assistant with its blueprint. Oura and + Polar are covered on the docs site only. - [`adr/`](./adr/) — architecture decision records. - [`security/`](./security/) — threat-model notes. - [`diagrams/`](./diagrams/) — architecture diagrams rendered through @@ -70,4 +76,4 @@ notes, and machine-readable specs that ship alongside the source: If you're a self-hoster looking to spin up an instance, start at or jump to the [Quick Start in the project -README](../README.md#quick-start). +README](../README.md#quick-start-docker-compose). diff --git a/docs/integrations/apple-health.md b/docs/integrations/apple-health.md index dc7a3d615..33f72b545 100644 --- a/docs/integrations/apple-health.md +++ b/docs/integrations/apple-health.md @@ -295,7 +295,7 @@ Concrete consequences: RMSSD HRV has its own ladder, **WHOOP ≻ OURA ≻ POLAR ≻ APPLE_HEALTH**, separate from the SDNN one. -Override per-user via Settings → Sources (`/settings/sources`). +Override per-user via Settings → Source priority (`/settings/sources`). ## Failure modes diff --git a/docs/integrations/dawarich.md b/docs/integrations/dawarich.md index 7e19680f9..20540d2e7 100644 --- a/docs/integrations/dawarich.md +++ b/docs/integrations/dawarich.md @@ -70,7 +70,7 @@ The variable is on the `environment:` whitelist in `docker-compose.yml`. - Time away from home and the distance travelled join the correlation scan as two more factors. - Runs of at least two travel days that no trip covers yet are offered as - trips under **Settings → Environment → Location periods**. Using one fills + trips under **Settings → Location & weather → Location periods**. Using one fills in the dates; you pick the place. - The Coach and the MCP `get_day` tool see the figures and whether it was a travel day, never the country. diff --git a/docs/integrations/fitbit.md b/docs/integrations/fitbit.md index 4c5684c21..c9eb2eae5 100644 --- a/docs/integrations/fitbit.md +++ b/docs/integrations/fitbit.md @@ -103,7 +103,7 @@ The Fitbit client id/secret are per-user (Settings), not env vars. If Fitbit and another source (Withings, WHOOP, Apple Health) both supply the same vital — resting heart rate, blood oxygen, body temperature or respiratory rate — -the source priority ladder in **Settings → Sources** decides which one wins +the source priority ladder in **Settings → Source priority** decides which one wins per metric. FITBIT is ranked in every default metric ladder; override per metric on that page. diff --git a/docs/integrations/google-health.md b/docs/integrations/google-health.md index 069d09e83..3176a827d 100644 --- a/docs/integrations/google-health.md +++ b/docs/integrations/google-health.md @@ -305,7 +305,7 @@ triggers the verification + CASA requirements above.) If Google Health and another source (Withings, WHOOP, Apple Health) both supply the same vital — resting heart rate, blood oxygen, respiratory rate — the source -priority ladder in **Settings → Sources** decides which one wins +priority ladder in **Settings → Source priority** decides which one wins per metric. ## Disconnect diff --git a/docs/integrations/strava.md b/docs/integrations/strava.md index da6a7ed3a..5607894fb 100644 --- a/docs/integrations/strava.md +++ b/docs/integrations/strava.md @@ -34,7 +34,7 @@ default the device-native capture (Apple Watch, WHOOP, Withings) wins over the Strava copy, because a Strava activity is often a re-upload with lower-fidelity or missing heart-rate data. A run you record **only** on Strava (phone GPS) survives untouched. You can promote Strava above the wearables in -**Settings → Sources** if you prefer the Strava copy. +**Settings → Source priority** if you prefer the Strava copy. ## 1. Register a Strava API application diff --git a/docs/integrations/withings.md b/docs/integrations/withings.md index 99ddc858a..a211978c9 100644 --- a/docs/integrations/withings.md +++ b/docs/integrations/withings.md @@ -88,7 +88,7 @@ or right away after reconnecting Withings once. needed — `user.metrics` covers weight / BP / heart-list / sleep / SpO₂ / body comp / temperature / VO₂ max; `user.activity` is the workout-equivalent stream for steps / energy / distance / floors. - See `WITHINGS_OAUTH_SCOPE` in `src/lib/withings/client.ts`. + See `WITHINGS_OAUTH_SCOPE` in `src/lib/withings/oauth.ts`. 4. Save the application. Withings will issue a **Client ID** and a **Client Secret** — keep the secret out of any chat log; it grants read access to every linked account's full health history. @@ -145,7 +145,7 @@ callback. The portal only owns the OAuth callback URL. ## 4. Link from the Settings page With the credentials saved in Settings, open -`/settings/integrations/withings` and click **Connect Withings**. The flow: +`/settings/integrations#withings` and click **Connect Withings**. The flow: 1. The Connect button redirects to Withings' `oauth2/authorize`. 2. The user signs in on Withings and grants the requested scopes. @@ -181,7 +181,7 @@ import: when Apple Health is also linked, so the Apple Health stream wins for the cumulative axis. The Withings rows stay in the DB. -Override per-user via Settings → Sources (`/settings/sources`). +Override per-user via Settings → Source priority (`/settings/sources`). ## Troubleshooting @@ -199,7 +199,7 @@ Override per-user via Settings → Sources (`/settings/sources`). ### The webhook stops firing Webhook subscriptions can lapse after a token revoke or a portal- -side cleanup. Re-subscribe by opening `/settings/integrations/withings` +side cleanup. Re-subscribe by opening `/settings/integrations#withings` and clicking **Disconnect** then **Connect Withings** again — the callback exchange re-runs `Notify.subscribe()` with a fresh callback URL. diff --git a/docs/ops/deploy.md b/docs/ops/deploy.md index 0a1f8ff7c..b6cfee395 100644 --- a/docs/ops/deploy.md +++ b/docs/ops/deploy.md @@ -270,10 +270,13 @@ always confirm the served build after a deploy: ```sh # Asserts prod + demo serve 1.9.0, prints each target's buildSha, # exits non-zero on a mismatch or timeout after a bounded retry loop. -pnpm dlx tsx scripts/assert-deploy.ts 1.9.0 +HEALTHLOG_PROD_URL=https://healthlog.example.com \ + pnpm dlx tsx scripts/assert-deploy.ts 1.9.0 ``` -Pass `--only=prod` / `--only=demo` to check a single target, or +`HEALTHLOG_PROD_URL` is the base URL of your instance; it has no default and +is required unless you pass `--only=demo`. Pass `--only=prod` / +`--only=demo` to check a single target, or `--attempts=N --interval=ms` to widen the retry window while a slow pull settles. diff --git a/docs/ops/measurement-maintenance.md b/docs/ops/measurement-maintenance.md index f7ca054ab..c43caa52d 100644 --- a/docs/ops/measurement-maintenance.md +++ b/docs/ops/measurement-maintenance.md @@ -96,8 +96,9 @@ maintenance refuses to start while one is. Have at least twice the largest size free on the database volume. -- **Take a backup** (Settings, Data, Backups), as before any operation on the - whole table. +- **Take a backup**: a database dump, as in + [Updates](../self-hosting/getting-started.md#updates), as before any + operation on the whole table. ## Starting it diff --git a/docs/self-hosting/account-sharing.md b/docs/self-hosting/account-sharing.md index 7cc76a1de..9e75646fa 100644 --- a/docs/self-hosting/account-sharing.md +++ b/docs/self-hosting/account-sharing.md @@ -214,11 +214,15 @@ as something to log in as. or delete the profile under a fresh second factor. Deleting the profile deletes everything recorded for it. - A guardian may keep that record's settings: modules, units, thresholds, - its language and timezone, and its notification preferences. Connected - services are visible as a status only. Connect, callback, sync, - credential, disconnect, resume and test are not guardian surfaces in - this release, and their deep links land on the shared-record unavailable - page rather than on a raw refusal. + its language and timezone, and its notification preferences. A guardian + can also connect and disconnect the record's Withings, WHOOP, Oura, Polar, + Strava and Google Health from its integrations page; the data lands in the + profile's record, and a profile without its own provider app uses the + guardian's. Fitbit (no new connections), Nightscout and Dawarich cannot be + connected for a profile. Sync, the connection test and provider + credentials are not guardian surfaces: once connected, the record syncs on + its own schedule, and their deep links land on the shared-record + unavailable page rather than on a raw refusal. - **Its reminders go to its guardians.** A record with no login has no channels of its own, so medication reminders, measurement reminders, safety-floor alerts and low-stock alerts fan out to every active diff --git a/docs/self-hosting/environment.md b/docs/self-hosting/environment.md index 51dbb6d3d..095b63fd7 100644 --- a/docs/self-hosting/environment.md +++ b/docs/self-hosting/environment.md @@ -86,10 +86,11 @@ backfill button in settings fetches both for the range it is given. Since v1.42 the home, each dated location period and each stored day keep their coarse location sealed with the instance's encryption key (one value per row, `{ lat, lon, label }`, under the label -`healthlog/environment-location/v1`). The boot-time encryption backfill seals -rows written by older releases and clears their readable columns; the -readable columns are dropped in a later release once the backfill reports -zero rows everywhere. Key rotation (`scripts/rotate-encryption-key.ts`, or the +`healthlog/environment-location/v1`). The boot-time encryption backfill sealed +rows written by older releases and cleared their readable columns, and +migration `0409` (v1.43) dropped those columns; it refuses to run while a row +still holds a readable location +([migration-0409-plaintext-location.md](../ops/migration-0409-plaintext-location.md)). Key rotation (`scripts/rotate-encryption-key.ts`, or the in-app rotation) covers all three. What the person sees is unchanged: the settings page and diff --git a/docs/self-hosting/getting-started.md b/docs/self-hosting/getting-started.md index f4fb5d208..c791df566 100644 --- a/docs/self-hosting/getting-started.md +++ b/docs/self-hosting/getting-started.md @@ -9,8 +9,7 @@ already installed, longer if a TLS-fronted public hostname is in scope - **Docker 24+ with Compose v2.** The bundled `docker-compose.yml` pulls a multi-arch image from GitHub Container Registry (linux/amd64 - - linux/arm64), so a Raspberry Pi 5 or an x86 VPS both work - unchanged. + and linux/arm64), so a Raspberry Pi 5 or an x86 VPS both work unchanged. - **3 GB RAM (4 GB recommended), 10 GB free disk.** The Postgres data volume grows with measurement history; the Apple Health import worker can briefly hold a 1.5 GB upload in `/tmp` while it parses. The app @@ -23,11 +22,12 @@ already installed, longer if a TLS-fronted public hostname is in scope below the supported floor. On a busy instance, raise it. - **Outbound HTTPS** to `ghcr.io` (image pulls) plus whichever integration endpoints you plan to enable (Withings, OpenAI, etc.). - The opt-in environmental-context module additionally reaches - `archive-api.open-meteo.com` and `geocoding-api.open-meteo.com` for - weather and daylight; it stays off until you - enable it and set a home location, and every outbound call goes - through the SSRF-guarded egress wrapper. + The environmental-context module additionally reaches + `archive-api.open-meteo.com`, `air-quality-api.open-meteo.com` and + `geocoding-api.open-meteo.com` for weather, air quality and the city + search. It is on by default, but nothing leaves the host until someone + sets a home location, and every outbound call goes through the + SSRF-guarded egress wrapper. Three requests happen by default without anything connected: the HaveIBeenPwned range query when a password is set (only the first five characters of the password's SHA-1 are sent), a daily check of the diff --git a/docs/self-hosting/mcp.md b/docs/self-hosting/mcp.md index a54b2d1c5..9f4334175 100644 --- a/docs/self-hosting/mcp.md +++ b/docs/self-hosting/mcp.md @@ -19,7 +19,7 @@ on top of it, see [building skills](../api/mcp-skills.md). > **Read this before enabling writes.** A remote, write-capable connector > reached over OAuth is a powerful combination. Read the -> [trust model](#trust-model-read-this-before-enabling-writes) at the end +> [trust model](#trust-model--read-this-before-enabling-writes) at the end > of this page first. ## Prerequisites @@ -43,7 +43,7 @@ LAN-only origin works for Claude Desktop over stdio). The switch lives with every other module: **Settings → Modules**, the **MCP** row. Turn it on and the `/mcp` endpoint goes live for your account. -**Settings → MCP** is where you use it. The page has three parts: +**Settings → MCP connector** is where you use it. The page has three parts: - **State** — whether the module is on, with a link to its switch in Modules, and your endpoint URL (`/mcp`). Turning the module off @@ -55,7 +55,7 @@ The switch lives with every other module: **Settings → Modules**, the ## 2. Mint a connector token -Still in **Settings → MCP → Tokens**: +Still in **Settings → MCP connector → Tokens**: 1. Choose the scope with the **write** toggle: - **off** → a read-only token (`health:read`). This is the default and @@ -106,7 +106,7 @@ refused (`403`) as a DNS-rebinding defence. client-id-metadata-document clients with dynamic registration as a fallback. 3. Approve the consent screen. The connection then appears under - **Settings → MCP → Connections**, where you can revoke it. + **Settings → MCP connector → Connections**, where you can revoke it. No token to paste — OAuth mints and rotates a short-lived `health:read` access token (60 minutes, with a refresh token) behind the scenes. diff --git a/docs/self-hosting/reproductive-health-data.md b/docs/self-hosting/reproductive-health-data.md index 128dbc6d3..314b65aa2 100644 --- a/docs/self-hosting/reproductive-health-data.md +++ b/docs/self-hosting/reproductive-health-data.md @@ -95,7 +95,7 @@ different things depending on which button you press. two actions below, the row stays. A start that was folded into a later cycle stays for as long as that cycle does, because deleting the later cycle gives it back. -- **Settings → Delete All Data** removes the rows outright, tombstones +- **Settings → Advanced → Delete all data** removes the rows outright, tombstones included. Everything the account owns goes except what you sign in with; the confirmation dialog lists it. - **Deleting the account** removes the account row, and the database diff --git a/docs/self-hosting/truenas.md b/docs/self-hosting/truenas.md index d312e4fb7..f8a1b6328 100644 --- a/docs/self-hosting/truenas.md +++ b/docs/self-hosting/truenas.md @@ -49,9 +49,10 @@ HealthLog behind a reverse proxy with its own domain, change App URL to that Settings the form does not show go under Additional Environment Variables, for example `UPDATE_CHECK_DISABLED=1`. The full list is in -[environment.md](environment.md). To use your own GeoLite2 databases, add the -directory under Additional Storage and point `GEOLITE2_DIR` at its mount path -([geolite2.md](geolite2.md)). +[`.env.production.example`](../../.env.production.example) and on the +[environment variables reference](https://docs.healthlog.dev/reference/environment-variables/). +To use your own GeoLite2 databases, add the directory under Additional Storage +and point `GEOLITE2_DIR` at its mount path ([geolite2.md](geolite2.md)). ## Plain HTTP on the LAN @@ -206,15 +207,15 @@ before you touch it. Until the catalog app is listed, install HealthLog with Apps, Discover Apps, the menu next to Custom App, Install via YAML, using these values: -| Setting | Value | -| ------------ | ---------------------------------------------------------------------------------------- | -| Image | `ghcr.io/mbombeck/healthlog`, tag `X.Y.Z` (amd64 and arm64) | -| Port | `3000` (HTTP) | -| User | runs as uid `1001` by default; no extra capabilities, `no-new-privileges` is fine | -| Health check | `GET /api/health`, `200` when ready | -| Start period | at least 60 s, with more retries than the image's 5 × 30 s: migrations run at every boot | -| Database | PostgreSQL 18 (16 works too), no extensions, in its own container with its own dataset | -| Volumes | none for the app; the database gets its own dataset | +| Setting | Value | +| ------------ | -------------------------------------------------------------------------------------- | +| Image | `ghcr.io/mbombeck/healthlog`, tag `X.Y.Z` (amd64 and arm64) | +| Port | `3000` (HTTP) | +| User | runs as uid `1001` by default; no extra capabilities, `no-new-privileges` is fine | +| Health check | `GET /api/health`, `200` when ready | +| Start period | 10 min, like the image: migrations run at every boot and wait out a held lock | +| Database | PostgreSQL 18 (16 works too), no extensions, in its own container with its own dataset | +| Volumes | none for the app; the database gets its own dataset | | Variable | Required | Notes | | ----------------------- | -------- | ---------------------------------------------------------------------------------------- | diff --git a/docs/self-hosting/unraid.md b/docs/self-hosting/unraid.md index 87c0e91df..e0d9564b6 100644 --- a/docs/self-hosting/unraid.md +++ b/docs/self-hosting/unraid.md @@ -116,8 +116,8 @@ PostgreSQL container you installed in Step 1. Back that one up. `SESSION_COOKIE_SECURE=false` is required for plain-HTTP LAN installs (`http://tower:3000`). Left at its default, the session cookie carries the -`Secure` flag, the browser refuses to send it over HTTP, and login appears to -"do nothing" — the page reloads to the login screen with no error. +`Secure` flag, which the browser would drop over HTTP. The login page says so +before anything is sent: "Signing in cannot complete on this page". Set `SESSION_COOKIE_SECURE=true` (or leave it unset) only when a TLS reverse proxy serves HealthLog over HTTPS. In that case also point `NEXT_PUBLIC_APP_URL` diff --git a/docs/ui-guidelines.md b/docs/ui-guidelines.md index 093a0ef35..5b00604e2 100644 --- a/docs/ui-guidelines.md +++ b/docs/ui-guidelines.md @@ -311,7 +311,7 @@ Tiles at the top of the dashboard live in a **single row**. Width caps at the chart's width directly below. **Never** wrap to a second row. If the user enables more tiles than fit, the dashboard surfaces a banner -inviting them to deactivate or reorder via Settings → Dashboard. The +inviting them to deactivate or reorder via Settings → Arrangement. The layout editor in Settings shows a live preview of the constraint and flags overflow before save. From a641a615c31931e5f7da3fe802c507a2c81f9abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 11:56:28 +0100 Subject: [PATCH 08/19] docs: point the Strava and Dawarich notes at their site guides Both files had drifted copies of the docs-site guides, which carry the same sections and more, and nothing in the app, the scripts, the other docs or the site links to them. Each is now a short pointer to its guide. --- docs/integrations/dawarich.md | 85 +++-------------------------------- docs/integrations/strava.md | 76 ++----------------------------- 2 files changed, 9 insertions(+), 152 deletions(-) diff --git a/docs/integrations/dawarich.md b/docs/integrations/dawarich.md index 20540d2e7..3f16b3f08 100644 --- a/docs/integrations/dawarich.md +++ b/docs/integrations/dawarich.md @@ -1,82 +1,7 @@ # Dawarich integration -HealthLog can read daily movement context from a [Dawarich](https://dawarich.app) -instance: how far you moved, how much of it by car or by train, how many places -you visited, how long you were away from home, and whether a day was spent in -another country. It is part of the Environment module and runs only while that -module is on. - -HealthLog never stores a coordinate, a track, a place name or a visit. It reads -the day summaries and keeps one row of numbers per day. The place list is read -live when you pick your home and is not saved. - -## What is read - -Once a night (03:20 Europe/Berlin) HealthLog fetches the last seven days, ending -yesterday. Connecting fetches the last 30 days. Each request carries your API -key as `Authorization: Bearer `. - -| Dawarich endpoint | What HealthLog keeps | -| ----------------------- | -------------------------------------------------------------------------------------- | -| `GET /api/v1/timeline` | total distance, distance by car (`driving`), distance by train, places visited | -| `GET /api/v1/visits` | minutes spent at your home place, to work out time away from home | -| `GET /api/v1/residency` | the country of each day, sealed at rest, and whether it differs from your home country | -| `GET /api/v1/places` | nothing; the list is shown once so you can pick your home | -| `GET /api/v1/users/me` | nothing; used to check the key | - -Time away from home is the day's length minus the time a visit at your home -place covers. Without a home place, or on a day without a single visit, it is -left empty rather than shown as zero. Dawarich leaves out a mode that moved -less than half a kilometre; HealthLog then reads that mode as zero for the day. - -Residency needs API access on a hosted Dawarich plan; a self-hosted instance -has it. Without it there are no travel days, and everything else still works. -Dawarich keys residency by UTC date, so on a day that crosses a border late in -the evening the country can be the next morning's. - -## Connect - -1. In Dawarich, open your account settings and copy the API key. -2. In HealthLog, turn on the Environment module, then open - **Settings → Integrations → Dawarich**. -3. Enter the address of the instance and the key, and connect. -4. Pick your home place from the list, and set your home country if you like. - Without one, the country you spent most days in over the last year counts. - -Run Dawarich and HealthLog in the same time zone. Dawarich groups the -timeline by its own zone, HealthLog by yours; if they differ, a day can come -back without its distances. - -## A Dawarich instance on your own network - -A public instance works without any server setting. An instance on a private -network (a LAN address, a `.lan` name, a container on the same host) is refused -unless the operator lists its exact origin: - -```env -DAWARICH_PRIVATE_ORIGINS="http://dawarich.lan:3000" -``` - -The grammar is the one `NIGHTSCOUT_PRIVATE_ORIGINS` uses -([Nightscout](./nightscout.md)): one `scheme://host[:port]` per entry, comma -separated, no path, no wildcard and no range. Listing a host does not open its -subdomains or another port. The unspecified address, link-local and the -cloud-metadata range can never be listed. Restart HealthLog after changing it. -The variable is on the `environment:` whitelist in `docker-compose.yml`. - -## Where the values show up - -- The day view has an **On the move** section with the day's figures. -- Time away from home and the distance travelled join the correlation scan as - two more factors. -- Runs of at least two travel days that no trip covers yet are offered as - trips under **Settings → Location & weather → Location periods**. Using one fills - in the dates; you pick the place. -- The Coach and the MCP `get_day` tool see the figures and whether it was a - travel day, never the country. - -## Disconnect - -Disconnecting stops the nightly fetch and keeps the stored days. **Disconnect -and delete days** removes them as well. The days are part of the backup; the -address and key are not, so a restored account connects again. +The Dawarich guide lives on the docs site: +. It covers what is read +and what is never stored, connecting an instance, allowing one on your own +network (`DAWARICH_PRIVATE_ORIGINS`), where the values show up, and +disconnecting. diff --git a/docs/integrations/strava.md b/docs/integrations/strava.md index 5607894fb..3e9bb2804 100644 --- a/docs/integrations/strava.md +++ b/docs/integrations/strava.md @@ -1,74 +1,6 @@ # Strava integration -HealthLog reads your Strava activities and folds them into the same workout -history as every other source. Strava is a **workout source only** — its API is -activity-centric and exposes no sleep, recovery, body-composition or glucose -data, so connecting it adds runs, rides, swims and the rest to your workout -surface and nothing else. - -The OAuth app is brought **per user**: each self-hoster registers their own -Strava API application and pastes the client id/secret into Settings. Strava -caps every newly-created app at "single-player mode" (an athlete capacity of 1) -until you request a capacity increase — perfectly fine for a personal -self-host, and the reason a single shared app cannot serve many operators. - -## What you get — and what you don't - -**Covered** (one `Workout` row per activity): sport type, start and end time, -moving duration, distance, elevation gain, average and maximum heart rate (when -the activity recorded it), and calories. Power, cadence, the "trainer"/"commute" -flags and the activity name are kept on the row's metadata for provenance but -are **not** surfaced as training analytics — HealthLog treats Strava as another -workout source, not a training-load platform. - -**Not covered**: Strava exposes no daily metrics (steps, sleep, resting heart -rate, weight, …). Those come from your wearable's own integration or from Apple -Health / Google Health Connect. - -## Duplicate activities are collapsed automatically - -If you record a run on an Apple Watch **and** it also lands in Strava, the same -run arrives twice — once via Apple Health, once via Strava. HealthLog keeps one -canonical row per logical workout using your **source-priority ladder**: by -default the device-native capture (Apple Watch, WHOOP, Withings) wins over the -Strava copy, because a Strava activity is often a re-upload with lower-fidelity -or missing heart-rate data. A run you record **only** on Strava (phone GPS) -survives untouched. You can promote Strava above the wearables in -**Settings → Source priority** if you prefer the Strava copy. - -## 1. Register a Strava API application - -1. Sign in at . -2. Create an application. For **Authorization Callback Domain** enter the host - of your HealthLog instance (the domain only, no scheme or path). -3. Copy the **Client ID** and **Client Secret**. - -## 2. Paste the credentials into HealthLog - -1. Open **Settings → Integrations** and find the **Strava** card. -2. Paste the Client ID and Client Secret into the **API Credentials** fields and - save. (An operator can instead configure a shared app via the - `STRAVA_CLIENT_ID` / `STRAVA_CLIENT_SECRET` environment variables; per-user - credentials always take precedence.) -3. Click **Connect Strava** and approve the access request. HealthLog requests - the `activity:read_all` scope so it can read every activity, including those - you marked "only me". - -## 3. First sync - -The first history import runs in the background shortly after you connect; -afterwards an hourly poll pulls new activities. Strava rotates its refresh token -on every refresh, and HealthLog persists the rotated token each time, so the -connection stays healthy without you re-authorising. - -## Disconnecting - -Disconnecting revokes the grant at Strava and clears the stored tokens. -Activities already imported stay in your history; reconnecting resumes the sync. - -## Related - -- Wearable that syncs to Strava but not directly to HealthLog? See the device's - own page, or ingest via [Apple Health](./apple-health.md) / - [Google Health Connect](./google-health.md). -- **Garmin owners:** Garmin has no direct connector — see [garmin.md](./garmin.md). +The Strava guide lives on the docs site: +. It covers what Strava +adds (workouts only), how duplicate activities from a wearable are collapsed, +registering the Strava API application, the first sync and disconnecting. From 93fb0f68d89651d7e8ec2d2a36db1c0a15f9c183 Mon Sep 17 00:00:00 2001 From: Usama Muhammad <46465291+muhdusama@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:17:59 +1100 Subject: [PATCH 09/19] fix(mcp): declare labs availability metadata (cherry picked from commit c1a5861fdcbfe374fcbb5362eed3717a5bd9b15d) --- src/lib/mcp/__tests__/tools.test.ts | 46 +++++++++++++++++++++++++++++ src/lib/mcp/tools.ts | 7 ++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/lib/mcp/__tests__/tools.test.ts b/src/lib/mcp/__tests__/tools.test.ts index 5b9d9a8c1..3de4475d4 100644 --- a/src/lib/mcp/__tests__/tools.test.ts +++ b/src/lib/mcp/__tests__/tools.test.ts @@ -510,6 +510,52 @@ describe("get_illness_recovery — condition body site", () => { }); describe("get_labs", () => { + it.each([{}, { analyte: "LDL" }])( + "declares empty-read availability for ordinary and filtered reads: %j", + async (args) => { + const { z } = await import("zod/v4"); + const empty = { + present: false, + reason: "out_of_window", + searchedWindow: "lastYear", + available: { + count: 2, + firstDate: "2020-01-01", + lastDate: "2020-01-02", + reachableWithWindow: null, + }, + }; + vi.mocked(executeCoachTool).mockResolvedValue(empty); + const def = tool("get_labs"); + const result = await def.run(CTX, args); + expect(executeCoachTool).toHaveBeenCalledWith({ + userId: CTX.userId, + name: "get_labs", + rawArguments: JSON.stringify(args), + }); + expect(z.object(def.outputShape!).strict().parse(result)).toEqual(empty); + }, + ); + + it("keeps output validation closed and availability typed", async () => { + const { z } = await import("zod/v4"); + const schema = z.object(tool("get_labs").outputShape!).strict(); + expect(z.toJSONSchema(schema).additionalProperties).toBe(false); + expect(schema.safeParse({ present: false, unexpected: true }).success).toBe( + false, + ); + expect( + schema.safeParse({ present: false, searchedWindow: 7 }).success, + ).toBe(false); + expect( + schema.safeParse({ present: false, available: { count: "2" } }).success, + ).toBe(false); + expect(schema.parse({ present: false, reason: "no_data" })).toEqual({ + present: false, + reason: "no_data", + }); + }); + it("forwards the optional analyte filter and returns readings with units + bands", async () => { vi.mocked(executeCoachTool).mockResolvedValue({ present: true, diff --git a/src/lib/mcp/tools.ts b/src/lib/mcp/tools.ts index 2c9813c30..c26fc1daf 100644 --- a/src/lib/mcp/tools.ts +++ b/src/lib/mcp/tools.ts @@ -1201,11 +1201,8 @@ const listMetricsOutput: z.ZodRawShape = { /** Output schema for `get_labs` — latest-per-biomarker OR a paginated history. */ const getLabsOutput: z.ZodRawShape = { - present: z.boolean(), - reason: z.string().optional(), - // Latest-mode payload (the Coach labs section, carried opaque). - data: z.unknown().optional(), - grounding: z.string().optional(), + // Latest-mode payload, including typed empty-read availability metadata. + ...coachReadOutput, // History-mode payload (per-analyte trajectory + pagination). analyte: z.string().optional(), readings: z From 6d3a1582fa3b75224d75254a950c4fb53b50ca19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 12:51:39 +0100 Subject: [PATCH 10/19] feat(backup): take a nightly whole-database dump with a manifest The per-account copies roll one account back on the same server; nothing the app made could rebuild a lost one. A scheduled job now runs pg_dump over the whole database (every schema, pg-boss included) into DB_BACKUP_DIR as healthlog--daily.dump, with a manifest beside it: app version and build, newest migration, Postgres version, size and SHA-256, exact row counts of every table, and the fingerprints of the keys the database needs. The counts read the same exported snapshot pg_dump reads, so a dump taken while the app writes still matches its manifest. The archive is written as .partial, listed back with pg_restore --list, and renamed after its manifest; a failed run leaves nothing named like a good dump. Retention removes only files this job wrote, oldest first, and never the newest good one (DB_BACKUP_KEEP, default 7; two pre-migration dumps). With the off-host bucket configured, each dump and its manifest go up under database/ in the envelope the account copies use, streamed. Every run lands in the new database_backup_runs ledger (migration 0486), and getDatabaseBackupHealth() answers the last good dump, the last failure, whether the directory is writable and whether off-host is configured. The boot readiness summary says where dumps go, or that none can be kept. The dump, naming, retention and restore rules live in one plain ESM module so the image's own scripts can share them without tsx. --- .../0486_database_backup_runs/migration.sql | 30 + prisma/schema.prisma | 29 + .../boot/__tests__/readiness-summary.test.ts | 53 + src/lib/boot/readiness-summary.ts | 78 +- src/lib/data-wipe/wipe-plan.ts | 2 + .../__tests__/dump-core.test.ts | 630 +++++++++ src/lib/database-backup/dump-core.mjs | 1183 +++++++++++++++++ src/lib/database-backup/health.ts | 117 ++ src/lib/database-backup/run.ts | 243 ++++ src/lib/jobs/database-backup-queue.ts | 40 + src/lib/jobs/job-outcome.ts | 6 + src/lib/jobs/offhost-backup.ts | 8 +- src/lib/jobs/queue-runtime.ts | 12 + src/lib/jobs/registry/maintenance.ts | 18 +- src/lib/jobs/reminder/backup-handlers.ts | 33 +- src/lib/jobs/subsystem-surface.ts | 1 + .../__golden__/queue-registration.json | 40 +- tests/integration/database-backup.test.ts | 511 +++++++ 18 files changed, 3025 insertions(+), 9 deletions(-) create mode 100644 prisma/migrations/0486_database_backup_runs/migration.sql create mode 100644 src/lib/database-backup/__tests__/dump-core.test.ts create mode 100644 src/lib/database-backup/dump-core.mjs create mode 100644 src/lib/database-backup/health.ts create mode 100644 src/lib/database-backup/run.ts create mode 100644 src/lib/jobs/database-backup-queue.ts create mode 100644 tests/integration/database-backup.test.ts diff --git a/prisma/migrations/0486_database_backup_runs/migration.sql b/prisma/migrations/0486_database_backup_runs/migration.sql new file mode 100644 index 000000000..37bd430af --- /dev/null +++ b/prisma/migrations/0486_database_backup_runs/migration.sql @@ -0,0 +1,30 @@ +-- The nightly whole-database dump records each run: whether a dump was +-- written in DB_BACKUP_DIR and copied to the off-host bucket, and why not. +-- Instance metadata with no account; the admin backups card and the backup +-- alerts read the newest rows. +-- +-- The table is new, so nothing waits on its lock in practice; the wait is +-- bounded in a DO block anyway, and every statement is safe to run again, so +-- the boot retries the file on a lock timeout. +-- +-- @retry-on-lock-timeout + +DO $migration$ +BEGIN + PERFORM set_config('lock_timeout', '5s', true); + CREATE TABLE IF NOT EXISTS "database_backup_runs" ( + "id" TEXT NOT NULL, + "started_at" TIMESTAMP(3) NOT NULL, + "finished_at" TIMESTAMP(3), + "status" TEXT NOT NULL, + "file_name" TEXT, + "dump_bytes" BIGINT, + "local_result" TEXT NOT NULL, + "offhost_result" TEXT NOT NULL, + "error" TEXT, + CONSTRAINT "database_backup_runs_pkey" PRIMARY KEY ("id") + ); + CREATE INDEX IF NOT EXISTS "database_backup_runs_started_at_idx" + ON "database_backup_runs"("started_at"); +END +$migration$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8397e0cb5..ae2d0c292 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5929,6 +5929,35 @@ model OffhostPurgeRequest { @@map("offhost_purge_requests") } +/// One run of the nightly whole-database dump (the `database-backup` job): +/// whether a dump was written on this server and copied off-host, and why not +/// when it was not. Instance metadata, not an account's data. Written by +/// `runDatabaseBackup`; read by `getDatabaseBackupHealth`, which feeds the +/// database row of the admin backups card and the backup alerts. A row left +/// `running` by a process that died is read as a failed run once the job's +/// expiry has passed. +model DatabaseBackupRun { + id String @id @default(cuid()) + startedAt DateTime @map("started_at") + finishedAt DateTime? @map("finished_at") + /// `running`, `succeeded`, `failed` or `skipped`. + status String + /// The dump's file name, once one was written and checked. + fileName String? @map("file_name") + /// Size of the dump in bytes. + dumpBytes BigInt? @map("dump_bytes") + /// `written` (kept in DB_BACKUP_DIR), `no_directory` (no writable + /// directory, the dump went off-host only), or `failed`. + localResult String @map("local_result") + /// `uploaded`, `not_configured` or `failed`. + offhostResult String @map("offhost_result") + /// Why the run failed or was skipped, trimmed. Never key material. + error String? + + @@index([startedAt]) + @@map("database_backup_runs") +} + // ─── Apple Health export.zip import (v1.4.34) ─────────────── // // Mirrors a `pg-boss` `apple-health-import` job's lifecycle so the diff --git a/src/lib/boot/__tests__/readiness-summary.test.ts b/src/lib/boot/__tests__/readiness-summary.test.ts index fc23b92ee..003ade157 100644 --- a/src/lib/boot/__tests__/readiness-summary.test.ts +++ b/src/lib/boot/__tests__/readiness-summary.test.ts @@ -149,3 +149,56 @@ describe("collectReadiness — retired AI grant", () => { expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined(); }); }); + +describe("collectReadiness — the database dump", () => { + const offhost = { + BACKUP_S3_ENDPOINT: "https://s3.example.test", + BACKUP_S3_BUCKET: "b", + BACKUP_S3_ACCESS_KEY: "a", + BACKUP_S3_SECRET_KEY: "s", + BACKUP_ENCRYPTION_KEY: HEX32, + }; + const dumpLine = ( + env: Record, + writable: boolean, + ) => + collectReadiness(env, () => writable).lines.find( + (l) => l.label === "Database dump", + ); + + it("names the schedule, the directory and the copies kept", () => { + const line = dumpLine(baseEnv(), true); + expect(line?.status).toBe("ok"); + expect(line?.detail).toBe( + "nightly (30 1 * * *) into /var/lib/healthlog/backups, keeping 7", + ); + expect(dumpLine({ ...baseEnv(), ...offhost }, true)?.detail).toContain( + "copied off-host", + ); + }); + + it("warns when nothing can be kept on the server", () => { + expect(dumpLine({ ...baseEnv(), ...offhost }, false)).toMatchObject({ + status: "warn", + detail: expect.stringContaining("goes off-host only"), + }); + expect(dumpLine(baseEnv(), false)).toMatchObject({ + status: "warn", + detail: expect.stringContaining("no database dump can be taken"), + }); + }); + + it("warns when it is switched off, and names a schedule it ignored", () => { + expect( + dumpLine({ ...baseEnv(), DB_BACKUP_ENABLED: "false" }, true), + ).toMatchObject({ status: "warn" }); + const report = collectReadiness( + { ...baseEnv(), DB_BACKUP_SCHEDULE: "nightly" }, + () => true, + ); + expect( + report.lines.find((l) => l.label === "DB_BACKUP_SCHEDULE")?.detail, + ).toBe("not a five-field cron; using 30 1 * * *"); + expect(report.hasBlocker).toBe(false); + }); +}); diff --git a/src/lib/boot/readiness-summary.ts b/src/lib/boot/readiness-summary.ts index 14e707157..e8907cb29 100644 --- a/src/lib/boot/readiness-summary.ts +++ b/src/lib/boot/readiness-summary.ts @@ -19,6 +19,13 @@ import { LEGACY_ANY_HOST_WARNING, legacyAnyHostConfigured, } from "@/lib/ai/local-host-allowlist"; +import { + databaseBackupEnabled, + isWritableDirectory, + resolveBackupDir, + resolveBackupSchedule, + resolveDailyKeep, +} from "@/lib/database-backup/dump-core.mjs"; export interface ReadinessLine { /** Short label, e.g. "ENCRYPTION_KEY". */ @@ -60,8 +67,73 @@ function anySet(env: Env, keys: string[]): boolean { return keys.some((k) => (env[k] ?? "").trim().length > 0); } -/** Inspect the environment and build the readiness report. Pure + side-effect free. */ -export function collectReadiness(env: Env = process.env): ReadinessReport { +/** + * The nightly database dump's line. The dump is what rebuilds a lost server, + * so a host where it cannot be kept anywhere is said out loud at every boot, + * and so is one where it only goes off-host. + */ +export function databaseBackupReadiness( + env: Env, + writable: (dir: string) => boolean, +): ReadinessLine[] { + const label = "Database dump"; + if (!databaseBackupEnabled(env)) { + return [ + { + label, + status: "warn", + detail: + "DB_BACKUP_ENABLED=false: no nightly database dump is taken; back the database up another way", + }, + ]; + } + const lines: ReadinessLine[] = []; + const schedule = resolveBackupSchedule(env); + if (!schedule.valid) { + lines.push({ + label: "DB_BACKUP_SCHEDULE", + status: "warn", + detail: `not a five-field cron; using ${schedule.cron}`, + }); + } + const dir = resolveBackupDir(env); + const offhost = allSet(env, [ + "BACKUP_S3_ENDPOINT", + "BACKUP_S3_BUCKET", + "BACKUP_S3_ACCESS_KEY", + "BACKUP_S3_SECRET_KEY", + "BACKUP_ENCRYPTION_KEY", + ]); + if (writable(dir)) { + lines.push({ + label, + status: "ok", + detail: `nightly (${schedule.cron}) into ${dir}, keeping ${resolveDailyKeep(env)}${offhost ? ", copied off-host" : ""}`, + }); + } else if (offhost) { + lines.push({ + label, + status: "warn", + detail: `${dir} is not writable: the nightly dump goes off-host only and none is kept on this server. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`, + }); + } else { + lines.push({ + label, + status: "warn", + detail: `${dir} is not writable and no off-host bucket is configured: no database dump can be taken. Mount the backups volume (see docker-compose.yml) or set DB_BACKUP_DIR`, + }); + } + return lines; +} + +/** + * Inspect the environment and build the readiness report. Pure apart from + * `writable`, the probe of the dump directory, which tests replace. + */ +export function collectReadiness( + env: Env = process.env, + writable: (dir: string) => boolean = isWritableDirectory, +): ReadinessReport { const lines: ReadinessLine[] = []; // --- Core secrets (boot blockers) --------------------------------------- @@ -139,6 +211,8 @@ export function collectReadiness(env: Env = process.env): ReadinessReport { }); } + lines.push(...databaseBackupReadiness(env, writable)); + // --- Optional subsystems (informational) -------------------------------- const optional: Array<[string, boolean]> = [ ["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])], diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 92cead915..211e00af6 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -353,6 +353,8 @@ export const INSTANCE_SCOPED: Readonly> = { HostMetric: "instance-wide host telemetry, not attributable to an account", OffhostBackupKeyUse: "per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account", + DatabaseBackupRun: + "the instance's ledger of nightly whole-database dumps; describes the server, not any account", OffhostPurgeRequest: "the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account", CycleSymptomCategory: diff --git a/src/lib/database-backup/__tests__/dump-core.test.ts b/src/lib/database-backup/__tests__/dump-core.test.ts new file mode 100644 index 000000000..fad98e4a3 --- /dev/null +++ b/src/lib/database-backup/__tests__/dump-core.test.ts @@ -0,0 +1,630 @@ +/** + * The rules of the whole-database dump that need no database: names, + * retention, the manifest, key fingerprints, row-count comparison, the + * connection the client tools get, and the off-host envelope read back by the + * restore's own reader. + * + * The fingerprints and the envelope are checked against the app's own code + * (`src/lib/crypto.ts`, `src/lib/jobs/offhost-backup.ts`), not against a copy + * of the rule: the restore script cannot import TypeScript, so these are the + * tests that keep its plain-JS reading of both from drifting. + */ +import { + mkdtempSync, + createReadStream, + rmSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + applyRetention, + buildManifest, + compareKeyFingerprints, + compareRowCounts, + connectionFromUrl, + decodeKeyMaterial, + decryptOffhostFile, + dumpBaseName, + formatDumpTimestamp, + keyFingerprints, + listDumpSets, + offhostObjectKey, + parseDumpFileName, + parseManifest, + resolveBackupSchedule, + resolveDailyKeep, + selectForDeletion, + type DumpSet, +} from "../dump-core.mjs"; +import type { S3Like } from "@/lib/jobs/offhost-backup"; + +/** Every byte of a stream, in order. */ +async function collect(stream: AsyncIterable): Promise { + const chunks: Buffer[] = []; + for await (const c of stream) chunks.push(Buffer.from(c as Uint8Array)); + return Buffer.concat(chunks); +} + +const HEX_A = "a".repeat(64); +const HEX_B = "b".repeat(64); +const HEX_C = "c".repeat(64); +const HEX_D = "d".repeat(64); + +const dirs: string[] = []; +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), "hl-dump-core-")); + dirs.push(dir); + return dir; +} +afterEach(() => { + for (const dir of dirs.splice(0)) + rmSync(dir, { recursive: true, force: true }); + vi.unstubAllEnvs(); +}); + +describe("dump names", () => { + it("names a dump by its UTC instant and kind", () => { + const at = new Date("2026-10-25T01:30:07.123Z"); + expect(formatDumpTimestamp(at)).toBe("20261025T013007Z"); + expect(dumpBaseName("daily", at)).toBe("healthlog-20261025T013007Z-daily"); + expect(dumpBaseName("pre-migration", at)).toBe( + "healthlog-20261025T013007Z-pre-migration", + ); + }); + + it("parses every file it writes, and back to the same instant", () => { + const base = "healthlog-20261025T013007Z-daily"; + const cases: Array<[string, string, string]> = [ + [`${base}.dump`, "dump", "final"], + [`${base}.manifest.json`, "manifest", "final"], + [`${base}.dump.partial`, "dump", "partial"], + [`${base}.manifest.json.partial`, "manifest", "partial"], + [`${base}.dump.enc`, "dump", "encrypted"], + [`${base}.manifest.json.enc`, "manifest", "encrypted"], + ]; + for (const [name, role, variant] of cases) { + const parsed = parseDumpFileName(name); + expect(parsed, name).not.toBeNull(); + expect(parsed?.base).toBe(base); + expect(parsed?.kind).toBe("daily"); + expect(parsed?.role).toBe(role); + expect(parsed?.variant).toBe(variant); + expect(parsed?.timestamp.toISOString()).toBe("2026-10-25T01:30:07.000Z"); + } + expect( + parseDumpFileName("healthlog-20261025T013007Z-pre-migration.dump")?.kind, + ).toBe("pre-migration"); + }); + + it("refuses every name it did not write", () => { + for (const name of [ + "healthlog.dump", + "backup-20261025T013007Z-daily.dump", + "healthlog-20261025T013007Z-weekly.dump", + "healthlog-20261025-daily.dump", + "healthlog-20261025T013007Z-daily.sql", + "healthlog-20261025T013007Z-daily.dump.bak", + "healthlog-20261399T013007Z-daily.dump", + "notes.txt", + ]) { + expect(parseDumpFileName(name), name).toBeNull(); + } + }); +}); + +describe("configuration", () => { + it("keeps at least one daily dump and ignores what is not a number", () => { + expect(resolveDailyKeep({})).toBe(7); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "14" })).toBe(14); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "0" })).toBe(1); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "-3" })).toBe(7); + expect(resolveDailyKeep({ DB_BACKUP_KEEP: "seven" })).toBe(7); + }); + + it("takes a five-field cron and says when it ignored one", () => { + expect(resolveBackupSchedule({})).toEqual({ + cron: "30 1 * * *", + valid: true, + configured: false, + }); + expect( + resolveBackupSchedule({ DB_BACKUP_SCHEDULE: " 0 2 * * 1-5 " }), + ).toEqual({ + cron: "0 2 * * 1-5", + valid: true, + configured: true, + }); + for (const bad of ["daily", "0 2 * *", "0 2 * * * *", "0 2 * * mon; rm"]) { + expect(resolveBackupSchedule({ DB_BACKUP_SCHEDULE: bad })).toEqual({ + cron: "30 1 * * *", + valid: false, + configured: true, + }); + } + }); +}); + +/** A dump set as `listDumpSets` reports it, for the pure retention rule. */ +function set( + kind: "daily" | "pre-migration", + iso: string, + parts: { dump?: boolean; manifest?: boolean; partialAgeMs?: number } = {}, + now = Date.parse("2026-10-25T12:00:00Z"), +): DumpSet { + const base = dumpBaseName(kind, new Date(iso)); + return { + base, + kind, + timestamp: new Date(iso), + dump: parts.dump === false ? null : { path: `/b/${base}.dump`, bytes: 1 }, + manifest: parts.manifest === false ? null : `/b/${base}.manifest.json`, + encryptedDump: null, + encryptedManifest: null, + leftovers: + parts.partialAgeMs === undefined + ? [] + : [ + { + path: `/b/${base}.dump.partial`, + mtimeMs: now - parts.partialAgeMs, + }, + ], + }; +} + +describe("retention", () => { + const now = Date.parse("2026-10-25T12:00:00Z"); + + it("keeps the newest N daily dumps and removes the rest, both files each", () => { + const sets = [1, 2, 3, 4, 5].map((d) => + set("daily", `2026-10-2${d}T01:30:00Z`), + ); + const doomed = selectForDeletion(sets, { + keep: { daily: 3, "pre-migration": 2 }, + now, + }); + expect(doomed.sort()).toEqual( + [ + "/b/healthlog-20261021T013000Z-daily.dump", + "/b/healthlog-20261021T013000Z-daily.manifest.json", + "/b/healthlog-20261022T013000Z-daily.dump", + "/b/healthlog-20261022T013000Z-daily.manifest.json", + ].sort(), + ); + }); + + it("never removes the newest good dump, whatever keep says", () => { + const sets = [ + set("daily", "2026-10-24T01:30:00Z"), + set("daily", "2026-10-23T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 0, "pre-migration": 0 }, + now, + }); + expect(doomed).not.toContain("/b/healthlog-20261024T013000Z-daily.dump"); + expect(doomed).toContain("/b/healthlog-20261023T013000Z-daily.dump"); + }); + + it("counts only good dumps: an unfinished newer one does not push out the newest good one", () => { + const sets = [ + set("daily", "2026-10-25T01:30:00Z", { + dump: false, + partialAgeMs: 60_000, + }), + set("daily", "2026-10-24T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 1, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([]); + }); + + it("keeps two pre-migration dumps apart from the daily count", () => { + const sets = [ + set("pre-migration", "2026-10-20T08:00:00Z"), + set("pre-migration", "2026-10-10T08:00:00Z"), + set("pre-migration", "2026-10-01T08:00:00Z"), + set("daily", "2026-10-24T01:30:00Z"), + ]; + const doomed = selectForDeletion(sets, { + keep: { daily: 1, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([ + "/b/healthlog-20261001T080000Z-pre-migration.dump", + "/b/healthlog-20261001T080000Z-pre-migration.manifest.json", + ]); + }); + + it("removes a .partial only once it is older than twelve hours", () => { + const fresh = set("daily", "2026-10-25T01:30:00Z", { + dump: false, + manifest: false, + partialAgeMs: 60 * 60_000, + }); + const stale = set("daily", "2026-10-23T01:30:00Z", { + dump: false, + manifest: false, + partialAgeMs: 13 * 60 * 60_000, + }); + const doomed = selectForDeletion([fresh, stale], { + keep: { daily: 7, "pre-migration": 2 }, + now, + }); + expect(doomed).toEqual([ + "/b/healthlog-20261023T013000Z-daily.dump.partial", + ]); + }); + + it("works on a real directory and leaves files it did not write alone", async () => { + const dir = tempDir(); + const names: string[] = []; + for (let day = 15; day <= 24; day++) { + const base = `healthlog-202610${day}T013000Z-daily`; + names.push(`${base}.dump`, `${base}.manifest.json`); + } + // Two that are not ours, and an off-host copy an operator put here. + names.push( + "my-own-dump.dump", + "healthlog-20261001T013000Z-daily.dump.bak", + "healthlog-20261002T013000Z-daily.dump.enc", + ); + for (const name of names) writeFileSync(join(dir, name), "x"); + // A crash's leftovers: one old, one from a run that may still be going. + const oldPartial = join( + dir, + "healthlog-20261003T013000Z-daily.dump.partial", + ); + writeFileSync(oldPartial, "x"); + const day = 24 * 60 * 60; + utimesSync(oldPartial, Date.now() / 1000 - day, Date.now() / 1000 - day); + writeFileSync( + join(dir, "healthlog-20261025T013000Z-daily.dump.partial"), + "x", + ); + + const removed = await applyRetention(dir, { dailyKeep: 7 }); + expect(removed).toHaveLength(7); + const left = (await readdir(dir)).sort(); + expect(left).toContain("my-own-dump.dump"); + expect(left).toContain("healthlog-20261001T013000Z-daily.dump.bak"); + expect(left).toContain("healthlog-20261002T013000Z-daily.dump.enc"); + expect(left).toContain("healthlog-20261025T013000Z-daily.dump.partial"); + expect(left).not.toContain("healthlog-20261003T013000Z-daily.dump.partial"); + expect(left.filter((n) => n.endsWith("-daily.dump"))).toEqual( + [18, 19, 20, 21, 22, 23, 24].map( + (d) => `healthlog-202610${d}T013000Z-daily.dump`, + ), + ); + expect(listDumpSets(dir)[1].base).toBe("healthlog-20261024T013000Z-daily"); + }); +}); + +describe("key fingerprints", () => { + it("are the fingerprints the app's own key code gives, for the map and the single key", async () => { + vi.stubEnv("ENCRYPTION_KEYS", JSON.stringify({ v1: HEX_A, v2: HEX_B })); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "v2"); + vi.stubEnv("ENCRYPTION_KEY", ""); + const crypto = await import("@/lib/crypto"); + crypto._resetCryptoCacheForTests(); + const prints = keyFingerprints(process.env); + expect(prints.encryption.activeKeyId).toBe("v2"); + expect(prints.encryption.keys).toEqual([ + { id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }, + { id: "v2", fingerprint: crypto.getKeyFingerprint("v2") }, + ]); + + vi.stubEnv("ENCRYPTION_KEYS", ""); + vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", ""); + const base64 = Buffer.from(HEX_C, "hex").toString("base64"); + vi.stubEnv("ENCRYPTION_KEY", base64); + crypto._resetCryptoCacheForTests(); + const single = keyFingerprints(process.env); + expect(single.encryption).toEqual({ + activeKeyId: "v1", + keys: [{ id: "v1", fingerprint: crypto.getKeyFingerprint("v1") }], + }); + crypto._resetCryptoCacheForTests(); + }); + + it("names the off-host key the way an off-host object records it", async () => { + const { offhostKeyId } = await import("@/lib/jobs/offhost-backup"); + const prints = keyFingerprints({ + ENCRYPTION_KEY: HEX_A, + BACKUP_ENCRYPTION_KEY: HEX_D, + BACKUP_ENCRYPTION_PREVIOUS_KEYS: ` ${HEX_C} ,`, + API_TOKEN_HMAC_KEY: "hmac-secret-of-some-length-0123456789", + }); + expect(prints.backupEncryptionKey).toBe( + offhostKeyId(Buffer.from(HEX_D, "hex")), + ); + expect(prints.backupEncryptionPreviousKeys).toEqual([ + offhostKeyId(Buffer.from(HEX_C, "hex")), + ]); + expect(prints.apiTokenHmacKey).toMatch(/^[0-9a-f]{12}$/); + }); + + it("never carries key material", () => { + const env = { + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A }), + BACKUP_ENCRYPTION_KEY: HEX_B, + API_TOKEN_HMAC_KEY: HEX_C, + }; + const text = JSON.stringify(keyFingerprints(env)); + for (const secret of [HEX_A, HEX_B, HEX_C]) { + expect(text).not.toContain(secret.slice(0, 16)); + } + }); + + it("decodes keys the way the app does, and nothing that is not a key", () => { + expect(decodeKeyMaterial(HEX_A)?.length).toBe(32); + expect( + decodeKeyMaterial(Buffer.alloc(32, 7).toString("base64"))?.length, + ).toBe(32); + expect(decodeKeyMaterial("")).toBeNull(); + expect(decodeKeyMaterial("not a key!")).toBeNull(); + }); +}); + +describe("comparing a dump's keys with the environment's", () => { + const written = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_B }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + + it("finds nothing when every key is there", () => { + const same = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B, v1: HEX_A }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + expect(compareKeyFingerprints(written, same)).toEqual({ + findings: [], + refuse: false, + }); + }); + + it("refuses when the active key is different, and says which", () => { + const wrong = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v1: HEX_A, v2: HEX_D }), + ENCRYPTION_ACTIVE_KEY_ID: "v2", + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-one", + }); + const result = compareKeyFingerprints(written, wrong); + expect(result.refuse).toBe(true); + expect(result.findings).toHaveLength(1); + expect(result.findings[0]).toMatchObject({ + key: "ENCRYPTION_KEYS[v2]", + state: "different", + severity: "refuse", + }); + }); + + it("refuses when the active key is missing, warns for a retired one", () => { + const onlyOld = keyFingerprints({ ENCRYPTION_KEY: HEX_A }); + const result = compareKeyFingerprints(written, onlyOld); + expect(result.refuse).toBe(true); + const byKey = Object.fromEntries(result.findings.map((f) => [f.key, f])); + expect(byKey["ENCRYPTION_KEYS[v2]"]).toMatchObject({ + state: "missing", + severity: "refuse", + }); + expect(byKey["ENCRYPTION_KEYS[v1]"]).toBeUndefined(); + expect(byKey.BACKUP_ENCRYPTION_KEY).toMatchObject({ + state: "missing", + severity: "warn", + }); + expect(byKey.API_TOKEN_HMAC_KEY).toMatchObject({ + state: "missing", + severity: "warn", + }); + + const noRetired = keyFingerprints({ + ENCRYPTION_KEYS: JSON.stringify({ v2: HEX_B }), + BACKUP_ENCRYPTION_KEY: HEX_C, + API_TOKEN_HMAC_KEY: "hmac-two", + }); + const second = compareKeyFingerprints(written, noRetired); + expect(second.refuse).toBe(false); + expect(second.findings.map((f) => [f.key, f.severity])).toEqual([ + ["ENCRYPTION_KEYS[v1]", "warn"], + ["API_TOKEN_HMAC_KEY", "warn"], + ]); + }); +}); + +describe("the manifest", () => { + const manifest = buildManifest({ + kind: "daily", + file: "healthlog-20261025T013000Z-daily.dump", + createdAt: new Date("2026-10-25T01:30:00Z"), + appVersion: "1.46.0", + buildSha: "abc1234", + latestMigration: "0486_database_backup_runs", + server: { version: "18.6", versionNum: 180006 }, + pgDump: "pg_dump (PostgreSQL) 18.6", + archive: { bytes: 1234, sha256: "f".repeat(64), entries: 900 }, + rowCounts: { "public.users": 3, "pgboss.job": 12, "public.a": 0 }, + env: { ENCRYPTION_KEY: HEX_A, API_TOKEN_HMAC_KEY: "h" }, + }); + + it("carries what a restore and an operator need", () => { + expect(manifest).toMatchObject({ + format: "healthlog-database-dump", + formatVersion: 1, + kind: "daily", + file: "healthlog-20261025T013000Z-daily.dump", + createdAt: "2026-10-25T01:30:00.000Z", + appVersion: "1.46.0", + buildSha: "abc1234", + latestMigration: "0486_database_backup_runs", + postgres: { serverVersion: "18.6", serverVersionNum: 180006 }, + dump: { bytes: 1234, sha256: "f".repeat(64), entries: 900 }, + }); + expect(Object.keys(manifest.rowCounts)).toEqual([ + "pgboss.job", + "public.a", + "public.users", + ]); + expect(manifest.keys.encryption.keys[0].id).toBe("v1"); + }); + + it("reads back, and refuses anything else", () => { + expect(parseManifest(JSON.parse(JSON.stringify(manifest)))).toEqual( + manifest, + ); + expect(() => parseManifest({ format: "something-else" })).toThrow( + /not a HealthLog/, + ); + expect(() => parseManifest({ ...manifest, formatVersion: 2 })).toThrow(); + expect(() => parseManifest(null)).toThrow(); + }); +}); + +describe("row counts after a restore", () => { + it("names every table that is missing, extra or different", () => { + expect( + compareRowCounts( + { "public.a": 1, "public.b": 2 }, + { "public.a": 1, "public.b": 2 }, + ), + ).toEqual([]); + expect( + compareRowCounts( + { "public.a": 1, "public.b": 2, "public.c": 5 }, + { "public.a": 1, "public.b": 0, "public.d": 1 }, + ), + ).toEqual([ + "public.b: 2 rows in the dump, 0 restored", + "public.c: in the dump with 5 rows, missing now", + "public.d: restored but not in the dump's manifest", + ]); + }); +}); + +describe("the connection the client tools get", () => { + it("drops Prisma's parameters and keeps the password out of the arguments", () => { + const c = connectionFromUrl( + "postgresql://health%40log:p%40ss%2Fw@db:6543/hl?schema=app&connection_limit=20&pool_timeout=20&sslmode=require", + ); + expect(c.pgEnv).toEqual({ + PGHOST: "db", + PGPORT: "6543", + PGUSER: "health@log", + PGPASSWORD: "p@ss/w", + PGDATABASE: "hl", + PGSSLMODE: "require", + }); + expect(c.schema).toBe("app"); + const cs = String(c.clientConfig.connectionString); + expect(cs).not.toContain("connection_limit"); + expect(cs).not.toContain("schema="); + expect(cs).toContain("sslmode=require"); + }); +}); + +describe("an off-host copy read back by the restore", () => { + /** The uploader the job uses, into memory. */ + async function upload(objectKey: string, key: Buffer, body: Buffer) { + const { uploadEncryptedBackup } = await import("@/lib/jobs/offhost-backup"); + const stored = new Map(); + const s3: S3Like = { + putStream: async (k: string, stream: Readable) => { + stored.set(k, await collect(stream)); + }, + putObject: async () => {}, + getObject: async () => Buffer.alloc(0), + headObject: async () => false, + listObjects: async () => [], + deleteObject: async () => {}, + }; + await uploadEncryptedBackup(s3, objectKey, key, async (write) => { + // Several pieces, as the job streams the file. + for (let i = 0; i < body.length; i += 1000) { + await write(body.subarray(i, i + 1000)); + } + }); + return stored.get(objectKey) as Buffer; + } + + it("opens what the job's uploader sealed, byte for byte", async () => { + const dir = tempDir(); + const key = Buffer.from(HEX_D, "hex"); + const body = Buffer.from(Array.from({ length: 50_000 }, (_, i) => i % 251)); + const name = "healthlog-20261025T013000Z-daily.dump.enc"; + const sealed = await upload(offhostObjectKey(name), key, body); + writeFileSync(join(dir, name), sealed); + + await decryptOffhostFile({ + inPath: join(dir, name), + outPath: join(dir, "opened.dump"), + objectKey: offhostObjectKey(name), + keys: [Buffer.from(HEX_A, "hex"), key], + }); + expect( + (await collect(createReadStream(join(dir, "opened.dump")))).equals(body), + ).toBe(true); + }); + + it("refuses a copy renamed, altered or under a key it does not hold, and leaves nothing", async () => { + const dir = tempDir(); + const key = Buffer.from(HEX_D, "hex"); + const body = Buffer.from("PGDMP fake archive bytes".repeat(100)); + const name = "healthlog-20261025T013000Z-daily.dump.enc"; + const sealed = await upload(offhostObjectKey(name), key, body); + writeFileSync(join(dir, name), sealed); + const out = join(dir, "opened.dump"); + + // Renamed: the object key is bound as associated data. + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey( + "healthlog-20261024T013000Z-daily.dump.enc", + ), + keys: [key], + }), + ).rejects.toThrow(/does not open/); + expect(await readdir(dir)).toEqual([name]); + + // Altered. + const altered = Buffer.from(sealed); + altered[altered.length - 40] ^= 1; + writeFileSync(join(dir, name), altered); + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey(name), + keys: [key], + }), + ).rejects.toThrow(); + expect(await readdir(dir)).toEqual([name]); + + // A key it was not written under. + writeFileSync(join(dir, name), sealed); + await expect( + decryptOffhostFile({ + inPath: join(dir, name), + outPath: out, + objectKey: offhostObjectKey(name), + keys: [Buffer.from(HEX_A, "hex")], + }), + ).rejects.toThrow(/neither BACKUP_ENCRYPTION_KEY nor/); + expect(await readdir(dir)).toEqual([name]); + }); +}); diff --git a/src/lib/database-backup/dump-core.mjs b/src/lib/database-backup/dump-core.mjs new file mode 100644 index 000000000..14f13d2ca --- /dev/null +++ b/src/lib/database-backup/dump-core.mjs @@ -0,0 +1,1183 @@ +/** + * The whole-database dump: naming, writing, checking, keeping and reading it + * back. One module for three callers that cannot share TypeScript: + * + * - the nightly job in the app (`src/lib/database-backup/run.ts`), bundled; + * - the pre-migration dump the entrypoint takes (`scripts/db-dump.mjs`); + * - the restore (`scripts/db-restore.mjs`), which runs in the image with + * plain `node` on a server that has nothing but the dump and the keys. + * + * It is plain ESM with no imports beyond Node and `pg`, for the same reason + * `src/lib/auth/argon2-params.mjs` is: the production image strips `tsx`, and + * a restore that needs a source checkout is not a restore. Every rule that + * decides whether a dump is good, which files may be deleted and whether the + * keys fit lives here once, so the writer and the reader cannot drift. + * + * What a dump is: `pg_dump --format=custom` of the whole database (every + * schema, pg-boss included), named `healthlog--.dump`, + * with `healthlog--.manifest.json` beside it. A dump is + * good only when both exist under their final names: the archive is written + * as `.partial`, read back with `pg_restore --list`, the manifest is written + * and renamed first, and the archive is renamed last. A run that dies at any + * point leaves nothing that looks good. + * + * The row counts in the manifest come from the same snapshot `pg_dump` reads + * (`pg_export_snapshot()` + `--snapshot`), so a dump taken while the app is + * writing still restores to exactly the counts it claims. + */ +import { spawn } from "node:child_process"; +import { createDecipheriv, createHash } from "node:crypto"; +import { + accessSync, + closeSync, + constants as fsConstants, + createReadStream, + createWriteStream, + mkdirSync, + openSync, + readSync, + readdirSync, + statSync, +} from "node:fs"; +import { readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; + +/** Where dumps are written when `DB_BACKUP_DIR` is unset. */ +export const DEFAULT_BACKUP_DIR = "/var/lib/healthlog/backups"; + +/** Daily dumps kept when `DB_BACKUP_KEEP` is unset. */ +export const DEFAULT_DAILY_KEEP = 7; + +/** Pre-migration dumps kept: the one before this update and the one before. */ +export const PRE_MIGRATION_KEEP = 2; + +/** The cron the nightly dump runs on when `DB_BACKUP_SCHEDULE` is unset. */ +export const DEFAULT_BACKUP_SCHEDULE = "30 1 * * *"; + +/** The kinds of dump this module writes. */ +export const DUMP_KINDS = /** @type {const} */ (["daily", "pre-migration"]); + +/** The manifest's own format marker. */ +export const MANIFEST_FORMAT = "healthlog-database-dump"; +export const MANIFEST_FORMAT_VERSION = 1; + +/** The bucket prefix off-host database dumps live under. */ +export const OFFHOST_DATABASE_PREFIX = "database/"; + +/** + * A leftover `.partial` or an orphaned manifest younger than this may belong + * to a run that is still going (another container, the entrypoint), so it is + * left alone. + */ +const STALE_LEFTOVER_MS = 12 * 60 * 60 * 1000; + +const NAME_PATTERN = + /^healthlog-(\d{8}T\d{6}Z)-(daily|pre-migration)\.(dump|manifest\.json)(\.partial|\.enc)?$/; + +/** + * @typedef {"daily" | "pre-migration"} DumpKind + * @typedef {{ + * base: string; + * kind: DumpKind; + * timestamp: Date; + * role: "dump" | "manifest"; + * variant: "final" | "partial" | "encrypted"; + * }} ParsedDumpName + */ + +/** `20261025T013000Z` for an instant, in UTC. */ +export function formatDumpTimestamp(date) { + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[-:]/g, ""); +} + +/** + * The shared stem of a dump and its manifest. + * + * @param {DumpKind} kind + * @param {Date} date + */ +export function dumpBaseName(kind, date) { + return `healthlog-${formatDumpTimestamp(date)}-${kind}`; +} + +/** @param {string} base */ +export function dumpFileName(base) { + return `${base}.dump`; +} + +/** @param {string} base */ +export function manifestFileName(base) { + return `${base}.manifest.json`; +} + +/** + * Parse a file name this module writes. Anything else answers null, which is + * what keeps retention away from files it did not write. + * + * @param {string} name + * @returns {ParsedDumpName | null} + */ +export function parseDumpFileName(name) { + const m = NAME_PATTERN.exec(name); + if (!m) return null; + const [, stamp, kind, role, suffix] = m; + const iso = `${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)}T${stamp.slice(9, 11)}:${stamp.slice(11, 13)}:${stamp.slice(13, 15)}Z`; + const timestamp = new Date(iso); + if (Number.isNaN(timestamp.getTime())) return null; + return { + base: `healthlog-${stamp}-${kind}`, + kind: /** @type {DumpKind} */ (kind), + timestamp, + role: role === "dump" ? "dump" : "manifest", + variant: + suffix === ".partial" + ? "partial" + : suffix === ".enc" + ? "encrypted" + : "final", + }; +} + +// ── Configuration ────────────────────────────────────────────────────────── + +/** @typedef {Record} Env */ + +/** @param {string | undefined} raw */ +function isOff(raw) { + const v = (raw ?? "").trim().toLowerCase(); + return v === "false" || v === "0" || v === "no" || v === "off"; +} + +/** `DB_BACKUP_ENABLED`, on unless it says off. @param {Env} env */ +export function databaseBackupEnabled(env = process.env) { + return !isOff(env.DB_BACKUP_ENABLED); +} + +/** `DB_BACKUP_BEFORE_MIGRATE`, on unless it says off. @param {Env} env */ +export function preMigrationDumpEnabled(env = process.env) { + return !isOff(env.DB_BACKUP_BEFORE_MIGRATE); +} + +/** `DB_BACKUP_DIR`, or the image's default. @param {Env} env */ +export function resolveBackupDir(env = process.env) { + return (env.DB_BACKUP_DIR ?? "").trim() || DEFAULT_BACKUP_DIR; +} + +/** + * `DB_BACKUP_KEEP`: how many daily dumps stay on disk. At least one, so the + * newest good dump is never deleted; a value that is not a whole number + * keeps the default. + * + * @param {Env} env + */ +export function resolveDailyKeep(env = process.env) { + const raw = (env.DB_BACKUP_KEEP ?? "").trim(); + if (!/^\d+$/.test(raw)) return DEFAULT_DAILY_KEEP; + return Math.min(Math.max(Number.parseInt(raw, 10), 1), 366); +} + +const CRON_FIELD = /^[0-9*,/-]+$/; + +/** + * `DB_BACKUP_SCHEDULE` when it is a five-field cron of digits, `*`, `,`, `/` + * and `-`, otherwise the default. `valid` says which, so the boot summary can + * name a schedule it ignored. + * + * @param {Env} env + * @returns {{ cron: string; valid: boolean; configured: boolean }} + */ +export function resolveBackupSchedule(env = process.env) { + const raw = (env.DB_BACKUP_SCHEDULE ?? "").trim(); + if (!raw) { + return { cron: DEFAULT_BACKUP_SCHEDULE, valid: true, configured: false }; + } + const fields = raw.split(/\s+/); + const valid = fields.length === 5 && fields.every((f) => CRON_FIELD.test(f)); + return { + cron: valid ? fields.join(" ") : DEFAULT_BACKUP_SCHEDULE, + valid, + configured: true, + }; +} + +/** + * Whether `dir` exists (or can be created) and this process may write in it. + * Never throws. + * + * @param {string} dir + */ +export function isWritableDirectory(dir) { + try { + mkdirSync(dir, { recursive: true }); + if (!statSync(dir).isDirectory()) return false; + accessSync(dir, fsConstants.W_OK | fsConstants.X_OK); + return true; + } catch { + return false; + } +} + +// ── Keys ─────────────────────────────────────────────────────────────────── + +/** + * Decode a 32-byte key the way the app does (`decodeKey` in + * `src/lib/crypto.ts`): 64 hex characters, or base64 of 32 bytes. Shorter hex + * is padded with its own SHA-256 there, outside production only; it is + * decoded the same way here so a development instance's fingerprints match + * what its admin page shows. Null when the value is not a key at all. + * + * @param {string | undefined} raw + * @returns {Buffer | null} + */ +export function decodeKeyMaterial(raw) { + const value = (raw ?? "").trim(); + if (!value) return null; + if (/^[0-9a-fA-F]{64}$/.test(value)) return Buffer.from(value, "hex"); + if (/^[A-Za-z0-9+/=]+$/.test(value)) { + const buf = Buffer.from(value, "base64"); + if (buf.length === 32) return buf; + } + if (/^[0-9a-fA-F]+$/.test(value) && value.length >= 32) { + const padded = + value + createHash("sha256").update(value, "utf8").digest("hex"); + return Buffer.from(padded.slice(0, 64), "hex"); + } + return null; +} + +/** + * The public fingerprint of a key: the first twelve hex characters of + * SHA-256 over its bytes. The same rule as `fingerprintKeyBytes` in + * `src/lib/crypto.ts` (what the admin's key-backup step shows) and as the key + * id an off-host object records, so one number names one key everywhere. + * + * @param {Buffer} bytes + */ +export function fingerprintBytes(bytes) { + return createHash("sha256").update(bytes).digest("hex").slice(0, 12); +} + +/** + * @typedef {{ + * encryption: { + * activeKeyId: string | null; + * keys: Array<{ id: string; fingerprint: string }>; + * }; + * backupEncryptionKey: string | null; + * backupEncryptionPreviousKeys: string[]; + * apiTokenHmacKey: string | null; + * }} KeyFingerprints + */ + +/** + * Fingerprints of every key a restored database needs, read from an + * environment. Never the key material. A malformed value is left out, which + * the comparison then reports as missing. + * + * `ENCRYPTION_KEY` joins the map as `v1` unless the map names `v1` itself, + * exactly as the app's key loader does. + * + * @param {Env} env + * @returns {KeyFingerprints} + */ +export function keyFingerprints(env = process.env) { + /** @type {Map} */ + const keys = new Map(); + let activeKeyId = null; + const rawMap = (env.ENCRYPTION_KEYS ?? "").trim(); + if (rawMap) { + try { + const parsed = JSON.parse(rawMap); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + for (const [id, value] of Object.entries(parsed)) { + const bytes = + typeof value === "string" ? decodeKeyMaterial(value) : null; + if (bytes) keys.set(id, fingerprintBytes(bytes)); + } + } + } catch { + // Malformed JSON: no keys from the map. The entrypoint refuses to boot + // on it anyway; here it only means nothing matches. + } + activeKeyId = (env.ENCRYPTION_ACTIVE_KEY_ID ?? "").trim() || null; + if (!activeKeyId && keys.size === 1) activeKeyId = [...keys.keys()][0]; + } + const legacy = decodeKeyMaterial(env.ENCRYPTION_KEY); + if (legacy && !keys.has("v1")) keys.set("v1", fingerprintBytes(legacy)); + if (!activeKeyId && legacy) activeKeyId = "v1"; + + const backup = decodeKeyMaterial(env.BACKUP_ENCRYPTION_KEY); + const previous = (env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "") + .split(",") + .map((raw) => decodeKeyMaterial(raw)) + .filter((b) => b !== null) + .map((b) => fingerprintBytes(/** @type {Buffer} */ (b))); + const hmac = (env.API_TOKEN_HMAC_KEY ?? "").trim(); + + return { + encryption: { + activeKeyId, + keys: [...keys.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([id, fingerprint]) => ({ id, fingerprint })), + }, + backupEncryptionKey: backup ? fingerprintBytes(backup) : null, + backupEncryptionPreviousKeys: previous, + // Used as a string by the app (`createHmac("sha256", key)`), so its + // fingerprint is over the string's bytes. + apiTokenHmacKey: hmac ? fingerprintBytes(Buffer.from(hmac, "utf8")) : null, + }; +} + +/** + * @typedef {{ + * key: string; + * state: "missing" | "different"; + * severity: "refuse" | "warn"; + * detail: string; + * }} KeyFinding + */ + +/** + * Compare the keys a dump was written with against the keys an environment + * holds. The active encryption key is the one that refuses: without it the + * app's start-up key check fails and nothing encrypted reads. Every other + * difference is a warning that says what stops working. + * + * @param {KeyFingerprints} expected the manifest's + * @param {KeyFingerprints} actual the environment's + * @returns {{ findings: KeyFinding[]; refuse: boolean }} + */ +export function compareKeyFingerprints(expected, actual) { + /** @type {KeyFinding[]} */ + const findings = []; + const have = new Map( + actual.encryption.keys.map((k) => [k.id, k.fingerprint]), + ); + const active = expected.encryption.activeKeyId; + for (const { id, fingerprint } of expected.encryption.keys) { + const mine = have.get(id); + if (mine === fingerprint) continue; + const isActive = id === active; + findings.push({ + key: `ENCRYPTION_KEYS[${id}]`, + state: mine === undefined ? "missing" : "different", + severity: isActive ? "refuse" : "warn", + detail: isActive + ? `the dump's active encryption key ${id} (fingerprint ${fingerprint}) is ${mine === undefined ? "not configured" : `configured with a different value (fingerprint ${mine})`}; nothing encrypted in this database can be read without it` + : `encryption key ${id} (fingerprint ${fingerprint}) is ${mine === undefined ? "not configured" : `different here (fingerprint ${mine})`}; rows still sealed under it will not open`, + }); + } + if ( + expected.backupEncryptionKey && + expected.backupEncryptionKey !== actual.backupEncryptionKey + ) { + findings.push({ + key: "BACKUP_ENCRYPTION_KEY", + state: actual.backupEncryptionKey ? "different" : "missing", + severity: "warn", + detail: `the off-host backup key (fingerprint ${expected.backupEncryptionKey}) is ${actual.backupEncryptionKey ? `different here (fingerprint ${actual.backupEncryptionKey})` : "not configured"}; off-host copies written before the restore will not open`, + }); + } + if ( + expected.apiTokenHmacKey && + expected.apiTokenHmacKey !== actual.apiTokenHmacKey + ) { + findings.push({ + key: "API_TOKEN_HMAC_KEY", + state: actual.apiTokenHmacKey ? "different" : "missing", + severity: "warn", + detail: `API_TOKEN_HMAC_KEY (fingerprint ${expected.apiTokenHmacKey}) is ${actual.apiTokenHmacKey ? `different here (fingerprint ${actual.apiTokenHmacKey})` : "not configured"}; every session, app token and share link will stop working, no data is lost`, + }); + } + return { + findings, + refuse: findings.some((f) => f.severity === "refuse"), + }; +} + +// ── Connection ───────────────────────────────────────────────────────────── + +/** + * Query parameters libpq understands, by the environment variable that + * carries each to `pg_dump` / `pg_restore`. `DATABASE_URL` also carries + * Prisma's own (`schema`, `connection_limit`, `pool_timeout`), which libpq + * rejects, so the URL is never handed over as it is. + */ +const LIBPQ_PARAMS = { + host: "PGHOST", + sslmode: "PGSSLMODE", + sslcert: "PGSSLCERT", + sslkey: "PGSSLKEY", + sslrootcert: "PGSSLROOTCERT", + connect_timeout: "PGCONNECT_TIMEOUT", + application_name: "PGAPPNAME", +}; + +/** + * @typedef {{ + * pgEnv: Record; + * clientConfig: Record; + * schema: string; + * }} Connection + */ + +/** + * The connection `DATABASE_URL` describes, as libpq environment variables + * (the password travels there, never in a process argument) and as a `pg` + * client config. + * + * @param {string} databaseUrl + * @returns {Connection} + */ +export function connectionFromUrl(databaseUrl) { + const url = new URL(databaseUrl); + const host = decodeURIComponent(url.hostname).replace(/^\[|\]$/g, ""); + /** @type {Record} */ + const pgEnv = { + PGHOST: host, + PGPORT: url.port || "5432", + PGUSER: decodeURIComponent(url.username), + PGPASSWORD: decodeURIComponent(url.password), + PGDATABASE: decodeURIComponent(url.pathname.replace(/^\//, "")), + }; + for (const [param, value] of url.searchParams) { + const name = /** @type {Record} */ (LIBPQ_PARAMS)[param]; + if (name) pgEnv[name] = value; + } + const clean = new URL(databaseUrl); + for (const param of [...clean.searchParams.keys()]) { + if (!(param in LIBPQ_PARAMS)) clean.searchParams.delete(param); + } + return { + pgEnv, + clientConfig: { + connectionString: clean.toString(), + connectionTimeoutMillis: 10_000, + }, + schema: url.searchParams.get("schema") || "public", + }; +} + +/** + * A connected `pg` client for `DATABASE_URL`. + * + * @param {string} databaseUrl + */ +export async function connect(databaseUrl) { + const { default: pg } = await import("pg"); + const client = new pg.Client(connectionFromUrl(databaseUrl).clientConfig); + await client.connect(); + // The server may carry a default statement timeout (the app sets one per + // session through `options`); counting a large table must not hit it. + await client.query("SET statement_timeout = 0"); + return client; +} + +// ── Facts about a database ───────────────────────────────────────────────── + +/** + * @typedef {{ query: (sql: string, params?: unknown[]) => Promise<{ rows: any[] }> }} Queryable + */ + +/** Tables outside the system schemas, `schema.table`, sorted. @param {Queryable} client */ +export async function listUserTables(client) { + const { rows } = await client.query( + `SELECT format('%I.%I', n.nspname, c.relname) AS name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg\\_%' + ORDER BY 1`, + ); + return rows.map((r) => /** @type {string} */ (r.name)); +} + +/** + * Exact row counts of every user table, keyed `schema.table`. + * + * @param {Queryable} client + * @returns {Promise>} + */ +export async function countRows(client) { + /** @type {Record} */ + const counts = {}; + for (const table of await listUserTables(client)) { + const { rows } = await client.query( + `SELECT count(*)::bigint AS n FROM ${table}`, + ); + counts[table] = Number(rows[0].n); + } + return counts; +} + +/** + * The newest finished migration, or null on a database Prisma never touched. + * + * @param {Queryable} client + * @param {string} schema + */ +export async function latestMigration(client, schema) { + const table = await client.query( + "SELECT to_regclass(format('%I._prisma_migrations', $1::text)) AS t", + [schema], + ); + if (!table.rows[0]?.t) return null; + const { rows } = await client.query( + `SELECT migration_name FROM ${table.rows[0].t} + WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL + ORDER BY migration_name DESC LIMIT 1`, + ); + return rows[0]?.migration_name ?? null; +} + +/** @param {Queryable} client */ +export async function serverVersion(client) { + const { rows } = await client.query( + "SELECT current_setting('server_version') AS v, current_setting('server_version_num')::int AS n", + ); + return { version: String(rows[0].v), versionNum: Number(rows[0].n) }; +} + +// ── Running the Postgres client tools ────────────────────────────────────── + +/** + * Run one Postgres client tool. Standard output goes to `stdout` when given + * (a file stream) and is collected otherwise; standard input comes from + * `stdin` when given. Rejects with the tool's own last lines on a non-zero + * exit, and kills it at `deadline` (epoch ms). + * + * @param {string} tool + * @param {string[]} args + * @param {{ + * env?: Record; + * stdout?: NodeJS.WritableStream; + * stdin?: NodeJS.ReadableStream; + * deadline?: number; + * }} [options] + * @returns {Promise} + */ +export function runTool(tool, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(tool, args, { + env: { ...process.env, ...(options.env ?? {}) }, + stdio: [options.stdin ? "pipe" : "ignore", "pipe", "pipe"], + }); + /** @type {Buffer[]} */ + const out = []; + let err = ""; + let timedOut = false; + const timer = + options.deadline !== undefined + ? setTimeout( + () => { + timedOut = true; + child.kill("SIGTERM"); + }, + Math.max(0, options.deadline - Date.now()), + ) + : null; + if (options.stdout) { + // A destination that fails (a full disk) must stop the tool: unpiped, + // it would block on a full pipe until its deadline, or forever. + options.stdout.on("error", (e) => { + child.kill("SIGTERM"); + reject(e); + }); + child.stdout.pipe(options.stdout); + } else { + child.stdout.on("data", (c) => out.push(c)); + } + child.stderr.on("data", (c) => { + err = (err + c.toString("utf8")).slice(-4000); + }); + if (options.stdin && child.stdin) { + options.stdin.on("error", (e) => { + child.kill("SIGTERM"); + reject(e); + }); + child.stdin.on("error", () => { + // The tool closed its input early; its exit code says why. + }); + options.stdin.pipe(child.stdin); + } + child.on("error", (e) => { + if (timer) clearTimeout(timer); + reject( + /** @type {NodeJS.ErrnoException} */ (e).code === "ENOENT" + ? new Error( + `${tool} is not installed in this environment (the HealthLog image ships it; on another host install the PostgreSQL client tools)`, + ) + : e, + ); + }); + child.on("close", (code, signal) => { + if (timer) clearTimeout(timer); + if (timedOut) { + reject(new Error(`${tool} ran past its time limit and was stopped`)); + } else if (code === 0) { + resolve(Buffer.concat(out).toString("utf8")); + } else { + const tail = err.trim().split("\n").slice(-3).join(" | "); + reject( + new Error( + `${tool} exited ${code ?? signal}${tail ? `: ${tail}` : ""}`, + ), + ); + } + }); + }); +} + +/** `pg_dump (PostgreSQL) 18.6`, or null when it cannot be asked. */ +export async function pgDumpVersion() { + try { + return (await runTool("pg_dump", ["--version"])).trim(); + } catch { + return null; + } +} + +/** + * Read an archive's table of contents back. A file `pg_restore` cannot list + * is not a dump, whatever its name. Answers the number of entries. + * + * @param {string} path + */ +export async function verifyArchive(path) { + const listing = await runTool("pg_restore", ["--list"], { + stdin: createReadStream(path), + }); + const entries = listing + .split("\n") + .filter((line) => line.trim() && !line.startsWith(";")).length; + if (entries === 0) throw new Error("the archive lists no entries"); + return entries; +} + +/** SHA-256 and size of a file, streamed. @param {string} path */ +export async function hashFile(path) { + const hash = createHash("sha256"); + let bytes = 0; + for await (const chunk of createReadStream(path)) { + hash.update(chunk); + bytes += chunk.length; + } + return { sha256: hash.digest("hex"), bytes }; +} + +// ── Writing a dump ───────────────────────────────────────────────────────── + +/** + * @typedef {{ + * format: string; + * formatVersion: number; + * kind: DumpKind; + * file: string; + * createdAt: string; + * appVersion: string | null; + * buildSha: string | null; + * latestMigration: string | null; + * postgres: { serverVersion: string; serverVersionNum: number; pgDump: string | null }; + * dump: { bytes: number; sha256: string; entries: number }; + * rowCounts: Record; + * keys: KeyFingerprints; + * }} DumpManifest + */ + +/** + * Build the manifest of one dump. Pure, so the shape is tested on its own. + * + * @param {{ + * kind: DumpKind; + * file: string; + * createdAt: Date; + * appVersion?: string | null; + * buildSha?: string | null; + * latestMigration: string | null; + * server: { version: string; versionNum: number }; + * pgDump: string | null; + * archive: { bytes: number; sha256: string; entries: number }; + * rowCounts: Record; + * env: Env; + * }} input + * @returns {DumpManifest} + */ +export function buildManifest(input) { + return { + format: MANIFEST_FORMAT, + formatVersion: MANIFEST_FORMAT_VERSION, + kind: input.kind, + file: input.file, + createdAt: input.createdAt.toISOString(), + appVersion: input.appVersion || null, + buildSha: input.buildSha || null, + latestMigration: input.latestMigration, + postgres: { + serverVersion: input.server.version, + serverVersionNum: input.server.versionNum, + pgDump: input.pgDump, + }, + dump: { ...input.archive }, + rowCounts: Object.fromEntries( + Object.entries(input.rowCounts).sort(([a], [b]) => a.localeCompare(b)), + ), + keys: keyFingerprints(input.env), + }; +} + +/** + * Check a parsed manifest has the fields a restore reads. + * + * @param {unknown} value + * @returns {DumpManifest} + */ +export function parseManifest(value) { + const m = /** @type {Record} */ (value); + if ( + !m || + m.format !== MANIFEST_FORMAT || + m.formatVersion !== MANIFEST_FORMAT_VERSION || + typeof m.file !== "string" || + typeof m.dump?.sha256 !== "string" || + typeof m.dump?.bytes !== "number" || + typeof m.rowCounts !== "object" || + !m.keys?.encryption + ) { + throw new Error("not a HealthLog database dump manifest"); + } + return /** @type {DumpManifest} */ (m); +} + +/** + * Take one dump into `dir`, with its manifest, and answer both paths. + * + * The archive and the row counts read one exported snapshot. The archive is + * written as `.partial`, hashed and listed back; the manifest is written as + * `.partial` and renamed; the archive is renamed last. On any failure every + * file this call created is removed and the error is rethrown. + * + * @param {{ + * databaseUrl: string; + * dir: string; + * kind: DumpKind; + * now?: Date; + * env?: Env; + * appVersion?: string | null; + * buildSha?: string | null; + * deadline?: number; + * }} options + * @returns {Promise<{ base: string; dumpPath: string; manifestPath: string; manifest: DumpManifest }>} + */ +export async function createDatabaseDump(options) { + const now = options.now ?? new Date(); + const env = options.env ?? process.env; + const base = dumpBaseName(options.kind, now); + const dumpPath = join(options.dir, dumpFileName(base)); + const manifestPath = join(options.dir, manifestFileName(base)); + const partialDump = `${dumpPath}.partial`; + const partialManifest = `${manifestPath}.partial`; + const connection = connectionFromUrl(options.databaseUrl); + // Two runs in the same second would share a name. The second one must not + // replace, or on failure remove, what the first one wrote. + for (const taken of [dumpPath, manifestPath]) { + if (await statOrNull(taken)) { + throw new Error(`a dump named ${basename(taken)} already exists`); + } + } + + const client = await connect(options.databaseUrl); + let finished = false; + /** The files this call created, and only those, go on a failure. */ + const created = []; + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY"); + const { rows } = await client.query( + "SELECT pg_export_snapshot() AS snapshot", + ); + const snapshot = String(rows[0].snapshot); + + const out = createWriteStream(partialDump, { flags: "wx", mode: 0o600 }); + const opened = new Promise((resolve, reject) => { + out.once("open", resolve); + out.once("error", reject); + }); + await opened; + created.push(partialDump); + const closed = new Promise((resolve, reject) => { + out.once("close", resolve); + out.once("error", reject); + }); + // Awaited below on success; on a failure the run is already rejecting + // and this must not surface a second time as unhandled. + closed.catch(() => {}); + // `pg_dump` reads the snapshot in its own session while this one counts + // the rows of the same snapshot; the queries on this client run one + // after another (a `pg` client does not overlap them). + const facts = async () => ({ + counts: await countRows(client), + server: await serverVersion(client), + migration: await latestMigration(client, connection.schema), + }); + const [, { counts, server, migration }, pgDump] = await Promise.all([ + runTool( + "pg_dump", + [ + "--format=custom", + "--no-password", + `--snapshot=${snapshot}`, + "--lock-wait-timeout=120000", + ], + { env: connection.pgEnv, stdout: out, deadline: options.deadline }, + ), + facts(), + pgDumpVersion(), + ]); + await closed; + await client.query("COMMIT"); + + const { sha256, bytes } = await hashFile(partialDump); + const entries = await verifyArchive(partialDump); + const manifest = buildManifest({ + kind: options.kind, + file: dumpFileName(base), + createdAt: now, + appVersion: options.appVersion, + buildSha: options.buildSha, + latestMigration: migration, + server, + pgDump, + archive: { bytes, sha256, entries }, + rowCounts: counts, + env, + }); + await writeFile(partialManifest, `${JSON.stringify(manifest, null, 2)}\n`, { + flag: "wx", + mode: 0o600, + }); + created.push(partialManifest); + await rename(partialManifest, manifestPath); + created.push(manifestPath); + await rename(partialDump, dumpPath); + finished = true; + return { base, dumpPath, manifestPath, manifest }; + } finally { + await client.end().catch(() => {}); + if (!finished) { + // Only what this call created: a `wx` open that failed created nothing. + await Promise.all(created.map((p) => rm(p, { force: true }))); + } + } +} + +// ── What is on disk, and what may go ─────────────────────────────────────── + +/** + * @typedef {{ + * base: string; + * kind: DumpKind; + * timestamp: Date; + * dump: { path: string; bytes: number } | null; + * manifest: string | null; + * encryptedDump: string | null; + * encryptedManifest: string | null; + * leftovers: Array<{ path: string; mtimeMs: number }>; + * }} DumpSet + */ + +/** + * Every dump this module wrote into `dir`, newest first, grouped by stem. + * Files with any other name are not looked at. + * + * @param {string} dir + * @returns {DumpSet[]} + */ +export function listDumpSets(dir) { + /** @type {Map} */ + const sets = new Map(); + let names = []; + try { + names = readdirSync(dir); + } catch { + return []; + } + for (const name of names) { + const parsed = parseDumpFileName(name); + if (!parsed) continue; + const path = join(dir, name); + let info; + try { + info = statSync(path); + } catch { + continue; + } + if (!info.isFile()) continue; + const set = sets.get(parsed.base) ?? { + base: parsed.base, + kind: parsed.kind, + timestamp: parsed.timestamp, + dump: null, + manifest: null, + encryptedDump: null, + encryptedManifest: null, + leftovers: [], + }; + if (parsed.variant === "partial") { + set.leftovers.push({ path, mtimeMs: info.mtimeMs }); + } else if (parsed.variant === "encrypted") { + if (parsed.role === "dump") set.encryptedDump = path; + else set.encryptedManifest = path; + } else if (parsed.role === "dump") { + set.dump = { path, bytes: info.size }; + } else { + set.manifest = path; + } + sets.set(parsed.base, set); + } + return [...sets.values()].sort( + (a, b) => b.timestamp.getTime() - a.timestamp.getTime(), + ); +} + +/** A set is good when its archive and its manifest both carry final names. @param {DumpSet} set */ +export function isGoodSet(set) { + return set.dump !== null && set.manifest !== null; +} + +/** + * Which files retention removes. Pure. + * + * Per kind, the newest `keep` good dumps stay (at least one, so the newest + * good dump is never removed) and every older good dump goes, archive and + * manifest together. A `.partial` or a manifest without its archive goes once + * it is older than twelve hours, so a run still in progress keeps its files. + * A final-named archive without a manifest is not something this module + * leaves behind, so it is not touched. Encrypted copies (`.enc`) an operator + * placed here to restore from are never touched either. + * + * @param {DumpSet[]} sets + * @param {{ keep: Record; now: number }} options + * @returns {string[]} + */ +export function selectForDeletion(sets, options) { + /** @type {string[]} */ + const remove = []; + for (const kind of DUMP_KINDS) { + const keep = Math.max(1, options.keep[kind] ?? 1); + const good = sets + .filter((s) => s.kind === kind && isGoodSet(s)) + .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()); + for (const set of good.slice(keep)) { + remove.push( + /** @type {{ path: string }} */ (set.dump).path, + /** @type {string} */ (set.manifest), + ); + } + } + for (const set of sets) { + for (const leftover of set.leftovers) { + if (options.now - leftover.mtimeMs > STALE_LEFTOVER_MS) { + remove.push(leftover.path); + } + } + if (set.manifest && !set.dump && set.leftovers.length === 0) { + try { + if (options.now - statSync(set.manifest).mtimeMs > STALE_LEFTOVER_MS) { + remove.push(set.manifest); + } + } catch { + // Gone already. + } + } + } + return remove; +} + +/** + * Apply retention to `dir`. Answers the paths removed. + * + * @param {string} dir + * @param {{ dailyKeep?: number; now?: number }} [options] + */ +export async function applyRetention(dir, options = {}) { + const doomed = selectForDeletion(listDumpSets(dir), { + keep: { + daily: options.dailyKeep ?? DEFAULT_DAILY_KEEP, + "pre-migration": PRE_MIGRATION_KEEP, + }, + now: options.now ?? Date.now(), + }); + for (const path of doomed) await rm(path, { force: true }); + return doomed; +} + +/** @param {string} path */ +export async function readManifestFile(path) { + return parseManifest(JSON.parse(await readFile(path, "utf8"))); +} + +// ── Off-host copies ──────────────────────────────────────────────────────── + +/** + * The bucket key of an off-host database object. It is also the associated + * data its encryption is bound to, so a copy cannot be passed off under + * another name. + * + * @param {string} fileName e.g. `healthlog-…-daily.dump.enc` + */ +export function offhostObjectKey(fileName) { + return `${OFFHOST_DATABASE_PREFIX}${fileName}`; +} + +const ENVELOPE_MAGIC = "HLBK"; +const ENVELOPE_KEYED = 0x04; +const AAD_PREFIX = "healthlog/offhost-backup/v4|"; +const IV_LENGTH = 12; +const TAG_LENGTH = 16; + +/** + * Open an off-host object written in the version-4 envelope + * (`src/lib/jobs/offhost-backup.ts`: `HLBK`, 0x04, key id, IV, gzip + * ciphertext, tag, the object key as associated data) into `outPath`, + * streaming, without holding the file. + * + * GCM's tag sits at the end, so the plaintext is written before the tag is + * checked; it goes to `outPath` and is removed again unless the tag verifies. + * Nothing reads `outPath` before this resolves. + * + * @param {{ inPath: string; outPath: string; objectKey: string; keys: Buffer[] }} options + */ +export async function decryptOffhostFile(options) { + const size = statSync(options.inPath).size; + const fd = openSync(options.inPath, "r"); + let header; + let tag; + try { + const head = Buffer.alloc(Math.min(size, 6 + 255 + IV_LENGTH)); + readSync(fd, head, 0, head.length, 0); + if ( + head.subarray(0, 4).toString("latin1") !== ENVELOPE_MAGIC || + head[4] !== ENVELOPE_KEYED + ) { + throw new Error( + "not an off-host database copy (unknown envelope); expected a .dump.enc written by HealthLog", + ); + } + const idLen = head[5]; + const keyId = head.subarray(6, 6 + idLen).toString("latin1"); + const iv = head.subarray(6 + idLen, 6 + idLen + IV_LENGTH); + header = { keyId, iv, length: 6 + idLen + IV_LENGTH }; + tag = Buffer.alloc(TAG_LENGTH); + readSync(fd, tag, 0, TAG_LENGTH, size - TAG_LENGTH); + } finally { + closeSync(fd); + } + if (size < header.length + TAG_LENGTH) { + throw new Error("the off-host copy is truncated"); + } + const key = options.keys.find((k) => fingerprintBytes(k) === header.keyId); + if (!key) { + throw new Error( + `the off-host copy was written under backup key ${header.keyId}, which is neither BACKUP_ENCRYPTION_KEY nor in BACKUP_ENCRYPTION_PREVIOUS_KEYS`, + ); + } + const decipher = createDecipheriv("aes-256-gcm", key, header.iv); + decipher.setAAD(Buffer.from(`${AAD_PREFIX}${options.objectKey}`, "utf8")); + decipher.setAuthTag(tag); + try { + await pipeline( + createReadStream(options.inPath, { + start: header.length, + end: size - TAG_LENGTH - 1, + }), + decipher, + createGunzip(), + createWriteStream(options.outPath, { flags: "wx", mode: 0o600 }), + ); + } catch (err) { + await rm(options.outPath, { force: true }); + const reason = err instanceof Error ? err.message : String(err); + throw new Error( + /auth|unable to authenticate/i.test(reason) + ? "the off-host copy does not open with this key under this name (altered, renamed, or written with another key)" + : `could not open the off-host copy: ${reason}`, + ); + } +} + +/** + * The off-host keys an environment holds, current first. + * + * @param {Env} env + * @returns {Buffer[]} + */ +export function offhostKeysFromEnv(env = process.env) { + return [ + env.BACKUP_ENCRYPTION_KEY, + ...(env.BACKUP_ENCRYPTION_PREVIOUS_KEYS ?? "").split(","), + ] + .map((raw) => decodeKeyMaterial(raw)) + .filter((k) => k !== null) + .map((k) => /** @type {Buffer} */ (k)); +} + +// ── Restore ──────────────────────────────────────────────────────────────── + +/** + * Compare the counts a restore produced with the manifest's. Every table the + * manifest names must be there with its count, and no table may be there that + * the manifest does not name. + * + * @param {Record} expected + * @param {Record} actual + * @returns {string[]} one line per difference + */ +export function compareRowCounts(expected, actual) { + /** @type {string[]} */ + const problems = []; + for (const [table, count] of Object.entries(expected)) { + if (!(table in actual)) { + problems.push(`${table}: in the dump with ${count} rows, missing now`); + } else if (actual[table] !== count) { + problems.push( + `${table}: ${count} rows in the dump, ${actual[table]} restored`, + ); + } + } + for (const table of Object.keys(actual)) { + if (!(table in expected)) { + problems.push(`${table}: restored but not in the dump's manifest`); + } + } + return problems; +} + +/** + * Drop every non-system schema and recreate an empty `public`, so a restore + * that failed its check leaves the database as empty as it found it. + * + * @param {Queryable} client + */ +export async function emptyDatabase(client) { + const { rows } = await client.query( + `SELECT quote_ident(nspname) AS s FROM pg_namespace + WHERE nspname NOT LIKE 'pg\\_%' AND nspname <> 'information_schema'`, + ); + for (const { s } of rows) await client.query(`DROP SCHEMA ${s} CASCADE`); + await client.query("CREATE SCHEMA IF NOT EXISTS public"); +} + +/** Stat helper that answers null for a missing file. @param {string} path */ +export async function statOrNull(path) { + try { + return await stat(path); + } catch { + return null; + } +} diff --git a/src/lib/database-backup/health.ts b/src/lib/database-backup/health.ts new file mode 100644 index 000000000..2039df249 --- /dev/null +++ b/src/lib/database-backup/health.ts @@ -0,0 +1,117 @@ +/** + * Whether the nightly whole-database dump is happening, as facts a page or an + * alert can act on. + * + * Read from the ledger the job writes (`DatabaseBackupRun`), plus two facts of + * this process: whether `DB_BACKUP_DIR` is writable here, and whether the + * off-host bucket is configured. A ledger row that says a copy went off-host + * outranks this process's environment, for the reason the off-host card gives + * (a split install can hand the bucket variables to the worker only). + */ +import { prisma } from "@/lib/db"; +import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue"; +import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; + +import { + databaseBackupEnabled, + isWritableDirectory, + resolveBackupDir, +} from "./dump-core.mjs"; + +export interface DatabaseBackupHealth { + /** `DB_BACKUP_ENABLED` is not off. */ + enabled: boolean; + /** The directory dumps are kept in (`DB_BACKUP_DIR`). */ + directory: string; + /** This process can write there. False means dumps only go off-host. */ + localDirWritable: boolean; + /** The off-host bucket is configured, or a dump has been uploaded to it. */ + offhostConfigured: boolean; + /** The newest dump that was kept on the server or reached the bucket. */ + lastGood: { + at: string; + fileName: string; + sizeBytes: number | null; + local: boolean; + offhost: boolean; + } | null; + /** The newest run that failed, or that a dying process left unfinished. */ + lastFailure: { at: string; error: string } | null; +} + +/** + * A run still `running` this long after it started was cut off by its + * process: the job's expiry, past which pg-boss has failed it anyway. + */ +const ABANDONED_AFTER_MS = DATABASE_BACKUP_EXPIRE_SECONDS * 1000; + +export async function getDatabaseBackupHealth( + now: Date = new Date(), + env: NodeJS.ProcessEnv = process.env, +): Promise { + const directory = resolveBackupDir(env); + const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all( + [ + prisma.databaseBackupRun.findFirst({ + where: { + fileName: { not: null }, + OR: [{ localResult: "written" }, { offhostResult: "uploaded" }], + }, + orderBy: { startedAt: "desc" }, + select: { + startedAt: true, + finishedAt: true, + fileName: true, + dumpBytes: true, + localResult: true, + offhostResult: true, + }, + }), + prisma.databaseBackupRun.findFirst({ + where: { status: "failed" }, + orderBy: { startedAt: "desc" }, + select: { startedAt: true, finishedAt: true, error: true }, + }), + prisma.databaseBackupRun.findFirst({ + where: { + status: "running", + startedAt: { lt: new Date(now.getTime() - ABANDONED_AFTER_MS) }, + }, + orderBy: { startedAt: "desc" }, + select: { startedAt: true }, + }), + prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }), + ], + ); + + const failures = [ + lastFailed && { + at: (lastFailed.finishedAt ?? lastFailed.startedAt).toISOString(), + error: lastFailed.error ?? "failed without a reason", + }, + lastAbandoned && { + at: lastAbandoned.startedAt.toISOString(), + error: "the run did not finish (the process stopped under it)", + }, + ] + .filter((f): f is { at: string; error: string } => Boolean(f)) + .sort((a, b) => b.at.localeCompare(a.at)); + + return { + enabled: databaseBackupEnabled(env), + directory, + localDirWritable: isWritableDirectory(directory), + offhostConfigured: offhostBackupConfigured() || everUploaded > 0, + lastGood: lastGood + ? { + at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(), + fileName: lastGood.fileName ?? "", + sizeBytes: + lastGood.dumpBytes === null ? null : Number(lastGood.dumpBytes), + local: lastGood.localResult === "written", + offhost: lastGood.offhostResult === "uploaded", + } + : null, + lastFailure: failures[0] ?? null, + }; +} diff --git a/src/lib/database-backup/run.ts b/src/lib/database-backup/run.ts new file mode 100644 index 000000000..4f3e3d99e --- /dev/null +++ b/src/lib/database-backup/run.ts @@ -0,0 +1,243 @@ +/** + * One run of the nightly whole-database dump. + * + * The dump is the artifact that rebuilds a lost server: every account, every + * login, every grant and managed profile, the job queue and the instance + * settings, in one `pg_dump --format=custom` file with a manifest beside it + * (`dump-core.mjs`). It is written to `DB_BACKUP_DIR` (a named volume in the + * bundled compose file) and, when the off-host bucket is configured, uploaded + * under `database/` in the same envelope the per-account copies use, sealed + * with `BACKUP_ENCRYPTION_KEY`. + * + * Without a writable directory (an install that runs the image without the + * volume) the dump is written to a temporary directory, uploaded and removed: + * the off-host copy still happens, nothing is kept on the server, and the + * ledger and the boot summary say so. With neither, the run is skipped and + * recorded as such. + * + * Every run leaves one `DatabaseBackupRun` row: started, then finished with + * what happened on each leg. `getDatabaseBackupHealth` reads them. + */ +import { createReadStream } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; + +import type { PrismaClient } from "@/generated/prisma/client"; +import { + getS3Client, + loadOffhostConfig, + uploadEncryptedBackup, + type S3Like, +} from "@/lib/jobs/offhost-backup"; +import { annotate } from "@/lib/logging/context"; +import packageJson from "../../../package.json"; + +import { + applyRetention, + createDatabaseDump, + databaseBackupEnabled, + isWritableDirectory, + offhostObjectKey, + resolveBackupDir, + resolveDailyKeep, +} from "./dump-core.mjs"; + +/** How long ledger rows are kept. A run a day; a year and a bit of history. */ +const LEDGER_RETENTION_DAYS = 400; + +export type DatabaseBackupLocalResult = + "written" | "no_directory" | "failed" | "skipped"; +export type DatabaseBackupOffhostResult = + "uploaded" | "not_configured" | "failed" | "skipped"; + +export interface DatabaseBackupReport { + status: "succeeded" | "failed" | "skipped"; + fileName: string | null; + dumpBytes: number | null; + local: DatabaseBackupLocalResult; + offhost: DatabaseBackupOffhostResult; + /** Dumps and manifests retention removed. */ + removed: number; + error: string | null; +} + +export interface RunDatabaseBackupOptions { + now?: Date; + /** Epoch ms at which `pg_dump` is stopped (`jobDeadline`). */ + deadline?: number; + env?: NodeJS.ProcessEnv; + /** A bucket double for tests; the configured bucket otherwise. */ + s3?: S3Like; +} + +/** The running build's version, as `/api/version` answers it. */ +function appVersion(env: NodeJS.ProcessEnv): string { + return env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version; +} + +/** An error as one line for the ledger and the admin page. */ +function reason(err: unknown): string { + const text = err instanceof Error ? err.message : String(err); + return text.replace(/\s+/g, " ").trim().slice(0, 500); +} + +/** Upload one local file as an off-host object, streamed. */ +async function uploadFile( + s3: S3Like, + key: Buffer, + path: string, +): Promise { + return uploadEncryptedBackup( + s3, + offhostObjectKey(`${basename(path)}.enc`), + key, + async (write) => { + for await (const chunk of createReadStream(path)) { + await write(chunk as Buffer); + } + }, + ); +} + +/** + * Take tonight's dump, keep it, copy it off-host, prune the old ones, and + * record what happened. Never throws: a failure is the report's `status`. + */ +export async function runDatabaseBackup( + prisma: PrismaClient, + options: RunDatabaseBackupOptions = {}, +): Promise { + const env = options.env ?? process.env; + const now = options.now ?? new Date(); + const dir = resolveBackupDir(env); + const localWritable = isWritableDirectory(dir); + const offhostConfig = (() => { + try { + return loadOffhostConfig(); + } catch { + // A malformed BACKUP_ENCRYPTION_KEY: the account copies fail on the + // same value with their own message; here it reads as an upload error. + return "invalid" as const; + } + })(); + + // Each leg reads "failed" until it has succeeded, so a run the process died + // under says so once its expiry has passed. + const report: DatabaseBackupReport = { + status: "failed", + fileName: null, + dumpBytes: null, + local: localWritable ? "failed" : "no_directory", + offhost: offhostConfig ? "failed" : "not_configured", + removed: 0, + error: null, + }; + const run = await prisma.databaseBackupRun.create({ + data: { + startedAt: now, + status: "running", + localResult: report.local, + offhostResult: report.offhost, + }, + select: { id: true }, + }); + + let scratch: string | null = null; + try { + if (!databaseBackupEnabled(env)) { + report.status = "skipped"; + report.local = "skipped"; + report.offhost = "skipped"; + report.error = "DB_BACKUP_ENABLED is false"; + } else if (!localWritable && !offhostConfig) { + report.status = "skipped"; + report.error = `no writable backup directory (${dir}) and no off-host bucket configured`; + } else { + const target = localWritable + ? dir + : (scratch = await mkdtemp(join(tmpdir(), "healthlog-dump-"))); + const dump = await createDatabaseDump({ + databaseUrl: env.DATABASE_URL ?? "", + dir: target, + kind: "daily", + now, + env, + appVersion: appVersion(env), + buildSha: env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null, + deadline: options.deadline, + }); + if (localWritable) report.local = "written"; + report.fileName = basename(dump.dumpPath); + report.dumpBytes = dump.manifest.dump.bytes; + + let uploadError: string | null = null; + if (offhostConfig === "invalid") { + uploadError = + "BACKUP_ENCRYPTION_KEY must be 64 hex characters or 32 bytes of base64"; + } else if (offhostConfig) { + try { + const s3 = options.s3 ?? (await getS3Client(offhostConfig)); + // The manifest goes up first: a dump object in the bucket always + // has the manifest a restore checks it against. + await uploadFile(s3, offhostConfig.encryptionKey, dump.manifestPath); + await uploadFile(s3, offhostConfig.encryptionKey, dump.dumpPath); + report.offhost = "uploaded"; + } catch (err) { + uploadError = `off-host upload failed: ${reason(err)}`; + } + } + + if (localWritable) { + const removed = await applyRetention(dir, { + dailyKeep: resolveDailyKeep(env), + now: now.getTime(), + }); + report.removed = removed.length; + } + + // Good when the dump exists somewhere it survives: on the volume, or in + // the bucket. An upload that failed beside a kept dump still fails the + // run, because the operator asked for both. + report.status = uploadError ? "failed" : "succeeded"; + report.error = uploadError; + } + } catch (err) { + report.status = "failed"; + report.error = reason(err); + } finally { + if (scratch) await rm(scratch, { recursive: true, force: true }); + } + + await prisma.databaseBackupRun.update({ + where: { id: run.id }, + data: { + finishedAt: new Date(), + status: report.status, + fileName: report.fileName, + dumpBytes: report.dumpBytes === null ? null : BigInt(report.dumpBytes), + localResult: report.local, + offhostResult: report.offhost, + error: report.error, + }, + }); + await prisma.databaseBackupRun.deleteMany({ + where: { + startedAt: { + lt: new Date(now.getTime() - LEDGER_RETENTION_DAYS * 86_400_000), + }, + }, + }); + + annotate({ + action: { name: "database.backup.run" }, + meta: { + status: report.status, + local: report.local, + offhost: report.offhost, + dump_bytes: report.dumpBytes ?? 0, + removed: report.removed, + }, + }); + return report; +} diff --git a/src/lib/jobs/database-backup-queue.ts b/src/lib/jobs/database-backup-queue.ts new file mode 100644 index 000000000..5285095eb --- /dev/null +++ b/src/lib/jobs/database-backup-queue.ts @@ -0,0 +1,40 @@ +/** + * The queue facts of the nightly whole-database dump: its name, its schedule + * and its send options. Apart from the job so the worker registry can read + * them without loading it (`job-registry-graph.test.ts`). + */ +import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy"; +import { resolveBackupSchedule } from "@/lib/database-backup/dump-core.mjs"; + +export const DATABASE_BACKUP_QUEUE = "database-backup"; + +/** + * `DB_BACKUP_SCHEDULE`, read once when the worker loads its registry, in the + * zone every cron runs in (`createAndSchedule`, Europe/Berlin). Default 01:30, + * an hour before the off-host account copies so the two uploads do not meet. + * A value that is not a five-field cron keeps the default, and the boot + * readiness summary says so. + */ +export const DATABASE_BACKUP_CRON = resolveBackupSchedule(process.env).cron; + +/** + * Four hours. `pg_dump` of a large instance takes minutes, the upload of the + * dump about as long as the dump; the window is there to be impossible to hit + * for a working run. The handler kills `pg_dump` at three quarters of it + * (`jobDeadline`), so a hung dump ends as a failed run with a reason rather + * than an expired job. + */ +export const DATABASE_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60; + +/** + * What the nightly schedule sends with: the expiry, the backups' heartbeat, + * and one retry. A dump that failed for a passing reason (a lock it waited + * out, a bucket that answered 503) gets a second attempt; one that fails + * again waits for the next night, and the failure stays on the admin page. + */ +export const DATABASE_BACKUP_SEND_OPTIONS = { + expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS, + heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS, + retryLimit: 1, + retryDelay: 15 * 60, +} as const; diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 250838b7d..ab79d94c1 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -78,6 +78,12 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "continued", "conversation_titles_migrated", "created", + // The nightly whole-database dump: where it went and how big. + "database_backup_bytes", + "database_backup_local", + "database_backup_offhost", + "database_backup_removed", + "database_backup_status", "daily_rows_retired", "days_consolidated", "days_rebuilt", diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index cf6fbf1ba..f34ded204 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -494,9 +494,9 @@ function createEnvelopeStream( return { stream, bytes: () => written }; } -/** Produces the backup JSON in pieces. Every piece is written in order. */ +/** Produces the plaintext (JSON text, or a dump's bytes) in ordered pieces. */ export type BackupJsonProducer = ( - write: (chunk: string) => Promise, + write: (chunk: string | Buffer) => Promise, ) => Promise; export interface UploadBackupOptions { @@ -572,9 +572,9 @@ export async function uploadEncryptedBackup( }, ); - const write = async (chunk: string): Promise => { + const write = async (chunk: string | Buffer): Promise => { if (failure) throw failure; - if (gzip.write(chunk, "utf8")) return; + if (gzip.write(chunk, "utf8")) return; // a Buffer ignores the encoding await new Promise((resolve, reject) => { const onDrain = (): void => { gzip.off("error", onError); diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 28ed26810..9a2503c1b 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -42,6 +42,7 @@ import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key- import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill"; import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run"; import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy"; +import { DATABASE_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/database-backup-queue"; import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue"; import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore"; import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue"; @@ -332,6 +333,17 @@ export const QUEUE_RUNTIME: Readonly> = { exclusive: "lockedPass", why: "Serialises every account's whole record.", }, + "database-backup": { + runtime: "long", + expireInSeconds: DATABASE_BACKUP_EXPIRE_SECONDS, + expiryVia: ["DATABASE_BACKUP_SEND_OPTIONS"], + stop: { + file: "lib/jobs/reminder/backup-handlers.ts", + fn: "handleDatabaseBackup", + }, + exclusive: "lockedPass", + why: "Runs pg_dump over the whole database and uploads the dump off-host.", + }, "data-backup-offhost": { runtime: "long", expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS, diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts index 70429c156..6c84a3eaf 100644 --- a/src/lib/jobs/registry/maintenance.ts +++ b/src/lib/jobs/registry/maintenance.ts @@ -1,5 +1,6 @@ /** - * The housekeeping queues: backups (weekly in-database, nightly off-host), + * The housekeeping queues: backups (the nightly whole-database dump, weekly + * in-database account copies, nightly off-host), * the restore drill and restores, every ledger and retention cleanup, the * medication maintenance passes, the imports (Apple Health, Health Connect, * medication intake), the converging encryption backfills, the document vault @@ -47,6 +48,11 @@ import { DATA_BACKUP_QUEUE, DATA_BACKUP_SEND_OPTIONS, } from "@/lib/jobs/data-backup-policy"; +import { + DATABASE_BACKUP_CRON, + DATABASE_BACKUP_QUEUE, + DATABASE_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/database-backup-queue"; import { DAWARICH_SYNC_CRON, DAWARICH_SYNC_QUEUE, @@ -300,6 +306,16 @@ const jobs: readonly JobDefinition[] = [ lane: true, load: async () => (await backups()).handleDataBackup, }), + // The whole-database dump: the copy that rebuilds a lost server. Locked, so + // a retry that starts while a run is still dumping waits its turn. + defineJob({ + queue: DATABASE_BACKUP_QUEUE, + cron: DATABASE_BACKUP_CRON, + scheduleOptions: DATABASE_BACKUP_SEND_OPTIONS, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + load: async () => (await backups()).handleDatabaseBackup, + }), prune( RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index c8bc8fa83..845f03927 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -5,7 +5,7 @@ * schedules, and boss.work registrations. */ import { type Job, type JobWithMetadata } from "pg-boss"; -import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobBudget, jobDeadline } from "@/lib/jobs/job-budget"; import { recordError } from "@/lib/jobs/worker-status"; import { BackupBlobTooLargeError, @@ -31,6 +31,37 @@ import { } from "@/lib/jobs/backup-pass-attempts"; import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; +/** + * The nightly whole-database dump (`database-backup`). + * + * One `pg_dump` in a child process, so it is not in the heavy lane: the heap + * holds a part of the upload at a time, not the database. `pg_dump` is stopped + * at three quarters of the job's expiry (`jobDeadline`), so a dump that hangs + * on a lock ends as a failed run with its reason rather than an expired job. + * A skipped run (switched off, nowhere to put it) completes; the admin page + * and the boot summary carry why. + */ +export async function handleDatabaseBackup( + jobs: Job>[], +): Promise { + return withBackgroundEvent("job.database_backup", async () => { + const { runDatabaseBackup } = await import("@/lib/database-backup/run"); + const report = await runDatabaseBackup(getWorkerPrisma(), { + deadline: jobs[0] ? jobDeadline(jobs[0]) : undefined, + }); + const did = { + database_backup_status: report.status, + database_backup_local: report.local, + database_backup_offhost: report.offhost, + database_backup_bytes: report.dumpBytes ?? 0, + database_backup_removed: report.removed, + }; + return report.status === "failed" + ? jobFailed("database dump failed", report.error ?? undefined, did) + : jobDone(did); + }); +} + export interface DataBackupPayload { triggeredAt: string; } diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index d90560c84..c95d658a6 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -99,6 +99,7 @@ const subsystemSurface = { "audit-log-cleanup": { audience: "system" }, "step-up-elevation-cleanup": { audience: "system" }, "data-backup-offhost": { audience: "system" }, + "database-backup": { audience: "system" }, "data-restore-drill": { audience: "system" }, "offhost-backup-purge": { audience: "system" }, "backup-restore": { audience: "system" }, diff --git a/tests/integration/__golden__/queue-registration.json b/tests/integration/__golden__/queue-registration.json index d85705a21..31a41afe6 100644 --- a/tests/integration/__golden__/queue-registration.json +++ b/tests/integration/__golden__/queue-registration.json @@ -1,5 +1,5 @@ { - "registeredQueueCount": 122, + "registeredQueueCount": 123, "registeredQueues": [ "achievement-unlock-sweep", "apple-health-import", @@ -23,6 +23,7 @@ "data-backup", "data-backup-offhost", "data-restore-drill", + "database-backup", "dawarich-sync", "dense-intraday-hourly-rebuild", "dense-intraday-retention", @@ -455,6 +456,21 @@ "retry_delay_max": null, "retry_limit": 2 }, + { + "dead_letter": null, + "deletion_seconds": 604800, + "expire_seconds": 900, + "heartbeat_seconds": null, + "name": "database-backup", + "notify": false, + "partition": false, + "policy": "standard", + "retention_seconds": 1209600, + "retry_backoff": false, + "retry_delay": 0, + "retry_delay_max": null, + "retry_limit": 2 + }, { "dead_letter": null, "deletion_seconds": 604800, @@ -2137,6 +2153,21 @@ }, "timezone": "Europe/Berlin" }, + { + "cron": "30 1 * * *", + "data": {}, + "key": "", + "kind": "cron", + "name": "database-backup", + "options": { + "expireInSeconds": 14400, + "heartbeatSeconds": 300, + "retryDelay": 900, + "retryLimit": 1, + "tz": "Europe/Berlin" + }, + "timezone": "Europe/Berlin" + }, { "cron": "20 3 * * *", "data": {}, @@ -3022,6 +3053,13 @@ "pollingIntervalSeconds": 30 } }, + { + "queue": "database-backup", + "options": { + "localConcurrency": 1, + "pollingIntervalSeconds": 30 + } + }, { "queue": "dawarich-sync", "options": { diff --git a/tests/integration/database-backup.test.ts b/tests/integration/database-backup.test.ts new file mode 100644 index 000000000..58e778188 --- /dev/null +++ b/tests/integration/database-backup.test.ts @@ -0,0 +1,511 @@ +/** + * The nightly whole-database dump against a real Postgres: the job writes a + * dump and its manifest, records the run, copies both off-host in the + * envelope the restore opens, keeps only what retention allows; and the + * restore puts the dump into an empty database and refuses one with tables. + * + * `pg_dump` and `pg_restore` are the test container's own. A workstation may + * have none, or one of another major than the server (which `pg_dump` + * refuses), so the test puts two small forwarders on PATH that run the tool + * inside the database container with `docker exec`. Everything else, the + * snapshot, the counts, the manifest, the ledger and the restore's checks, + * runs exactly as in the image. + */ +import { execFileSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Readable } from "node:stream"; + +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +import { prisma } from "@/lib/db"; +import { getDatabaseBackupHealth } from "@/lib/database-backup/health"; +import { runDatabaseBackup } from "@/lib/database-backup/run"; +import { + connect, + countRows, + decryptOffhostFile, + offhostObjectKey, + readManifestFile, +} from "@/lib/database-backup/dump-core.mjs"; + +import { main as restoreMain } from "../../scripts/db-restore.mjs"; +import { shippedMigrations } from "../../scripts/pending-migrations.mjs"; + +const OFFHOST_KEY = "e".repeat(64); +const work = mkdtempSync(join(tmpdir(), "hl-db-backup-")); +const backups = join(work, "backups"); +const shims = join(work, "bin"); +const originalPath = process.env.PATH; + +/** The id of the container that serves DATABASE_URL. */ +function containerId(): string { + const port = new URL(process.env.DATABASE_URL ?? "").port; + const rows = execFileSync( + "docker", + ["ps", "--format", "{{.ID}} {{.Ports}}"], + { + encoding: "utf8", + }, + ).split("\n"); + const row = rows.find((r) => r.includes(`:${port}->5432/tcp`)); + if (!row) throw new Error(`no container publishes port ${port}`); + return row.split(" ")[0]; +} + +/** + * `pg_dump` / `pg_restore` that run inside the database container. The + * connection comes from the PG* variables the core sets; host and port become + * the container's own. A file argument that exists on this machine is sent on + * standard input, which both tools read when no file is named. + */ +function writeShims(container: string): void { + for (const tool of ["pg_dump", "pg_restore"]) { + const path = join(shims, tool); + writeFileSync( + path, + `#!/usr/bin/env node +const { spawn } = require("node:child_process"); +const { createReadStream, existsSync } = require("node:fs"); +const args = process.argv.slice(2); +let input = null; +if (args.length && !args[args.length - 1].startsWith("-") && existsSync(args[args.length - 1])) { + input = args.pop(); +} +const env = []; +for (const name of ["PGUSER", "PGPASSWORD", "PGDATABASE"]) { + if (process.env[name] !== undefined) env.push("-e", name + "=" + process.env[name]); +} +env.push("-e", "PGHOST=127.0.0.1", "-e", "PGPORT=5432"); +const child = spawn("docker", ["exec", "-i", ...env, ${JSON.stringify(container)}, ${JSON.stringify(tool)}, ...args], { + stdio: ["pipe", "inherit", "inherit"], +}); +(input ? createReadStream(input) : process.stdin).pipe(child.stdin); +child.on("exit", (code, signal) => process.exit(code ?? (signal ? 1 : 0))); +`, + ); + chmodSync(path, 0o755); + } +} + +/** A bucket in memory, with the one call the job's uploader uses. */ +function memoryBucket() { + const objects = new Map(); + return { + objects, + putStream: async (key: string, body: Readable) => { + const chunks: Buffer[] = []; + for await (const c of body) chunks.push(c as Buffer); + objects.set(key, Buffer.concat(chunks)); + }, + }; +} + +function backupEnv(extra: Record = {}): NodeJS.ProcessEnv { + return { ...process.env, DB_BACKUP_DIR: backups, ...extra }; +} + +beforeAll(async () => { + const { mkdirSync } = await import("node:fs"); + mkdirSync(shims, { recursive: true }); + mkdirSync(backups, { recursive: true }); + writeShims(containerId()); + process.env.PATH = `${shims}:${originalPath}`; + for (const name of ["dump-a", "dump-b", "dump-c"]) { + await prisma.user.create({ + data: { username: `${name}-${Date.now()}` }, + }); + } +}); + +afterAll(() => { + process.env.PATH = originalPath; + vi.unstubAllEnvs(); + rmSync(work, { recursive: true, force: true }); +}); + +describe("the nightly database dump", () => { + let dumpFile = ""; + + it("writes a checked dump and manifest, records the run and copies both off-host", async () => { + vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.invalid"); + vi.stubEnv("BACKUP_S3_BUCKET", "b"); + vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a"); + vi.stubEnv("BACKUP_S3_SECRET_KEY", "s"); + vi.stubEnv("BACKUP_ENCRYPTION_KEY", OFFHOST_KEY); + const bucket = memoryBucket(); + const now = new Date("2026-10-20T01:30:00Z"); + + const report = await runDatabaseBackup(prisma, { + now, + env: backupEnv(), + s3: bucket as never, + }); + expect(report).toMatchObject({ + status: "succeeded", + local: "written", + offhost: "uploaded", + error: null, + fileName: "healthlog-20261020T013000Z-daily.dump", + }); + dumpFile = join(backups, report.fileName ?? ""); + + // Both files, under their final names, and nothing half-written. + expect(readdirSync(backups).sort()).toEqual([ + "healthlog-20261020T013000Z-daily.dump", + "healthlog-20261020T013000Z-daily.manifest.json", + ]); + expect(readFileSync(dumpFile).subarray(0, 5).toString("latin1")).toBe( + "PGDMP", + ); + + const manifest = await readManifestFile( + join(backups, "healthlog-20261020T013000Z-daily.manifest.json"), + ); + expect(manifest.kind).toBe("daily"); + expect(manifest.dump.bytes).toBe(report.dumpBytes); + expect(manifest.latestMigration).toBe( + shippedMigrations(join(process.cwd(), "prisma", "migrations")).at(-1), + ); + expect(manifest.postgres.serverVersionNum).toBeGreaterThanOrEqual(160000); + expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count()); + expect(Object.keys(manifest.rowCounts).length).toBeGreaterThan(100); + expect(manifest.keys.encryption.keys.length).toBeGreaterThan(0); + expect(manifest.keys.backupEncryptionKey).toMatch(/^[0-9a-f]{12}$/); + + // The ledger row and what the admin page reads from it. + const runs = await prisma.databaseBackupRun.findMany({ + where: { startedAt: now }, + }); + expect(runs).toHaveLength(1); + expect(runs[0]).toMatchObject({ + status: "succeeded", + fileName: "healthlog-20261020T013000Z-daily.dump", + localResult: "written", + offhostResult: "uploaded", + error: null, + }); + expect(Number(runs[0].dumpBytes)).toBe(manifest.dump.bytes); + const health = await getDatabaseBackupHealth( + new Date("2026-10-20T02:00:00Z"), + backupEnv(), + ); + expect(health).toMatchObject({ + enabled: true, + localDirWritable: true, + offhostConfigured: true, + lastGood: { + fileName: "healthlog-20261020T013000Z-daily.dump", + sizeBytes: manifest.dump.bytes, + local: true, + offhost: true, + }, + }); + + // The off-host pair opens, with the restore's own reader, to the very + // bytes on disk. + expect([...bucket.objects.keys()].sort()).toEqual([ + "database/healthlog-20261020T013000Z-daily.dump.enc", + "database/healthlog-20261020T013000Z-daily.manifest.json.enc", + ]); + const opened = join(work, "opened"); + const { mkdirSync } = await import("node:fs"); + mkdirSync(opened); + for (const name of [ + "healthlog-20261020T013000Z-daily.dump", + "healthlog-20261020T013000Z-daily.manifest.json", + ]) { + writeFileSync( + join(opened, `${name}.enc`), + bucket.objects.get(offhostObjectKey(`${name}.enc`)) as Buffer, + ); + await decryptOffhostFile({ + inPath: join(opened, `${name}.enc`), + outPath: join(opened, name), + objectKey: offhostObjectKey(`${name}.enc`), + keys: [Buffer.from(OFFHOST_KEY, "hex")], + }); + expect( + readFileSync(join(opened, name)).equals( + readFileSync(join(backups, name)), + ), + ).toBe(true); + } + vi.unstubAllEnvs(); + }); + + it("keeps DB_BACKUP_KEEP nightly dumps and removes the oldest", async () => { + for (const day of ["21", "22", "23"]) { + const report = await runDatabaseBackup(prisma, { + now: new Date(`2026-10-${day}T01:30:00Z`), + env: backupEnv({ DB_BACKUP_KEEP: "2" }), + }); + expect(report.status).toBe("succeeded"); + expect(report.offhost).toBe("not_configured"); + } + expect(readdirSync(backups).filter((n) => n.endsWith(".dump"))).toEqual([ + "healthlog-20261022T013000Z-daily.dump", + "healthlog-20261023T013000Z-daily.dump", + ]); + }); + + it("records a failed dump, leaves nothing that looks good, and the page says so", async () => { + const before = readdirSync(backups).sort(); + const url = new URL(process.env.DATABASE_URL ?? ""); + url.password = "not-the-password"; + const now = new Date("2026-10-24T01:30:00Z"); + const report = await runDatabaseBackup(prisma, { + now, + env: backupEnv({ DATABASE_URL: url.toString() }), + }); + expect(report.status).toBe("failed"); + expect(report.local).toBe("failed"); + expect(report.error).toMatch(/password authentication failed/); + expect(readdirSync(backups).sort()).toEqual(before); + const health = await getDatabaseBackupHealth( + new Date("2026-10-24T02:00:00Z"), + backupEnv(), + ); + expect(health.lastFailure?.error).toMatch(/password authentication/); + expect(health.lastGood?.fileName).toBe( + "healthlog-20261023T013000Z-daily.dump", + ); + }); + + it("skips, and says why, with nowhere to put a dump", async () => { + const report = await runDatabaseBackup(prisma, { + now: new Date("2026-10-24T03:00:00Z"), + env: backupEnv({ DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here" }), + }); + expect(report).toMatchObject({ + status: "skipped", + local: "no_directory", + offhost: "not_configured", + }); + expect(report.error).toMatch(/no writable backup directory/); + const off = await runDatabaseBackup(prisma, { + now: new Date("2026-10-24T04:00:00Z"), + env: backupEnv({ DB_BACKUP_ENABLED: "false" }), + }); + expect(off).toMatchObject({ + status: "skipped", + local: "skipped", + offhost: "skipped", + }); + }); + + describe("restoring it", () => { + const target = `restore_${process.pid}_${Date.now()}`; + let targetUrl = ""; + + beforeAll(async () => { + const admin = await connect(process.env.DATABASE_URL ?? ""); + await admin.query(`CREATE DATABASE ${target}`); + await admin.end(); + const url = new URL(process.env.DATABASE_URL ?? ""); + url.pathname = `/${target}`; + targetUrl = url.toString(); + }); + + afterAll(async () => { + const admin = await connect(process.env.DATABASE_URL ?? ""); + await admin.query(`DROP DATABASE IF EXISTS ${target} WITH (FORCE)`); + await admin.end(); + }); + + it("refuses the database the dump came from, which has tables", async () => { + const errors: string[] = []; + const spy = vi + .spyOn(console, "error") + .mockImplementation((l: string) => void errors.push(String(l))); + const code = await restoreMain(["latest"], backupEnv()); + spy.mockRestore(); + expect(code).toBe(1); + expect(errors.join("\n")).toMatch(/only into an empty database/); + }); + + it("restores the newest dump into an empty database with every row count", async () => { + const lines: string[] = []; + const spy = vi + .spyOn(console, "log") + .mockImplementation((l: string) => void lines.push(String(l))); + const code = await restoreMain( + ["latest"], + backupEnv({ DATABASE_URL: targetUrl }), + ); + spy.mockRestore(); + expect(code, lines.join("\n")).toBe(0); + expect(lines.join("\n")).toMatch( + /all \d+ tables hold the row counts the manifest names/, + ); + + const manifest = await readManifestFile( + join(backups, "healthlog-20261023T013000Z-daily.manifest.json"), + ); + const client = await connect(targetUrl); + try { + expect(await countRows(client)).toEqual(manifest.rowCounts); + const { rows } = await client.query( + "SELECT count(*)::int AS n FROM users WHERE username LIKE 'dump-%'", + ); + expect(rows[0].n).toBeGreaterThanOrEqual(3); + } finally { + await client.end(); + } + // Restoring again into the now full database is refused. + const again = await restoreMain( + ["latest"], + backupEnv({ DATABASE_URL: targetUrl }), + ); + expect(again).toBe(1); + expect(existsSync(dumpFile)).toBe(false); // the first dump aged out + }); + }); +}); + +describe("the dump before migrating", () => { + const preDir = join(work, "pre"); + const appDir = join(work, "app"); + const failing = join(work, "failing-bin"); + let cwd = ""; + + beforeAll(async () => { + const { cpSync, mkdirSync } = await import("node:fs"); + // An image that ships one migration the database has not applied. + mkdirSync(join(appDir, "prisma"), { recursive: true }); + cpSync( + join(process.cwd(), "prisma", "migrations"), + join(appDir, "prisma", "migrations"), + { recursive: true }, + ); + mkdirSync(join(appDir, "prisma", "migrations", "9999_not_applied_yet")); + writeFileSync( + join( + appDir, + "prisma", + "migrations", + "9999_not_applied_yet", + "migration.sql", + ), + "", + ); + mkdirSync(preDir, { recursive: true }); + mkdirSync(failing, { recursive: true }); + writeFileSync( + join(failing, "pg_dump"), + "#!/bin/sh\necho 'pg_dump: error: could not write to output file: No space left on device' >&2\nexit 1\n", + ); + chmodSync(join(failing, "pg_dump"), 0o755); + cwd = process.cwd(); + process.chdir(appDir); + }); + + afterAll(() => { + process.chdir(cwd); + process.env.PATH = `${shims}:${originalPath}`; + }); + + async function preMigration(env: Record) { + const { preMigrationDump } = await import("../../scripts/db-dump.mjs"); + const errors: string[] = []; + const spyErr = vi + .spyOn(console, "error") + .mockImplementation((l: string) => void errors.push(String(l))); + const spyLog = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + const code = await preMigrationDump({ ...process.env, ...env }); + return { code, errors: errors.join("\n") }; + } finally { + spyErr.mockRestore(); + spyLog.mockRestore(); + } + } + + it("writes a checked dump with its manifest, and keeps the last two", async () => { + for (let i = 0; i < 3; i++) { + if (i > 0) await new Promise((r) => setTimeout(r, 1100)); + const { code } = await preMigration({ DB_BACKUP_DIR: preDir }); + expect(code).toBe(0); + } + const names = readdirSync(preDir).sort(); + expect(names).toHaveLength(4); + expect(names.filter((n) => n.endsWith("-pre-migration.dump"))).toHaveLength( + 2, + ); + const manifest = await readManifestFile( + join(preDir, names.find((n) => n.endsWith(".manifest.json")) ?? ""), + ); + expect(manifest.kind).toBe("pre-migration"); + expect(manifest.rowCounts["public.users"]).toBe(await prisma.user.count()); + }); + + it("does nothing when switched off", async () => { + const before = readdirSync(preDir); + const { code } = await preMigration({ + DB_BACKUP_DIR: preDir, + DB_BACKUP_BEFORE_MIGRATE: "false", + }); + expect(code).toBe(0); + expect(readdirSync(preDir)).toEqual(before); + }); + + it("warns and lets the boot go on without a writable directory", async () => { + const { code, errors } = await preMigration({ + DB_BACKUP_DIR: "/proc/healthlog-cannot-write-here", + }); + expect(code).toBe(0); + expect(errors).toMatch( + /is not writable, so no dump is taken before the 1 pending migration/, + ); + }); + + it("stops the boot when the dump fails on a writable directory, and leaves no dump behind", async () => { + // A second of its own, so the name cannot meet the last dump's. + await new Promise((r) => setTimeout(r, 1100)); + const before = readdirSync(preDir).sort(); + process.env.PATH = `${failing}:${originalPath}`; + const { code, errors } = await preMigration({ DB_BACKUP_DIR: preDir }); + process.env.PATH = `${shims}:${originalPath}`; + expect(code).toBe(1); + expect(errors).toMatch( + /the database dump before migrating failed: .*No space left/, + ); + expect(readdirSync(preDir).sort()).toEqual(before); + }); +}); + +describe("two dumps in the same second", () => { + it("refuses the second and leaves the first whole", async () => { + const { createDatabaseDump } = + await import("@/lib/database-backup/dump-core.mjs"); + const dir = join(work, "same-second"); + const { mkdirSync } = await import("node:fs"); + mkdirSync(dir); + const now = new Date("2026-10-26T01:30:00Z"); + const options = { + databaseUrl: process.env.DATABASE_URL ?? "", + dir, + kind: "daily" as const, + now, + }; + await createDatabaseDump(options); + const before = readdirSync(dir) + .sort() + .map((n) => [n, readFileSync(join(dir, n)).length]); + await expect(createDatabaseDump(options)).rejects.toThrow(/already exists/); + expect( + readdirSync(dir) + .sort() + .map((n) => [n, readFileSync(join(dir, n)).length]), + ).toEqual(before); + }); +}); From e665f17f521285f144828af8ce6bac49456a8e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 12:52:07 +0100 Subject: [PATCH 11/19] feat(backup): dump before migrating, and restore a dump into an empty database The image now carries postgresql18-client (pg_dump and pg_restore 18.6) and a /var/lib/healthlog/backups directory owned by the app user, which the bundled compose file backs with a new `backups` volume for `app` and the commented `app-worker`; a fresh named volume inherits that owner. Before `migrate deploy`, when migrations are pending on a database that has applied some, the entrypoint writes healthlog-

+ ); +} diff --git a/src/types/backups.ts b/src/types/backups.ts index f3c042999..5f2a28e2c 100644 --- a/src/types/backups.ts +++ b/src/types/backups.ts @@ -9,6 +9,7 @@ * both sides a route-handler-independent shared type home; the route * keeps owning the HTTP contract, the component keeps owning the UI. */ +import type { DatabaseBackupHealth } from "@/lib/database-backup/health"; import type { BackupPassActivity } from "@/lib/jobs/backup-pass-attempts"; import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status"; import type { OffhostBackupFreshness } from "@/lib/jobs/offhost-backup-freshness"; @@ -101,6 +102,12 @@ export interface BackupsList { * the operator's bucket, and those two fail independently. */ offhost: OffhostBackupOverview; + /** + * The nightly whole-database dump, the copy that rebuilds a lost server: + * the newest good dump (kept here, off-host, or both), the newest failed + * run, and whether this server can keep dumps at all. + */ + database: DatabaseBackupHealth; /** * Soft retention hint — the worker is configured for weekly backups * (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and From ca644647a0153a1fa8d783960b25b9db537b2442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 25 Oct 2026 12:52:56 +0100 Subject: [PATCH 13/19] docs(ops): describe disaster recovery from the database dump docs/ops/disaster-recovery.md says what is dumped and where, that local dumps are unencrypted and why, which keys a restore needs and what each one costs when missing, how to restore onto a new server or back to the state before a migration, and how the procedure is tested. The deploy runbook and the getting-started guide now describe the dump the entrypoint takes before migrating, and the off-host account copy page points at the dump for rebuilding a server. --- docs/ops/backup-restore.md | 5 + docs/ops/deploy.md | 35 ++++-- docs/ops/disaster-recovery.md | 164 +++++++++++++++++++++++++++ docs/self-hosting/getting-started.md | 28 ++++- 4 files changed, 215 insertions(+), 17 deletions(-) create mode 100644 docs/ops/disaster-recovery.md diff --git a/docs/ops/backup-restore.md b/docs/ops/backup-restore.md index 09524b2c8..f3d4c0571 100644 --- a/docs/ops/backup-restore.md +++ b/docs/ops/backup-restore.md @@ -5,6 +5,11 @@ user's JSON dump, encrypted with AES-256-GCM under a SEPARATE key (`BACKUP_ENCRYPTION_KEY`), to any S3-compatible bucket — Cloudflare R2, AWS S3, Backblaze B2, MinIO, etc. +These per-account copies roll one account back on the same server. To +rebuild a server, use the nightly database dump, which the same job +configuration also uploads (`database/healthlog-