From f685cbe4605afea673c43f1b031eee46a72faf4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 01:26:37 +0200 Subject: [PATCH 01/21] =?UTF-8?q?chore(release):=20v1.40.0=20=E2=80=94=20b?= =?UTF-8?q?ack=20up=20the=20encryption=20key,=20medication=20courses=20and?= =?UTF-8?q?=20categories,=20your=20own=20symptoms,=20document=20reads=20in?= =?UTF-8?q?=20the=20background?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 54 +++++++++++++++++++++++++++++++++++++++++++ README.md | 4 ++-- docs/api/openapi.yaml | 2 +- package.json | 2 +- public/sw.js | 2 +- 5 files changed, 59 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53cb335cb..a98d936ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,59 @@ # Changelog +## [1.40.0] — 2026-10-04 + +Back up the encryption key and refuse a key that cannot open the data; +several courses and custom categories for medications; custom symptoms; +document reads in the background; sick notes; lab unit refusal and charts +that follow the source priority. + +### Added + +- **Encryption key backup step (migration 0365).** Admin step with key id, + fingerprint and per-platform guidance (`HEALTHLOG_PLATFORM`), an optional + "check my copy" that compares and discards the pasted key, a reminder that + returns when the key changes. +- **Boot key check.** A canary per key id, probed oldest values first across + string and Bytes columns; a proven mismatch answers 503 + `encryption.key_mismatch` everywhere except `/api/health` and + `/api/version`, MCP included, and starts no worker; an unproven result + serves, warns and re-checks. `ENCRYPTION_KEY_CHECK=warn` as a last resort. +- **Medication courses and custom categories (#1024, #1041, migration + 0368).** Several courses per medication; compliance, streak, cadence and + dose history count every course and never the gaps; existing dated + medications become one course each. Own categories as `custom:` with + `categoryLabel`; the runtime category table is now in the schema. +- **Custom symptoms (#981, migration 0369).** Up to eight definitions, events + with a 0 to 10 intensity, a capture kind, one correlation channel per + symptom, a Coach block; gated on the Illness module. +- **Document AI as background runs (#1090, migrations 0366 and 0370).** + Index, summary, suggestions and extraction accept `Prefer: respond-async`; + the lab scan always runs in the background; `GET /api/ai-runs/{id}`. +- **`SICK_NOTE` document kind.** +- **Natural durations (#1101)** in the sleep field. + +### Fixed + +- **Lab unit mismatch (#1094).** A reading in another unit than its marker is + refused (`labs.unit.mismatch`) on every write path; a marker's unit is + locked while it has readings (`biomarkers.unit.locked`). +- **Source priority in series (#1108).** Pulse, HRV and SpO2 series read the + ladder's source per day; unranked types keep every source. +- **Sign-in over plain http (#1097).** Refused with + `auth.session.insecure_transport` before a password, code or challenge is + used; the sign-in page says so first. +- **Chart window (#1102).** Ends at the end of today in the profile time zone. +- **Workouts.** The insight cap survives a delete (migration 0367); a changed + `manual:` resubmit updates the workout and its paragraph. +- **Display.** Clinician decimals in the owner's locale, stated dates east of + UTC+12, archive month grouping, lab list columns without a range, icon + picker touch targets and labels, symptom entry load errors. + +### Changed + +- Quick-entry forms load on demand; the four eager route baselines are + lowered to the new measurements. + ## [1.39.9] — 2026-10-03 Workouts can be logged by hand, the body fat tile opens its own page, and diff --git a/README.md b/README.md index 806f6649d..f5a20e17e 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,9 @@ Actively developed — new releases roughly weekly, issue reports and PRs welcom HealthLog started as something I built for myself. That it has turned into a project other people use, care about and help shape still surprises me, and I am grateful for every part of it. -Thank you to everyone who sent code that shipped: [@TimonBed](https://github.com/TimonBed), [@Antiheld86](https://github.com/Antiheld86), [@BernhardBuckel](https://github.com/BernhardBuckel), [@balajiv113](https://github.com/balajiv113), [@aucun6352](https://github.com/aucun6352), [@mathewcsims](https://github.com/mathewcsims) and [@muhdusama](https://github.com/muhdusama). A special thanks to [@BernhardBuckel](https://github.com/BernhardBuckel), who has been working through the lab screens one careful pull request at a time, with tests and a plan before every change. +Thank you to everyone who sent code that shipped: [@TimonBed](https://github.com/TimonBed), [@Antiheld86](https://github.com/Antiheld86), [@BernhardBuckel](https://github.com/BernhardBuckel), [@balajiv113](https://github.com/balajiv113), [@aucun6352](https://github.com/aucun6352), [@mathewcsims](https://github.com/mathewcsims) and [@muhdusama](https://github.com/muhdusama). A special thanks to [@BernhardBuckel](https://github.com/BernhardBuckel), who has gone far beyond one fix: careful reports with the code paths already traced, pull requests with tests and a plan before every change, and fixes that everyone using HealthLog benefits from, from the lab screens to how charts pick a source and average a day. -Thank you to everyone who opened an issue, asked a question in the discussions or wrote me an email. Most of what got better in the last months started with someone describing what went wrong for them, often with a log, a screenshot and a second try when the first fix was not enough. +Thank you to the people whose reports shaped HealthLog, again and again: [@lutzkind](https://github.com/lutzkind), [@HercoGC](https://github.com/HercoGC), [@mills1975](https://github.com/mills1975), [@sreeramachandramurthy](https://github.com/sreeramachandramurthy), [@CritLoren](https://github.com/CritLoren), [@mbreitkreuz](https://github.com/mbreitkreuz), [@k-nacion](https://github.com/k-nacion), [@NebuPookins](https://github.com/NebuPookins), [@Nazza01](https://github.com/Nazza01), [@Cnote43](https://github.com/Cnote43) and [@juliaanvaneijndt](https://github.com/juliaanvaneijndt). And to everyone else who opened an issue, asked a question in the discussions or wrote me an email. Most of what got better started with someone describing what went wrong for them, often with a log, a screenshot and a second try when the first fix was not enough. And thank you to everyone who supported the project with a coffee or by buying the iPhone app. I don't name anyone here, but I read every message that comes with it. It pays for the servers and the Apple developer account, and it tells me that HealthLog is worth the evenings. diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 25431a0bd..8b11dbff7 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: HealthLog API - version: 1.39.9 + version: 1.40.0 description: >- Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest. diff --git a/package.json b/package.json index 8f2a1b5f0..74f0243db 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.39.9", + "version": "1.40.0", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "PolyForm-Noncommercial-1.0.0", "homepage": "https://healthlog.dev", diff --git a/public/sw.js b/public/sw.js index 12e07572b..32351a83e 100644 --- a/public/sw.js +++ b/public/sw.js @@ -36,7 +36,7 @@ try { // v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild. const CACHE_VERSION = (typeof self !== "undefined" && self.__APP_VERSION__) || - /* @sw-version-fallback */ "v1.39.9"; + /* @sw-version-fallback */ "v1.40.0"; const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`; const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`; // v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*` From 8121935642591936d6c0aea22ad91898a43e0344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 08:17:34 +0200 Subject: [PATCH 02/21] Carry cycle free text readable in a portable backup A portable file carried the day-log note, the sensitive-category envelope and custom symptom labels as the stored ciphertext, so a restore onto a host with another key either refused the file or wrote values nobody could open. The portable file now carries them readable and the restore seals them under the receiving host's key; a disaster-recovery file keeps the ciphertext. A file written before this still restores: each ciphertext value is opened with this host's keys first, and one that does not open is kept out of the row and reported in the skip summary as cycleCiphertext. The key preflight leaves the cycle sections to that per-value check. --- docs/api/openapi.yaml | 52 +++ messages/de.json | 3 +- messages/en.json | 3 +- messages/es.json | 3 +- messages/fr.json | 3 +- messages/it.json | 3 +- messages/ko.json | 3 +- messages/pl.json | 3 +- src/components/admin/backups-section.tsx | 3 + src/lib/cycle/__tests__/backup.test.ts | 89 ++++- src/lib/cycle/backup.ts | 179 ++++++++-- .../export/__tests__/backup-key-ids.test.ts | 24 ++ src/lib/export/backup-key-ids.ts | 35 +- src/lib/export/restore-skips.ts | 11 +- src/lib/validations/backup.ts | 24 ++ .../admin-backups-canonical-roundtrip.test.ts | 6 +- .../backup-cycle-cross-key-restore.test.ts | 313 ++++++++++++++++++ tests/integration/cycle-import-export.test.ts | 17 +- 18 files changed, 731 insertions(+), 43 deletions(-) create mode 100644 tests/integration/backup-cycle-cross-key-restore.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 25431a0bd..ffdc1ead6 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -36605,6 +36605,54 @@ components: anyOf: - type: string - type: "null" + note: + anyOf: + - type: string + - type: "null" + sensitive: + anyOf: + - anyOf: + - type: object + properties: + sexualActivity: + type: boolean + protectedSex: + anyOf: + - type: boolean + - type: "null" + pregnancyTest: + anyOf: + - type: string + enum: + - NEGATIVE + - POSITIVE + - INDETERMINATE + - type: "null" + progesteroneTest: + anyOf: + - type: string + enum: + - NEGATIVE + - POSITIVE + - INDETERMINATE + - type: "null" + contraceptive: + anyOf: + - type: string + enum: + - NONE + - UNSPECIFIED + - IMPLANT + - INJECTION + - IUD + - INTRAVAGINAL_RING + - ORAL + - PATCH + - EMERGENCY + - type: "null" + additionalProperties: false + - type: string + - type: "null" source: type: string enum: @@ -36708,6 +36756,10 @@ components: anyOf: - type: string - type: "null" + label: + anyOf: + - type: string + - type: "null" required: - key - labelKey diff --git a/messages/de.json b/messages/de.json index 010e91b92..5390c5fb4 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7554,7 +7554,8 @@ "restoreFailureKeyMissing": "Die Sicherung braucht den Verschlüsselungsschlüssel {keys}, mit dem dieser Server sie nicht öffnen kann. Nimm den Schlüssel wieder in ENCRYPTION_KEYS auf, starte neu und starte die Wiederherstellung erneut.", "restoreFailureForeignReference": "Die Datei verweist auf Einträge, die zu einem anderen Konto gehören. Es wurde nichts geschrieben.", "previewKeyMissing": "Diese Sicherung lässt sich auf diesem Server nicht wiederherstellen: Sie braucht den Verschlüsselungsschlüssel {keys}. Nimm den Schlüssel zuerst wieder in ENCRYPTION_KEYS auf und starte neu.", - "restoreSkippedSymptomEpisodeReference": "Symptom-Eintrag mit Verknüpfung zu einer Erkrankung, die die Datei nicht enthält (ohne Verknüpfung behalten)" + "restoreSkippedSymptomEpisodeReference": "Symptom-Eintrag mit Verknüpfung zu einer Erkrankung, die die Datei nicht enthält (ohne Verknüpfung behalten)", + "restoreSkippedCycleCiphertext": "Zyklus-Notiz oder -Bezeichnung mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (Tag oder Symptom ohne sie behalten)" }, "danger-zone": { "title": "Gefahrenzone", diff --git a/messages/en.json b/messages/en.json index b65774c65..dd65185b3 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7554,7 +7554,8 @@ "restoreFailureKeyMissing": "The backup needs encryption key {keys}, which this server cannot open it with. Add the key back to ENCRYPTION_KEYS, restart, and start the restore again.", "restoreFailureForeignReference": "The file points at records that belong to another account. Nothing was written.", "previewKeyMissing": "This backup cannot be restored on this server: it needs encryption key {keys}. Add the key back to ENCRYPTION_KEYS and restart first.", - "restoreSkippedSymptomEpisodeReference": "Symptom entry linked to an illness the file does not contain (kept without the link)" + "restoreSkippedSymptomEpisodeReference": "Symptom entry linked to an illness the file does not contain (kept without the link)", + "restoreSkippedCycleCiphertext": "Cycle note or label sealed with a key this server does not hold (day or symptom kept without it)" }, "danger-zone": { "title": "Danger Zone", diff --git a/messages/es.json b/messages/es.json index 1f9b678f2..8ec71bc7a 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7554,7 +7554,8 @@ "restoreFailureKeyMissing": "La copia necesita la clave de cifrado {keys}, con la que este servidor no puede abrirla. Vuelve a añadir la clave a ENCRYPTION_KEYS, reinicia e inicia de nuevo la restauración.", "restoreFailureForeignReference": "El archivo apunta a registros que pertenecen a otra cuenta. No se escribió nada.", "previewKeyMissing": "Esta copia no se puede restaurar en este servidor: necesita la clave de cifrado {keys}. Vuelve a añadir la clave a ENCRYPTION_KEYS y reinicia primero.", - "restoreSkippedSymptomEpisodeReference": "Entrada de síntoma vinculada a una enfermedad que el archivo no contiene (se conserva sin el vínculo)" + "restoreSkippedSymptomEpisodeReference": "Entrada de síntoma vinculada a una enfermedad que el archivo no contiene (se conserva sin el vínculo)", + "restoreSkippedCycleCiphertext": "Nota o etiqueta del ciclo cifrada con una clave que este servidor no tiene (el día o el síntoma se conserva sin ella)" }, "danger-zone": { "title": "Zona de peligro", diff --git a/messages/fr.json b/messages/fr.json index 1c76f7992..e9d1163a5 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7554,7 +7554,8 @@ "restoreFailureKeyMissing": "La sauvegarde a besoin de la clé de chiffrement {keys}, avec laquelle ce serveur ne peut pas l'ouvrir. Remettez la clé dans ENCRYPTION_KEYS, redémarrez, puis relancez la restauration.", "restoreFailureForeignReference": "Le fichier renvoie à des éléments qui appartiennent à un autre compte. Rien n'a été écrit.", "previewKeyMissing": "Cette sauvegarde ne peut pas être restaurée sur ce serveur : elle a besoin de la clé de chiffrement {keys}. Remettez d'abord la clé dans ENCRYPTION_KEYS et redémarrez.", - "restoreSkippedSymptomEpisodeReference": "Entrée de symptôme liée à une maladie absente du fichier (conservée sans le lien)" + "restoreSkippedSymptomEpisodeReference": "Entrée de symptôme liée à une maladie absente du fichier (conservée sans le lien)", + "restoreSkippedCycleCiphertext": "Note ou libellé du cycle chiffré avec une clé que ce serveur ne possède pas (jour ou symptôme conservé sans)" }, "danger-zone": { "title": "Zone de danger", diff --git a/messages/it.json b/messages/it.json index 61397873a..d1538b30b 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7554,7 +7554,8 @@ "restoreFailureKeyMissing": "Il backup richiede la chiave di cifratura {keys}, con cui questo server non riesce ad aprirlo. Aggiungi di nuovo la chiave a ENCRYPTION_KEYS, riavvia e avvia di nuovo il ripristino.", "restoreFailureForeignReference": "Il file rimanda a voci che appartengono a un altro account. Non è stato scritto nulla.", "previewKeyMissing": "Questo backup non può essere ripristinato su questo server: richiede la chiave di cifratura {keys}. Aggiungi prima di nuovo la chiave a ENCRYPTION_KEYS e riavvia.", - "restoreSkippedSymptomEpisodeReference": "Voce di sintomo collegata a una malattia assente dal file (mantenuta senza collegamento)" + "restoreSkippedSymptomEpisodeReference": "Voce di sintomo collegata a una malattia assente dal file (mantenuta senza collegamento)", + "restoreSkippedCycleCiphertext": "Nota o etichetta del ciclo cifrata con una chiave che questo server non possiede (giorno o sintomo mantenuto senza)" }, "danger-zone": { "title": "Zona pericolosa", diff --git a/messages/ko.json b/messages/ko.json index cdabb7b22..a61e278d9 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7533,7 +7533,8 @@ "restoreFailureKeyMissing": "이 백업에는 암호화 키 {keys}가 필요하지만 이 서버는 그 키로 백업을 열 수 없습니다. 키를 ENCRYPTION_KEYS에 다시 추가하고 재시작한 뒤 복원을 다시 시작하세요.", "restoreFailureForeignReference": "파일이 다른 계정에 속한 기록을 가리키고 있습니다. 아무것도 기록되지 않았습니다.", "previewKeyMissing": "이 서버에서는 이 백업을 복원할 수 없습니다. 암호화 키 {keys}가 필요합니다. 먼저 키를 ENCRYPTION_KEYS에 다시 추가하고 재시작하세요.", - "restoreSkippedSymptomEpisodeReference": "파일에 없는 질병과 연결된 증상 기록(연결 없이 유지됨)" + "restoreSkippedSymptomEpisodeReference": "파일에 없는 질병과 연결된 증상 기록(연결 없이 유지됨)", + "restoreSkippedCycleCiphertext": "이 서버에 없는 키로 암호화된 주기 메모 또는 이름(해당 날짜나 증상은 그 내용 없이 유지됨)" }, "danger-zone": { "title": "위험 구역", diff --git a/messages/pl.json b/messages/pl.json index 1f931f743..6a51e0eeb 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7554,7 +7554,8 @@ "restoreFailureKeyMissing": "Kopia wymaga klucza szyfrowania {keys}, którym ten serwer nie może jej otworzyć. Dodaj klucz z powrotem do ENCRYPTION_KEYS, uruchom ponownie i rozpocznij przywracanie jeszcze raz.", "restoreFailureForeignReference": "Plik wskazuje na wpisy należące do innego konta. Nic nie zostało zapisane.", "previewKeyMissing": "Tej kopii nie da się przywrócić na tym serwerze: wymaga klucza szyfrowania {keys}. Najpierw dodaj klucz z powrotem do ENCRYPTION_KEYS i uruchom ponownie.", - "restoreSkippedSymptomEpisodeReference": "Wpis objawu powiązany z chorobą, której nie ma w pliku (zachowany bez powiązania)" + "restoreSkippedSymptomEpisodeReference": "Wpis objawu powiązany z chorobą, której nie ma w pliku (zachowany bez powiązania)", + "restoreSkippedCycleCiphertext": "Notatka lub etykieta cyklu zaszyfrowana kluczem, którego ten serwer nie ma (dzień lub objaw zachowany bez niej)" }, "danger-zone": { "title": "Strefa zagrożenia", diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index 3a0550cc5..9a295f8a3 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -464,6 +464,9 @@ function catalogueLabel( if (catalogue === "symptomEpisodeReference") { return t("admin.section.backups.restoreSkippedSymptomEpisodeReference"); } + if (catalogue === "cycleCiphertext") { + return t("admin.section.backups.restoreSkippedCycleCiphertext"); + } if (catalogue === "medicationTarget") { return t("admin.section.backups.restoreSkippedMedicationTarget"); } diff --git a/src/lib/cycle/__tests__/backup.test.ts b/src/lib/cycle/__tests__/backup.test.ts index 2bcc923ec..cd7318a86 100644 --- a/src/lib/cycle/__tests__/backup.test.ts +++ b/src/lib/cycle/__tests__/backup.test.ts @@ -1,4 +1,7 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto"; +import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker"; import { buildCycleBackupSection } from "../backup"; @@ -236,3 +239,87 @@ describe("a symptom the account created survives the round trip", () => { expect(section.customSymptoms).toEqual([]); }); }); + +describe("a portable file carries the cycle free text readable", () => { + // Ciphertext in a portable file is noise on any host with another key, so + // the note, the sensitive envelope and a custom label travel opened, and + // the restore seals them under the receiving host's key. + const savedKey = process.env.ENCRYPTION_KEY; + beforeEach(() => { + process.env.ENCRYPTION_KEY = "ab".repeat(32); + _resetCryptoCacheForTests(); + }); + afterEach(() => { + process.env.ENCRYPTION_KEY = savedKey; + _resetCryptoCacheForTests(); + }); + + function client(dayLog: Record, label: string | null) { + return { + cycleProfile: { findUnique: vi.fn().mockResolvedValue(null) }, + menstrualCycle: { findMany: vi.fn().mockResolvedValue([]) }, + cycleDayLog: { + findMany: vi + .fn() + .mockResolvedValue([ + { date: "2026-07-02", symptomLinks: [], ...dayLog }, + ]), + }, + cycleSymptom: { + findMany: vi.fn().mockResolvedValue([ + { + key: "custom_ache", + labelKey: "cycle.symptom.custom_ache", + categoryId: "cat-1", + icon: null, + sortOrder: 0, + isActive: true, + labelEncrypted: label, + }, + ]), + }, + } as never; + } + + it("opens each sealed value and carries no ciphertext", async () => { + const section = await buildCycleBackupSection( + client( + { + notesEncrypted: encrypt("Cramps at four"), + sensitiveEncrypted: encrypt( + JSON.stringify({ sexualActivity: true, contraceptive: "ORAL" }), + ), + }, + encrypt("Back ache"), + ), + "user-1", + {}, + ); + const day = section.cycleDayLogs[0]; + expect(day.note).toBe("Cramps at four"); + expect(day.sensitive).toEqual({ + sexualActivity: true, + contraceptive: "ORAL", + }); + expect(day).not.toHaveProperty("notesEncrypted"); + expect(day).not.toHaveProperty("sensitiveEncrypted"); + expect(section.customSymptoms[0].label).toBe("Back ache"); + expect(section.customSymptoms[0]).not.toHaveProperty("labelEncrypted"); + }); + + it("writes the unreadable marker where this host cannot open a value", async () => { + const section = await buildCycleBackupSection( + client( + { + notesEncrypted: "v9.bm90LWEtcmVhbC1jaXBoZXJ0ZXh0LWF0LWFsbC1yZWFsbHk=", + }, + null, + ), + "user-1", + {}, + ); + expect(section.cycleDayLogs[0].note).toBe(UNREADABLE_EXPORT_MARKER); + expect(section.cycleDayLogs[0].sensitive).toBeNull(); + expect(section.customSymptoms[0].label).toBeNull(); + }); +}); diff --git a/src/lib/cycle/backup.ts b/src/lib/cycle/backup.ts index 1a216b812..4862a41a3 100644 --- a/src/lib/cycle/backup.ts +++ b/src/lib/cycle/backup.ts @@ -5,10 +5,25 @@ * never drift. Mirrors the `backupPayloadSchema` cycle shapes. * * Reads are scoped to `userId` + `deletedAt: null` and exclude predicted - * (forecast) cycle rows — only observed history round-trips. `notesEncrypted` - * is carried as the ciphertext envelope VERBATIM: the backup never decrypts - * the day-log note, so a wrong-surface plaintext leak is impossible and the - * owner's note round-trips encrypted at rest. + * (forecast) cycle rows — only observed history round-trips. + * + * ## Free text follows the note contract + * + * Three columns are sealed: the day-log note, the sensitive-category envelope + * and a custom symptom's label. A disaster-recovery file carries them as the + * stored ciphertext, which the same instance's key reads back. A portable file + * carries them readable (`note`, `sensitive`, `label`), and the restore seals + * them under the receiving host's key, because a portable file exists to be + * moved, and ciphertext from one host is noise on another. + * + * Until v1.40 the portable file carried the ciphertext too, so a note restored + * onto a host with a different key was unreadable for good. Such a file is + * still accepted: every ciphertext value in it is opened with this host's keys + * before it is written, and one that does not open is kept out of the row and + * named in the skip report (`cycleCiphertext`), never written back as + * ciphertext nobody can read. The same check covers a disaster-recovery file, + * which is why `RESTORE_SELF_VERIFIED_SECTIONS` lets these sections past the + * restore's key preflight. */ import type { CervicalMucus, @@ -25,6 +40,9 @@ import type { PrismaClient, SecondarySymptom, } from "@/generated/prisma/client"; +import { decrypt, encrypt } from "@/lib/crypto"; +import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker"; +import { getEvent } from "@/lib/logging/context"; import type { BackupPayload } from "@/lib/validations/backup"; import { recordUnknownKeys, @@ -91,8 +109,17 @@ export interface CycleBackupSection { pregnancyTest: string | null; progesteroneTest: string | null; contraceptive: string | null; - sensitiveEncrypted: string | null; - notesEncrypted: string | null; + /** Ciphertext; present on a disaster-recovery payload. */ + sensitiveEncrypted?: string | null; + /** Ciphertext; present on a disaster-recovery payload. */ + notesEncrypted?: string | null; + /** Plaintext; present on a portable payload. */ + note?: string | null; + /** + * The sensitive-category envelope opened; present on a portable payload. + * The unreadable marker when this instance could not open it. + */ + sensitive?: CycleSensitiveFields | string | null; source: string; externalId: string | null; tz: string | null; @@ -142,18 +169,54 @@ export interface CycleBackupSection { sortOrder: number; isActive: boolean; /** - * The user's own words for the symptom, carried as ciphertext verbatim. + * The user's own words for the symptom: ciphertext on a disaster-recovery + * payload, plaintext (`label`) on a portable one. * - * The builder has always emitted it and this interface did not declare it, - * which held while the payload was assembled key by key and stopped holding - * the moment the section is spread onto the wire: an undeclared field is a - * field the wire type says is not there. Declared, so the type describes - * the file. + * Declared rather than left to the spread, because an undeclared field is + * a field the wire type says is not there. */ - labelEncrypted: string | null; + labelEncrypted?: string | null; + label?: string | null; }>; } +/** The five intent fields the sensitive-category envelope holds. */ +export interface CycleSensitiveFields { + sexualActivity?: boolean; + protectedSex?: boolean | null; + pregnancyTest?: string | null; + progesteroneTest?: string | null; + contraceptive?: string | null; +} + +/** Open a sealed value for a portable file, or the unreadable marker. */ +function openForExport(sealed: string, field: string): string { + try { + return decrypt(sealed); + } catch (err) { + getEvent()?.addWarning( + `cycle ${field} decrypt failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return UNREADABLE_EXPORT_MARKER; + } +} + +/** The sensitive envelope opened for a portable file, or the marker. */ +function openSensitiveForExport( + sealed: string | null, +): CycleSensitiveFields | string | null { + if (!sealed) return null; + const opened = openForExport(sealed, "sensitive envelope"); + if (opened === UNREADABLE_EXPORT_MARKER) return opened; + try { + return JSON.parse(opened) as CycleSensitiveFields; + } catch { + return UNREADABLE_EXPORT_MARKER; + } +} + /** * Build the cycle section of a user's full backup. Accepts any client with * the cycle delegates (the route's global client OR the worker's local @@ -259,8 +322,17 @@ export async function buildCycleBackupSection( pregnancyTest: d.pregnancyTest, progesteroneTest: d.progesteroneTest, contraceptive: d.contraceptive, - sensitiveEncrypted: d.sensitiveEncrypted, - notesEncrypted: d.notesEncrypted, + ...(disasterRecovery + ? { + sensitiveEncrypted: d.sensitiveEncrypted, + notesEncrypted: d.notesEncrypted, + } + : { + sensitive: openSensitiveForExport(d.sensitiveEncrypted), + note: d.notesEncrypted + ? openForExport(d.notesEncrypted, "day-log note") + : null, + }), source: d.source, externalId: d.externalId, tz: d.tz, @@ -277,7 +349,13 @@ export async function buildCycleBackupSection( icon: sym.icon, sortOrder: sym.sortOrder, isActive: sym.isActive, - labelEncrypted: sym.labelEncrypted, + ...(disasterRecovery + ? { labelEncrypted: sym.labelEncrypted } + : { + label: sym.labelEncrypted + ? openForExport(sym.labelEncrypted, "symptom label") + : null, + }), })), }; } @@ -369,6 +447,39 @@ function severityOrNull(value: number | null | undefined): number | null { : null; } +/** A readable value the file could not supply: the writer could not open it. */ +const UNOPENED = Symbol("unopened"); + +/** + * The sealed column value for one free-text field from the file. + * + * Ciphertext is written back only when this host opens it — a value sealed + * under a key this host does not hold (or the same key id over other key + * material) would be a row nobody can read. A readable value is sealed under + * this host's active key. Anything kept out is recorded under `path`. + */ +function sealForRestore( + sealed: string | null | undefined, + readable: string | null | undefined | typeof UNOPENED, + path: string, + unopened: string[], +): string | null { + if (sealed) { + try { + decrypt(sealed); + return sealed; + } catch { + unopened.push(path); + return null; + } + } + if (readable === UNOPENED) { + unopened.push(path); + return null; + } + return readable ? encrypt(readable) : null; +} + function enumOrNull( value: string | null | undefined, allow: ReadonlySet, @@ -383,8 +494,9 @@ function enumOrNull( * inside an existing transaction. Delete-then-recreate, matching the * measurement/mood restore contract. Symptom links are re-resolved against * the seeded catalogue by key, and each one gets back the 1-4 intensity the - * file recorded for it, or none if the file recorded none. `notesEncrypted` is - * written back as the ciphertext envelope verbatim — never re-encrypted. + * file recorded for it, or none if the file recorded none. Sealed free text + * comes back as the file-header comment describes: readable values sealed + * under this host's key, ciphertext written back only when this host opens it. * * A key that resolves against nothing is dropped from the day's links and * recorded on `skips`. It is a required argument rather than an optional one: @@ -400,6 +512,9 @@ export async function restoreCycleData( payload: BackupPayload, skips: RestoreSkipLog, ): Promise { + // Sealed values from the file this host's keys do not open, by file path. + const unopened: string[] = []; + // Wipe (child links cascade off the day-log delete). const dayLogs = await tx.cycleDayLog.deleteMany({ where: { userId: ownerId }, @@ -488,7 +603,12 @@ export async function restoreCycleData( icon: sym.icon ?? null, sortOrder: sym.sortOrder, isActive: sym.isActive, - labelEncrypted: sym.labelEncrypted ?? null, + labelEncrypted: sealForRestore( + sym.labelEncrypted, + sym.label, + `customSymptoms.${sym.key}.labelEncrypted`, + unopened, + ), }, // `key` is globally unique, so a seeded key would collide. Leave the // catalogue row alone: the account's links resolve against it either way. @@ -591,9 +711,22 @@ export async function restoreCycleData( pregnancyTest: enumOrNull(d.pregnancyTest, HOME_TESTS), progesteroneTest: enumOrNull(d.progesteroneTest, HOME_TESTS), contraceptive: enumOrNull(d.contraceptive, CONTRACEPTIVES), - // Both ciphertext envelopes written back verbatim — never re-encrypted. - sensitiveEncrypted: d.sensitiveEncrypted ?? null, - notesEncrypted: d.notesEncrypted ?? null, + sensitiveEncrypted: sealForRestore( + d.sensitiveEncrypted, + d.sensitive === undefined || d.sensitive === null + ? d.sensitive + : typeof d.sensitive === "string" + ? UNOPENED + : JSON.stringify(d.sensitive), + `cycleDayLogs.${d.date}.sensitiveEncrypted`, + unopened, + ), + notesEncrypted: sealForRestore( + d.notesEncrypted, + d.note, + `cycleDayLogs.${d.date}.notesEncrypted`, + unopened, + ), source: enumOrNull(d.source, CYCLE_SOURCES) ?? "MANUAL", externalId: d.externalId ?? null, tz: d.tz ?? null, @@ -608,6 +741,8 @@ export async function restoreCycleData( }); } + recordUnknownKeys(skips, "cycleCiphertext", unopened, unopened); + return { cycles: cycles.count, cycleDayLogs: dayLogs.count, diff --git a/src/lib/export/__tests__/backup-key-ids.test.ts b/src/lib/export/__tests__/backup-key-ids.test.ts index 03377c809..386f12878 100644 --- a/src/lib/export/__tests__/backup-key-ids.test.ts +++ b/src/lib/export/__tests__/backup-key-ids.test.ts @@ -138,6 +138,30 @@ describe("assessBackupKeys", () => { ).toEqual([]); }); + it("leaves the cycle sections to the restore, which opens each value itself", () => { + // A same-id impostor in a cycle note must neither refuse the file nor be + // picked as the sample that proves the key for the sections that need it. + const fine = encrypt("a measurement note under the real key"); + useKeys({ old: K3, cur: K2 }, "old"); + const impostor = encrypt("x"); + useKeys({ old: K1, cur: K2 }, "cur"); + const file = { + measurements: [{ notesEncrypted: fine }], + cycleDayLogs: [ + { notesEncrypted: impostor }, + { notesEncrypted: `gone.${Buffer.alloc(40).toString("base64")}` }, + ], + customSymptoms: [ + { labelEncrypted: `gone.${Buffer.alloc(40).toString("base64")}` }, + ], + }; + const collector = new BackupKeyIdCollector(); + collector.visit(file); + const verdict = assessBackupKeys(collector); + expect(verdict.missing).toEqual([]); + expect(verdict.unreadable).toEqual([]); + }); + it("catches an id that is configured with different key material", () => { const file = fileWrittenUnderOld(); useKeys({ old: K3, cur: K2 }, "cur"); diff --git a/src/lib/export/backup-key-ids.ts b/src/lib/export/backup-key-ids.ts index acbe75e07..aead1cb94 100644 --- a/src/lib/export/backup-key-ids.ts +++ b/src/lib/export/backup-key-ids.ts @@ -97,6 +97,25 @@ export function isEncryptedMember(name: string): boolean { return name.endsWith("Encrypted") && name.length > "Encrypted".length; } +/** + * Sections whose restore opens every inner value with this host's keys itself, + * and keeps out (and names in the skip report) any value it cannot open. + * + * The cycle day-logs and custom cycle symptoms are here because a portable + * file written before v1.40 carried their free text as the stored ciphertext, + * not as readable text. Such a file is still restorable onto another host: + * what that host can open comes back, what it cannot stays out of the row and + * is reported. Refusing the whole file over a note would cost every other + * part of the account. The trade, stated: a disaster-recovery copy whose ONLY + * missing key sits in these sections restores with those values reported as + * skipped instead of being refused; any other section needing the key still + * refuses it, which in practice is every real copy. + */ +export const RESTORE_SELF_VERIFIED_SECTIONS: ReadonlySet = new Set([ + "cycleDayLogs", + "customSymptoms", +]); + interface KeyUse { /** How many values need this key. */ count: number; @@ -127,7 +146,10 @@ export class BackupKeyIdCollector { } use.count += 1; use.sections.add(section); + // A sample from a self-verified section proves nothing the restore needs: + // a value there that does not open is skipped, not fatal. if ( + !RESTORE_SELF_VERIFIED_SECTIONS.has(section) && value.length <= MAX_SAMPLE_CHARS && (!use.sample || value.length < use.sample.value.length) ) { @@ -257,7 +279,9 @@ function openSample(sample: { value: string; form: InnerCiphertextForm }) { * key to prove the key under that id is the right one. * * `ignoreSections` leaves out sections the caller will not write: the - * instance settings, when the operator has not asked for them back. + * instance settings, when the operator has not asked for them back. The + * sections in {@link RESTORE_SELF_VERIFIED_SECTIONS} are always left out, + * because their restore checks each value itself. */ export function assessBackupKeys( collector: BackupKeyIdCollector, @@ -274,7 +298,14 @@ export function assessBackupKeys( for (const [keyId, use] of [...collector.entries()].sort(([a], [b]) => a.localeCompare(b), )) { - if ([...use.sections].every((section) => ignore.has(section))) continue; + if ( + [...use.sections].every( + (section) => + ignore.has(section) || RESTORE_SELF_VERIFIED_SECTIONS.has(section), + ) + ) { + continue; + } keyIds.push(keyId); configured ??= new Set(getConfiguredKeyIds()); if (!configured.has(keyId)) { diff --git a/src/lib/export/restore-skips.ts b/src/lib/export/restore-skips.ts index d79d09bbd..2b2ca30fc 100644 --- a/src/lib/export/restore-skips.ts +++ b/src/lib/export/restore-skips.ts @@ -152,6 +152,14 @@ * real foreign key, so the POINTER is dropped and the occurrence restores * unlinked. A portable export omits soft-deleted episodes, which is the * ordinary way an occurrence ends up naming one the file does not carry. + * + * The sixteenth, `cycleCiphertext`, is not a reference at all. It names a + * sealed cycle value (a day-log note, the sensitive-category envelope, a + * custom symptom's label) the file carried as ciphertext this host's keys do + * not open — a portable file written before v1.40 on a host with another key. + * Writing it back would store a value no reader can open, so the FIELD is kept + * out and the day or the symptom restores without it. Its key is the path in + * the file (`cycleDayLogs..notesEncrypted`), never the value. */ export type SkippedCatalogue = | "cycleSymptom" @@ -172,7 +180,8 @@ export type SkippedCatalogue = | "scheduleRevisionLink" | "checkupClosure" | "accountSetting" - | "symptomEpisodeReference"; + | "symptomEpisodeReference" + | "cycleCiphertext"; /** One key this instance does not know, and the links it cost. */ export interface SkippedCatalogueKey { diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index 8a9af3f62..ad0334bcc 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -625,6 +625,15 @@ const cycleSymptomSeveritySchema = z }) .passthrough(); +/** The sensitive-category envelope's five fields, as a portable file carries them. */ +const cycleSensitiveFieldsSchema = z.object({ + sexualActivity: z.boolean().optional(), + protectedSex: z.boolean().nullable().optional(), + pregnancyTest: z.enum(HomeTestResult).nullable().optional(), + progesteroneTest: z.enum(HomeTestResult).nullable().optional(), + contraceptive: z.enum(ContraceptiveKind).nullable().optional(), +}); + const cycleDayLogSchema = z .object({ id: z.string().min(1).optional(), @@ -644,8 +653,20 @@ const cycleDayLogSchema = z pregnancyTest: z.enum(HomeTestResult).nullable().optional(), progesteroneTest: z.enum(HomeTestResult).nullable().optional(), contraceptive: z.enum(ContraceptiveKind).nullable().optional(), + // Free text travels like every other note: a disaster-recovery file + // carries the ciphertext (`*Encrypted`), a portable file the readable + // value (`note`, `sensitive`), which the restore seals under the + // receiving host's key. A portable file written before v1.40 carries the + // ciphertext too; the restore keeps only what this host can open. sensitiveEncrypted: z.string().nullable().optional(), notesEncrypted: z.string().nullable().optional(), + note: z.string().nullable().optional(), + // The five intent fields the sensitive-category envelope holds, or the + // unreadable marker when the writing host could not open the envelope. + sensitive: z + .union([cycleSensitiveFieldsSchema, z.string()]) + .nullable() + .optional(), source: z.enum(MeasurementSource).optional(), externalId: z.string().nullable().optional(), tz: z.string().nullable().optional(), @@ -752,7 +773,10 @@ const customCycleSymptomSchema = z sortOrder: z.number().int().default(0), isActive: z.boolean().default(true), // Same reasoning as the mood tag's: the user's own words live here. + // Ciphertext on a disaster-recovery file, readable `label` on a portable + // one (the day-log note's split). labelEncrypted: z.string().nullable().optional(), + label: z.string().nullable().optional(), }) .passthrough(); diff --git a/tests/integration/admin-backups-canonical-roundtrip.test.ts b/tests/integration/admin-backups-canonical-roundtrip.test.ts index ee3015901..bb20699a6 100644 --- a/tests/integration/admin-backups-canonical-roundtrip.test.ts +++ b/tests/integration/admin-backups-canonical-roundtrip.test.ts @@ -391,8 +391,10 @@ describe("canonical disaster-recovery backup round-trip", () => { cervixFirmness: "FIRM", cervixOpening: "CLOSED", sexualActivity: false, - sensitiveEncrypted: "sensitive-cycle-dr", - notesEncrypted: "notes-cycle-dr", + // Real ciphertext: the restore writes a sealed value back only when + // this host opens it. + sensitiveEncrypted: encrypt(JSON.stringify({ sexualActivity: false })), + notesEncrypted: encrypt("notes-cycle-dr"), source: "APPLE_HEALTH", externalId: "cycle-day-external-dr", tz: "Europe/London", diff --git a/tests/integration/backup-cycle-cross-key-restore.test.ts b/tests/integration/backup-cycle-cross-key-restore.test.ts new file mode 100644 index 000000000..4d3f50aee --- /dev/null +++ b/tests/integration/backup-cycle-cross-key-restore.test.ts @@ -0,0 +1,313 @@ +/** + * A portable file's cycle free text survives a move to a host with other keys. + * + * The day-log note, the sensitive-category envelope and a custom symptom's + * label are sealed under the writing host's key. A portable file used to carry + * those three as the stored ciphertext, so a restore onto a host with a + * different key either refused the whole file at the key check or, with a + * matching key id over different key material, wrote rows nobody could open. + * + * Both ends and the pipe: the file is built by the real `buildFullBackupPayload` + * under key A, stored and restored through the real route under key B, and + * the restored rows are opened under key B. + * + * The second case is the file written before the fix, which still carries + * ciphertext. A value this host's keys open comes back; a value they cannot + * open is kept out of the row and named in the skip report, rather than + * written back as ciphertext no reader can open. + * + * Mutation: restore the verbatim copy of `notesEncrypted` in + * `restoreCycleData` and the legacy case writes the foreign ciphertext back. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { _resetCryptoCacheForTests, decrypt, encrypt } from "@/lib/crypto"; +import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { parseBackupPayload } from "@/lib/validations/backup"; +import { POST } from "./restore-job-driver"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserData: vi.fn(), +})); + +const saved = { + key: process.env.ENCRYPTION_KEY, + keys: process.env.ENCRYPTION_KEYS, + active: process.env.ENCRYPTION_ACTIVE_KEY_ID, +}; + +/** Make this process the host holding exactly `keys`, writing under `active`. */ +function useKeys(keys: Record, active: string) { + process.env.ENCRYPTION_KEY = ""; + process.env.ENCRYPTION_KEYS = JSON.stringify(keys); + process.env.ENCRYPTION_ACTIVE_KEY_ID = active; + _resetCryptoCacheForTests(); +} + +const HOST_A = { a: "11".repeat(32) }; +const HOST_B = { b: "22".repeat(32) }; +/** Same key id as host B, different key material. */ +const HOST_B_IMPOSTOR = { b: "33".repeat(32) }; + +const NOTE = "Cramps woke me at four"; +const SENSITIVE = { + sexualActivity: true, + protectedSex: false, + pregnancyTest: "NEGATIVE", + progesteroneTest: null, + contraceptive: "ORAL", +}; +const LABEL = "Lower back ache"; + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +afterEach(() => { + process.env.ENCRYPTION_KEY = saved.key ?? ""; + process.env.ENCRYPTION_KEYS = saved.keys ?? ""; + process.env.ENCRYPTION_ACTIVE_KEY_ID = saved.active ?? ""; + _resetCryptoCacheForTests(); +}); + +function makeRequest(id: string) { + return new Request(`http://localhost/api/admin/backups/${id}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }); +} + +async function seedOwner() { + const prisma = getPrismaClient(); + const owner = await prisma.user.create({ + data: { + username: "cycle-cross-key-owner", + email: "cycle-cross-key-owner@example.test", + role: "ADMIN", + }, + }); + const session = await prisma.session.create({ + data: { userId: owner.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return owner; +} + +/** Store `payload` as a backup on this host and run the real restore. */ +async function restore(ownerId: string, payload: unknown) { + const prisma = getPrismaClient(); + const backup = await prisma.dataBackup.create({ + data: { + userId: ownerId, + type: "CYCLE_CROSS_KEY_RESTORE", + data: encrypt(JSON.stringify(payload)), + }, + }); + return POST(makeRequest(backup.id) as unknown as Parameters[0], { + params: Promise.resolve({ id: backup.id }), + }); +} + +describe("cycle free text in a portable file restored under another key", () => { + it("comes back readable under the receiving host's key", async () => { + const prisma = getPrismaClient(); + const owner = await seedOwner(); + + /* ── host A writes the account and exports it ─────────────────────── */ + useKeys(HOST_A, "a"); + const category = await prisma.cycleSymptomCategory.findFirstOrThrow(); + await prisma.cycleSymptom.create({ + data: { + userId: owner.id, + categoryId: category.id, + key: "custom:cross-key-ache", + labelKey: "cycle.symptom.custom", + labelEncrypted: encrypt(LABEL), + }, + }); + await prisma.cycleDayLog.create({ + data: { + userId: owner.id, + date: "2026-06-02", + flow: "MEDIUM", + notesEncrypted: encrypt(NOTE), + sensitiveEncrypted: encrypt(JSON.stringify(SENSITIVE)), + }, + }); + const { payload: built } = await buildFullBackupPayload(prisma, owner.id); + const file = JSON.parse(JSON.stringify(built)) as unknown; + + /* ── host B, which has never held key A, restores it ──────────────── */ + useKeys(HOST_B, "b"); + // Host B is another database: the account's own symptom row from host A + // is not there (the restore upserts by key and keeps an existing row). + await prisma.cycleDayLog.deleteMany({ where: { userId: owner.id } }); + await prisma.cycleSymptom.deleteMany({ where: { userId: owner.id } }); + const response = await restore(owner.id, file); + expect(response.status).toBe(200); + + const day = await prisma.cycleDayLog.findFirstOrThrow({ + where: { userId: owner.id }, + }); + expect(day.flow).toBe("MEDIUM"); + expect(day.notesEncrypted).not.toBeNull(); + expect(decrypt(day.notesEncrypted!)).toBe(NOTE); + expect(day.sensitiveEncrypted).not.toBeNull(); + expect(JSON.parse(decrypt(day.sensitiveEncrypted!))).toEqual(SENSITIVE); + + const symptom = await prisma.cycleSymptom.findUniqueOrThrow({ + where: { key: "custom:cross-key-ache" }, + }); + expect(decrypt(symptom.labelEncrypted!)).toBe(LABEL); + + const body = (await response.json()) as { + data: { skipped: { links: number } }; + }; + expect(body.data.skipped.links).toBe(0); + }); + + it("restores a legacy file's ciphertext only where this host can open it", async () => { + const prisma = getPrismaClient(); + const owner = await seedOwner(); + + // Written under host B's own key: opens here. + useKeys(HOST_B, "b"); + const ownNote = encrypt(NOTE); + // Written elsewhere: an id host B does not hold, and host B's id over + // different key material. + useKeys(HOST_A, "a"); + const foreignNote = encrypt("written under key a"); + const foreignLabel = encrypt("label under key a"); + useKeys(HOST_B_IMPOSTOR, "b"); + const impostorSensitive = encrypt(JSON.stringify(SENSITIVE)); + + // The shape a portable file had before the fix: every free-text field as + // the stored ciphertext, no plaintext beside it. + useKeys(HOST_B, "b"); + const { payload: built } = await buildFullBackupPayload(prisma, owner.id); + const file = JSON.parse(JSON.stringify(built)) as Record; + const category = await prisma.cycleSymptomCategory.findFirstOrThrow(); + file.customSymptoms = [ + { + key: "custom:legacy-foreign", + labelKey: "cycle.symptom.custom", + categoryId: category.id, + icon: null, + sortOrder: 0, + isActive: true, + labelEncrypted: foreignLabel, + }, + ]; + const day = ( + date: string, + notes: string | null, + sensitive: string | null, + ) => ({ + date, + flow: "LIGHT", + intermenstrualBleeding: false, + basalBodyTempC: null, + temperatureExcluded: false, + ovulationTest: null, + cervicalMucus: null, + cervixPosition: null, + cervixFirmness: null, + cervixOpening: null, + sexualActivity: false, + protectedSex: null, + pregnancyTest: null, + progesteroneTest: null, + contraceptive: null, + sensitiveEncrypted: sensitive, + notesEncrypted: notes, + source: "MANUAL", + externalId: null, + tz: null, + symptomKeys: [], + symptomSeverities: [], + }); + file.cycleDayLogs = [ + day("2026-06-01", ownNote, null), + day("2026-06-02", foreignNote, impostorSensitive), + ]; + // Still a file this release accepts. + parseBackupPayload(JSON.stringify(file)); + + const response = await restore(owner.id, file); + expect(response.status).toBe(200); + + const days = await prisma.cycleDayLog.findMany({ + where: { userId: owner.id }, + orderBy: { date: "asc" }, + }); + expect(days.map((d) => d.date)).toEqual(["2026-06-01", "2026-06-02"]); + expect(decrypt(days[0].notesEncrypted!)).toBe(NOTE); + // The day itself comes back; only what cannot be opened stays out. + expect(days[1].flow).toBe("LIGHT"); + expect(days[1].notesEncrypted).toBeNull(); + expect(days[1].sensitiveEncrypted).toBeNull(); + + const symptom = await prisma.cycleSymptom.findUniqueOrThrow({ + where: { key: "custom:legacy-foreign" }, + }); + expect(symptom.labelEncrypted).toBeNull(); + + const body = (await response.json()) as { + data: { + skipped: { + links: number; + catalogueKeys: Array<{ + catalogue: string; + key: string; + links: number; + }>; + }; + }; + }; + expect(body.data.skipped.catalogueKeys).toEqual([ + { + catalogue: "cycleCiphertext", + key: "customSymptoms.custom:legacy-foreign.labelEncrypted", + links: 1, + }, + { + catalogue: "cycleCiphertext", + key: "cycleDayLogs.2026-06-02.notesEncrypted", + links: 1, + }, + { + catalogue: "cycleCiphertext", + key: "cycleDayLogs.2026-06-02.sensitiveEncrypted", + links: 1, + }, + ]); + expect(body.data.skipped.links).toBe(3); + }); +}); diff --git a/tests/integration/cycle-import-export.test.ts b/tests/integration/cycle-import-export.test.ts index 73eed32e2..a6f3014b3 100644 --- a/tests/integration/cycle-import-export.test.ts +++ b/tests/integration/cycle-import-export.test.ts @@ -6,8 +6,8 @@ * into CYCLE day-logs (NOT Measurement), gated on cycle-tracking enabled, * merging same-day samples into one row with first-write-wins re-import * - the full-backup cycle section round-trips through - * `buildCycleBackupSection` → `restoreCycleData` with `notesEncrypted` - * preserved verbatim and symptom links re-resolved by key + * `buildCycleBackupSection` → `restoreCycleData` with the note carried + * readable, re-sealed on restore, and symptom links re-resolved by key */ import { writeFileSync, mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -17,6 +17,7 @@ import { beforeEach, describe, expect, it } from "vitest"; import { getPrismaClient, truncateAllTables } from "./setup"; import { streamParseExportXml } from "@/lib/measurements/import-apple-health-export"; import { buildCycleBackupSection, restoreCycleData } from "@/lib/cycle/backup"; +import { decrypt, encrypt } from "@/lib/crypto"; import type { RestoreSkipLog } from "@/lib/export/restore-skips"; import { parseBackupPayload } from "@/lib/validations/backup"; @@ -163,7 +164,7 @@ describe("cycle import — reproductive HK samples route to CycleDayLog", () => }); describe("cycle backup — round-trips through build + restore", () => { - it("preserves notesEncrypted ciphertext and symptom links by key", async () => { + it("carries the note readable and re-seals it, with symptom links by key", async () => { const prisma = getPrismaClient(); const cycle = await prisma.menstrualCycle.create({ @@ -184,7 +185,7 @@ describe("cycle backup — round-trips through build + restore", () => { cervixPosition: "HIGH", cervixFirmness: "SOFT", cervixOpening: "OPEN", - notesEncrypted: "v1:abc123:cipher-envelope", + notesEncrypted: encrypt("Cramps in the evening"), source: "MANUAL", }, }); @@ -207,9 +208,9 @@ describe("cycle backup — round-trips through build + restore", () => { const section = await buildCycleBackupSection(prisma, USER_ID); expect(section.cycles).toHaveLength(1); expect(section.cycleDayLogs).toHaveLength(1); - expect(section.cycleDayLogs[0].notesEncrypted).toBe( - "v1:abc123:cipher-envelope", - ); + // A portable section carries the note readable, never the ciphertext. + expect(section.cycleDayLogs[0].note).toBe("Cramps in the evening"); + expect(section.cycleDayLogs[0]).not.toHaveProperty("notesEncrypted"); expect(section.cycleDayLogs[0].symptomKeys).toEqual(["cramps"]); const payload = parseBackupPayload({ @@ -234,7 +235,7 @@ describe("cycle backup — round-trips through build + restore", () => { include: { symptomLinks: { include: { symptom: true } } }, }); expect(restored).toHaveLength(1); - expect(restored[0].notesEncrypted).toBe("v1:abc123:cipher-envelope"); + expect(decrypt(restored[0].notesEncrypted!)).toBe("Cramps in the evening"); expect(restored[0].cervixPosition).toBe("HIGH"); expect(restored[0].cervixFirmness).toBe("SOFT"); expect(restored[0].cervixOpening).toBe("OPEN"); From 6bc764fa289d49a39f8c1602193f3a9af2f0c8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 08:17:34 +0200 Subject: [PATCH 03/21] Write a medication PUT and its course in one transaction The row update (name, schedule, oneShot) committed before the course write. When a concurrent course made the course write refuse, the client got the refusal while the rest was already written. The schedule replace, the row update, the pause era and the course write now commit together, so a refusal leaves the medication unchanged. The rollup recompute runs after the commit. --- .../medications/[id]/__tests__/route.test.ts | 12 +- src/app/api/medications/[id]/route.ts | 691 +++++++++--------- src/lib/medications/courses.ts | 100 +-- .../medication-put-course-race.test.ts | 152 ++++ 4 files changed, 572 insertions(+), 383 deletions(-) create mode 100644 tests/integration/medication-put-course-race.test.ts diff --git a/src/app/api/medications/[id]/__tests__/route.test.ts b/src/app/api/medications/[id]/__tests__/route.test.ts index 5545404eb..67d9fe8f5 100644 --- a/src/app/api/medications/[id]/__tests__/route.test.ts +++ b/src/app/api/medications/[id]/__tests__/route.test.ts @@ -114,6 +114,11 @@ const ROUTE_CTX = { params: Promise.resolve({ id: "m1" }) }; beforeEach(() => { vi.resetAllMocks(); vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + // The PUT writes inside one interactive transaction; run its callback + // against the same mocked client. + vi.mocked(prisma.$transaction).mockImplementation((async ( + fn: (tx: typeof prisma) => unknown, + ) => fn(prisma)) as never); // v1.40 — the category and course helpers' defaults, which // `resetAllMocks` clears. vi.mocked(resolveCourseFields).mockResolvedValue(new Map()); @@ -203,9 +208,9 @@ describe("PUT /api/medications/[id] — v1.5 scheduling primitives", () => { vi.mocked(auditLog).mockResolvedValue(undefined); const res = await PUT(putReq({ endsOn: "2026-12-31" }), ROUTE_CTX); expect(res.status).toBe(200); - // v1.40 (#1024) — the window goes through the course writer, which - // writes the row and the latest course in one transaction; the plain - // row update no longer carries it. + // v1.40 (#1024) — the window goes through the course writer, inside the + // same transaction as the row update; the plain row update no longer + // carries it. const call = lastUpdateCall(); expect(call.data).not.toHaveProperty("endsOn"); expect(setCurrentWindow).toHaveBeenCalledWith( @@ -213,6 +218,7 @@ describe("PUT /api/medications/[id] — v1.5 scheduling primitives", () => { medicationId: "m1", endsOn: new Date("2026-12-31"), }), + prisma, ); }); diff --git a/src/app/api/medications/[id]/route.ts b/src/app/api/medications/[id]/route.ts index c30e57891..f7e62e198 100644 --- a/src/app/api/medications/[id]/route.ts +++ b/src/app/api/medications/[id]/route.ts @@ -667,7 +667,12 @@ export const PUT = apiHandler( // Tombstone today's and future open pending rows of this medication. // Runs on a schedule replace (the old anchors are stale) and when intake // tracking is switched off (nothing is due any more). - const tombstoneOpenSlots = async (): Promise => { + // The rollup days the tombstoned slots sat on, recomputed once the + // transaction below has committed the tombstones. + const staleRollupDays = new Set(); + const tombstoneOpenSlots = async ( + db: Prisma.TransactionClient, + ): Promise => { // A schedule replace invalidates the open slot anchors the projector / // reminder worker minted for the OLD times: a pending 08:00 row for a // medication that now doses at 20:00 would linger as a phantom slot, @@ -689,12 +694,12 @@ export const PUT = apiHandler( scheduledFor: { gte: todayStart }, } as const; // Capture the affected slots BEFORE the tombstone so the compliance - // rollups for those days can be recomputed immediately below — + // rollups for those days can be recomputed after the commit — // `updateMany` does not return the touched rows. Without the // recompute a tombstoned pending kept counting as `scheduled` in // the rollup row until the next event-driven write for that day, // depressing the rate the cards read in the meantime. - const tombstoned = await prisma.medicationIntakeEvent.findMany({ + const tombstoned = await db.medicationIntakeEvent.findMany({ where: tombstoneWhere, select: { scheduledFor: true }, }); @@ -708,332 +713,369 @@ export const PUT = apiHandler( tombstonedTo = row.scheduledFor; } } - await prisma.medicationIntakeEvent.updateMany({ + await db.medicationIntakeEvent.updateMany({ where: tombstoneWhere, data: { deletedAt: new Date(), syncVersion: { increment: 1 } }, }); - // Recompute each affected rollup day (deduped via the user-tz day - // key). Best-effort like every other rollup write-hook: a failure - // is annotated, never blocks the schedule replace. - const staleDayKeys = new Set( - tombstoned.map((row) => - dayKeyForScheduledFor(row.scheduledFor, userTz), - ), - ); - for (const dayKey of staleDayKeys) { - try { - await recomputeMedicationComplianceForDay( - user.id, - id, - dayKey, - userTz, + // Deduped via the user-tz day key. + for (const row of tombstoned) { + staleRollupDays.add(dayKeyForScheduledFor(row.scheduledFor, userTz)); + } + }; + + // v1.40 — everything the PUT writes to the medication, its schedule and + // its course commits as one transaction. The course write can still be + // refused after the check above (another request wrote a course in + // between); with the row update committed first, the client got the + // refusal while the name, the schedule and `oneShot` were already + // changed. Now a refusal rolls all of it back. The rollup recompute and + // the grace bridge stay outside: both read or write what this commit + // settles, and neither may undo it. + let medication; + try { + medication = await prisma.$transaction(async (tx) => { + // If schedules provided, replace all + if (schedules && normalisedSchedules) { + // v1.16.3 — effective dating. Before the wholesale replace below wipes + // the old rows, archive them as ONE revision covering + // `[previous validUntil | medication.createdAt, now)` — but only when a + // cadence-relevant field actually changes. A no-op edit (the Zeitplan + // tab echoing the same rows back) must not mint a phantom era. + const previousRows = await tx.medicationSchedule.findMany({ + where: { medicationId: id }, + }); + const previousEntries = previousRows.map((row) => + toRevisionPayloadEntry({ + timesOfDay: row.timesOfDay, + windowStart: row.windowStart, + windowEnd: row.windowEnd, + daysOfWeek: row.daysOfWeek, + rrule: row.rrule, + rollingIntervalDays: row.rollingIntervalDays, + scheduleType: row.scheduleType, + cyclicOnWeeks: row.cyclicOnWeeks, + cyclicOffWeeks: row.cyclicOffWeeks, + doseWindows: row.doseWindows, + label: row.label, + dose: row.dose, + reminderGraceMinutes: row.reminderGraceMinutes, + }), ); - } catch (err) { - annotate({ - meta: { - medication_compliance_rollup_failed: true, - medication_compliance_rollup_error: - err instanceof Error ? err.message : String(err), + replacedScheduleCount = previousRows.length; + replacedCadence = + previousRows.length > 0 + ? previousRows + .map((row) => + [ + row.timesOfDay, + row.rrule ?? "", + row.rollingIntervalDays !== null + ? `every ${row.rollingIntervalDays}d` + : "", + row.daysOfWeek ?? "", + ] + .filter((part) => part !== "") + .join(" "), + ) + .join(" | ") + : null; + + // v1.39.1 (#1033) — a stretch with intake tracking off expected + // nothing, so it archives with an empty payload whatever the rows + // said; and a tracking transition is itself an era boundary even when + // the rows stay the same. + const archivedEntries = wasTracked ? previousEntries : []; + if ( + previousRows.length > 0 && + (trackingChanged || + !wasTracked || + schedulesMateriallyDiffer( + previousEntries, + normalisedSchedules.map((n) => n.snapshot), + )) + ) { + const lastRevision = await tx.medicationScheduleRevision.findFirst({ + // Chain from the ACTIVE boundary: a superseded row is an + // audit record whose `validUntil` may sit past its correction. + where: { medicationId: id, supersededByRevisionId: null }, + orderBy: { validUntil: "desc" }, + select: { validUntil: true }, + }); + await tx.medicationScheduleRevision.create({ + data: { + medicationId: id, + validFrom: lastRevision?.validUntil ?? existing.createdAt, + validUntil: new Date(), + payload: archivedEntries as unknown as Prisma.InputJsonValue, + }, + }); + annotate({ + action: { + name: "medication.schedule.revision_archived", + entity_type: "medication", + entity_id: id, + }, + meta: { schedule_revision_rows: archivedEntries.length }, + }); + } else if ( + previousRows.length === 0 && + existing.asNeeded && + asNeeded === false && + normalisedSchedules.length > 0 + ) { + // v1.16.11 — flipping as-needed OFF. The medication carried ZERO + // schedule rows while as-needed, so the wholesale-replace archive + // above never fires — and without a revision the live era would + // start at the PREVIOUS revision's `validUntil` (or `createdAt`), + // retro-painting the schedule-less as-needed stretch with the NEW + // schedule's expected slots: every PRN day would read as missed. + // Archive an EMPTY era covering the as-needed stretch instead; an + // empty payload expands to zero schedules, so era-aware compliance + // expects nothing there — exactly the as-needed contract. + const lastRevision = await tx.medicationScheduleRevision.findFirst({ + where: { medicationId: id, supersededByRevisionId: null }, + orderBy: { validUntil: "desc" }, + select: { validUntil: true }, + }); + await tx.medicationScheduleRevision.create({ + data: { + medicationId: id, + validFrom: lastRevision?.validUntil ?? existing.createdAt, + validUntil: new Date(), + payload: [] as unknown as Prisma.InputJsonValue, + }, + }); + annotate({ + action: { + name: "medication.schedule.revision_archived", + entity_type: "medication", + entity_id: id, + }, + meta: { schedule_revision_rows: 0 }, + }); + } + await tombstoneOpenSlots(tx); + await tx.medicationSchedule.deleteMany({ + where: { medicationId: id }, + }); + } else if (trackingChanged) { + // v1.39.1 (#1033) — intake tracking switched without a schedule + // replace. Close the current era at this instant: switching OFF + // archives the schedule that was in force (the tracked stretch keeps + // its expected doses); switching back ON archives the untracked + // stretch with an empty payload, so compliance never counts it as + // missed and the live schedule, the projector and the reminder worker + // all resume from now. + const liveRows = await tx.medicationSchedule.findMany({ + where: { medicationId: id }, + }); + if (liveRows.length > 0) { + const lastRevision = await tx.medicationScheduleRevision.findFirst({ + where: { medicationId: id, supersededByRevisionId: null }, + orderBy: { validUntil: "desc" }, + select: { validUntil: true }, + }); + const archivedEntries = wasTracked + ? liveRows.map((row) => + toRevisionPayloadEntry({ + timesOfDay: row.timesOfDay, + windowStart: row.windowStart, + windowEnd: row.windowEnd, + daysOfWeek: row.daysOfWeek, + rrule: row.rrule, + rollingIntervalDays: row.rollingIntervalDays, + scheduleType: row.scheduleType, + cyclicOnWeeks: row.cyclicOnWeeks, + cyclicOffWeeks: row.cyclicOffWeeks, + doseWindows: row.doseWindows, + label: row.label, + dose: row.dose, + reminderGraceMinutes: row.reminderGraceMinutes, + }), + ) + : []; + await tx.medicationScheduleRevision.create({ + data: { + medicationId: id, + validFrom: lastRevision?.validUntil ?? existing.createdAt, + validUntil: new Date(), + payload: archivedEntries as unknown as Prisma.InputJsonValue, + }, + }); + annotate({ + action: { + name: "medication.schedule.revision_archived", + entity_type: "medication", + entity_id: id, + }, + meta: { + schedule_revision_rows: archivedEntries.length, + track_intake: willTrack, + }, + }); + } + } + + // v1.39.1 (#1033) — switching intake tracking off leaves nothing due: + // the open placeholders of today and later go the way a schedule + // replace sends them (a replace in the same request already did it). + if (trackingChanged && !willTrack && !schedules) { + await tombstoneOpenSlots(tx); + } + + const baseUpdateData = { + ...(name !== undefined && { name }), + ...(dose !== undefined && { dose }), + ...(treatmentClass !== undefined && { treatmentClass }), + ...(dosesPerUnit !== undefined && { dosesPerUnit }), + // v1.16.10 — units consumed per dose. Field-by-field; the stamp on + // already-taken intake events freezes their recorded consumption. + ...(unitsPerDose !== undefined && { unitsPerDose }), + // v1.17.0 — reorder lead override. Field-by-field; `null` is a + // valid explicit value (clears the override → user-level default). + ...(reorderLeadDays !== undefined && { reorderLeadDays }), + ...(deliveryForm !== undefined && { deliveryForm }), + // v1.8.5 — injection-site tracking opt-in + per-medication allowed + // sites. Field-by-field; `false` / `[]` are valid explicit values + // (deactivate tracking / clear the per-med restriction). + ...(trackInjectionSites !== undefined && { trackInjectionSites }), + ...(allowedInjectionSites !== undefined && { allowedInjectionSites }), + ...(active !== undefined && { active }), + ...(notificationsEnabled !== undefined && { notificationsEnabled }), + // v1.7.0 — iOS reminder flags, field-by-field. + ...(liveActivityEnabled !== undefined && { liveActivityEnabled }), + ...(criticalAlarmEnabled !== undefined && { criticalAlarmEnabled }), + // v1.9.0 — optional drug-classification codes (ATC / RxNorm). + // Field-by-field; `null` is a valid explicit value (clears the code). + ...(atcCode !== undefined && { atcCode }), + ...(rxNormCode !== undefined && { rxNormCode }), + // v1.5 scheduling primitives — pass-through when supplied. + // `startsOn` / `endsOn` are `Date | null | undefined` (the + // schema lets the user clear them explicitly with null). + // v1.40 (#1024) — `startsOn` / `endsOn` are not written here: the + // course writer below writes them together with the course they + // project, in one transaction, so the row and its courses cannot + // disagree if one of the two writes fails. + ...(oneShot !== undefined && { oneShot }), + // v1.16.11 — as-needed flag, field-by-field (the invariants above + // already guaranteed the medication ends schedule-consistent). + ...(asNeeded !== undefined && { asNeeded }), + // v1.39.1 (#1033) — intake tracking, field-by-field. + ...(trackIntake !== undefined && { trackIntake }), + ...(normalisedSchedules && { + schedules: { + create: normalisedSchedules.map((n) => n.createData), + }, + }), + }; + + const withoutNotifications = { ...baseUpdateData } as Record< + string, + unknown + >; + delete withoutNotifications.notificationsEnabled; + const hasPausedAtPatch = Object.keys(pausedAtPatch).length > 0; + const hasNotificationsPatch = notificationsEnabled !== undefined; + + const updateCandidates: Array> = [ + { ...baseUpdateData, ...pausedAtPatch }, + ]; + if (hasPausedAtPatch) { + updateCandidates.push(baseUpdateData); + } + if (hasNotificationsPatch) { + updateCandidates.push({ ...withoutNotifications, ...pausedAtPatch }); + if (hasPausedAtPatch) { + updateCandidates.push(withoutNotifications); + } + } + + let updated; + let lastUpdateErr: unknown; + for (const candidate of updateCandidates) { + try { + updated = await tx.medication.update({ + where: { id }, + data: candidate, + include: { schedules: true }, + }); + break; + } catch (updateErr) { + lastUpdateErr = updateErr; + } + } + if (!updated) throw lastUpdateErr; + + // v1.25 H-MED1 — durable pause intervals. The `pausedAt` column is a + // single live marker that resume clears, so the paused window is + // irrecoverable once resumed and every paused day's slot collapses to + // "missed" in the compliance denominator. Mirror the live transition + // onto an additive era record the rate paths read: a pause opens an era + // (resumedAt null), a resume closes the latest open era. `userId` comes + // from the authenticated session; the data object is built field-by-field. + if (active === false && existing.active) { + await tx.medicationPauseEra.create({ + data: { + medicationId: id, + userId: user.id, + pausedAt: new Date(), + resumedAt: null, }, }); + } else if (active === true && !existing.active) { + await tx.medicationPauseEra.updateMany({ + where: { medicationId: id, userId: user.id, resumedAt: null }, + data: { resumedAt: new Date() }, + }); } - } - }; - // If schedules provided, replace all - if (schedules && normalisedSchedules) { - // v1.16.3 — effective dating. Before the wholesale replace below wipes - // the old rows, archive them as ONE revision covering - // `[previous validUntil | medication.createdAt, now)` — but only when a - // cadence-relevant field actually changes. A no-op edit (the Zeitplan - // tab echoing the same rows back) must not mint a phantom era. - const previousRows = await prisma.medicationSchedule.findMany({ - where: { medicationId: id }, + // v1.40 (#1024) — write the window onto the latest course and + // project it back, then serve the projected row. + if (touchesWindow) { + const projected = await setCurrentWindow( + { + userId: user.id, + medicationId: id, + timeZone: courseTz, + ...nextWindow, + }, + tx, + ); + return { ...updated, ...projected }; + } + return updated; }); - const previousEntries = previousRows.map((row) => - toRevisionPayloadEntry({ - timesOfDay: row.timesOfDay, - windowStart: row.windowStart, - windowEnd: row.windowEnd, - daysOfWeek: row.daysOfWeek, - rrule: row.rrule, - rollingIntervalDays: row.rollingIntervalDays, - scheduleType: row.scheduleType, - cyclicOnWeeks: row.cyclicOnWeeks, - cyclicOffWeeks: row.cyclicOffWeeks, - doseWindows: row.doseWindows, - label: row.label, - dose: row.dose, - reminderGraceMinutes: row.reminderGraceMinutes, - }), - ); - replacedScheduleCount = previousRows.length; - replacedCadence = - previousRows.length > 0 - ? previousRows - .map((row) => - [ - row.timesOfDay, - row.rrule ?? "", - row.rollingIntervalDays !== null - ? `every ${row.rollingIntervalDays}d` - : "", - row.daysOfWeek ?? "", - ] - .filter((part) => part !== "") - .join(" "), - ) - .join(" | ") - : null; - - // v1.39.1 (#1033) — a stretch with intake tracking off expected - // nothing, so it archives with an empty payload whatever the rows - // said; and a tracking transition is itself an era boundary even when - // the rows stay the same. - const archivedEntries = wasTracked ? previousEntries : []; - if ( - previousRows.length > 0 && - (trackingChanged || - !wasTracked || - schedulesMateriallyDiffer( - previousEntries, - normalisedSchedules.map((n) => n.snapshot), - )) - ) { - const lastRevision = await prisma.medicationScheduleRevision.findFirst({ - // Chain from the ACTIVE boundary: a superseded row is an - // audit record whose `validUntil` may sit past its correction. - where: { medicationId: id, supersededByRevisionId: null }, - orderBy: { validUntil: "desc" }, - select: { validUntil: true }, - }); - await prisma.medicationScheduleRevision.create({ - data: { - medicationId: id, - validFrom: lastRevision?.validUntil ?? existing.createdAt, - validUntil: new Date(), - payload: archivedEntries as unknown as Prisma.InputJsonValue, - }, - }); - annotate({ - action: { - name: "medication.schedule.revision_archived", - entity_type: "medication", - entity_id: id, - }, - meta: { schedule_revision_rows: archivedEntries.length }, - }); - } else if ( - previousRows.length === 0 && - existing.asNeeded && - asNeeded === false && - normalisedSchedules.length > 0 - ) { - // v1.16.11 — flipping as-needed OFF. The medication carried ZERO - // schedule rows while as-needed, so the wholesale-replace archive - // above never fires — and without a revision the live era would - // start at the PREVIOUS revision's `validUntil` (or `createdAt`), - // retro-painting the schedule-less as-needed stretch with the NEW - // schedule's expected slots: every PRN day would read as missed. - // Archive an EMPTY era covering the as-needed stretch instead; an - // empty payload expands to zero schedules, so era-aware compliance - // expects nothing there — exactly the as-needed contract. - const lastRevision = await prisma.medicationScheduleRevision.findFirst({ - where: { medicationId: id, supersededByRevisionId: null }, - orderBy: { validUntil: "desc" }, - select: { validUntil: true }, - }); - await prisma.medicationScheduleRevision.create({ - data: { - medicationId: id, - validFrom: lastRevision?.validUntil ?? existing.createdAt, - validUntil: new Date(), - payload: [] as unknown as Prisma.InputJsonValue, - }, - }); - annotate({ - action: { - name: "medication.schedule.revision_archived", - entity_type: "medication", - entity_id: id, - }, - meta: { schedule_revision_rows: 0 }, - }); - } - await tombstoneOpenSlots(); - await prisma.medicationSchedule.deleteMany({ - where: { medicationId: id }, - }); - } else if (trackingChanged) { - // v1.39.1 (#1033) — intake tracking switched without a schedule - // replace. Close the current era at this instant: switching OFF - // archives the schedule that was in force (the tracked stretch keeps - // its expected doses); switching back ON archives the untracked - // stretch with an empty payload, so compliance never counts it as - // missed and the live schedule, the projector and the reminder worker - // all resume from now. - const liveRows = await prisma.medicationSchedule.findMany({ - where: { medicationId: id }, - }); - if (liveRows.length > 0) { - const lastRevision = await prisma.medicationScheduleRevision.findFirst({ - where: { medicationId: id, supersededByRevisionId: null }, - orderBy: { validUntil: "desc" }, - select: { validUntil: true }, - }); - const archivedEntries = wasTracked - ? liveRows.map((row) => - toRevisionPayloadEntry({ - timesOfDay: row.timesOfDay, - windowStart: row.windowStart, - windowEnd: row.windowEnd, - daysOfWeek: row.daysOfWeek, - rrule: row.rrule, - rollingIntervalDays: row.rollingIntervalDays, - scheduleType: row.scheduleType, - cyclicOnWeeks: row.cyclicOnWeeks, - cyclicOffWeeks: row.cyclicOffWeeks, - doseWindows: row.doseWindows, - label: row.label, - dose: row.dose, - reminderGraceMinutes: row.reminderGraceMinutes, - }), - ) - : []; - await prisma.medicationScheduleRevision.create({ - data: { - medicationId: id, - validFrom: lastRevision?.validUntil ?? existing.createdAt, - validUntil: new Date(), - payload: archivedEntries as unknown as Prisma.InputJsonValue, - }, - }); - annotate({ - action: { - name: "medication.schedule.revision_archived", - entity_type: "medication", - entity_id: id, - }, - meta: { - schedule_revision_rows: archivedEntries.length, - track_intake: willTrack, - }, - }); + } catch (err) { + // A concurrent course write landed between the check above and the + // course write. The transaction rolled back: nothing was written. + if (err instanceof CourseWriteError) { + const r = courseRefusalResponse(err.refusal); + return apiError(r.message, r.status, { errorCode: r.errorCode }); } + throw err; } - // v1.39.1 (#1033) — switching intake tracking off leaves nothing due: - // the open placeholders of today and later go the way a schedule - // replace sends them (a replace in the same request already did it). - if (trackingChanged && !willTrack && !schedules) { - await tombstoneOpenSlots(); - } - - const baseUpdateData = { - ...(name !== undefined && { name }), - ...(dose !== undefined && { dose }), - ...(treatmentClass !== undefined && { treatmentClass }), - ...(dosesPerUnit !== undefined && { dosesPerUnit }), - // v1.16.10 — units consumed per dose. Field-by-field; the stamp on - // already-taken intake events freezes their recorded consumption. - ...(unitsPerDose !== undefined && { unitsPerDose }), - // v1.17.0 — reorder lead override. Field-by-field; `null` is a - // valid explicit value (clears the override → user-level default). - ...(reorderLeadDays !== undefined && { reorderLeadDays }), - ...(deliveryForm !== undefined && { deliveryForm }), - // v1.8.5 — injection-site tracking opt-in + per-medication allowed - // sites. Field-by-field; `false` / `[]` are valid explicit values - // (deactivate tracking / clear the per-med restriction). - ...(trackInjectionSites !== undefined && { trackInjectionSites }), - ...(allowedInjectionSites !== undefined && { allowedInjectionSites }), - ...(active !== undefined && { active }), - ...(notificationsEnabled !== undefined && { notificationsEnabled }), - // v1.7.0 — iOS reminder flags, field-by-field. - ...(liveActivityEnabled !== undefined && { liveActivityEnabled }), - ...(criticalAlarmEnabled !== undefined && { criticalAlarmEnabled }), - // v1.9.0 — optional drug-classification codes (ATC / RxNorm). - // Field-by-field; `null` is a valid explicit value (clears the code). - ...(atcCode !== undefined && { atcCode }), - ...(rxNormCode !== undefined && { rxNormCode }), - // v1.5 scheduling primitives — pass-through when supplied. - // `startsOn` / `endsOn` are `Date | null | undefined` (the - // schema lets the user clear them explicitly with null). - // v1.40 (#1024) — `startsOn` / `endsOn` are not written here: the - // course writer below writes them together with the course they - // project, in one transaction, so the row and its courses cannot - // disagree if one of the two writes fails. - ...(oneShot !== undefined && { oneShot }), - // v1.16.11 — as-needed flag, field-by-field (the invariants above - // already guaranteed the medication ends schedule-consistent). - ...(asNeeded !== undefined && { asNeeded }), - // v1.39.1 (#1033) — intake tracking, field-by-field. - ...(trackIntake !== undefined && { trackIntake }), - ...(normalisedSchedules && { - schedules: { - create: normalisedSchedules.map((n) => n.createData), - }, - }), - }; - - const withoutNotifications = { ...baseUpdateData } as Record< - string, - unknown - >; - delete withoutNotifications.notificationsEnabled; - const hasPausedAtPatch = Object.keys(pausedAtPatch).length > 0; - const hasNotificationsPatch = notificationsEnabled !== undefined; - - const updateCandidates: Array> = [ - { ...baseUpdateData, ...pausedAtPatch }, - ]; - if (hasPausedAtPatch) { - updateCandidates.push(baseUpdateData); - } - if (hasNotificationsPatch) { - updateCandidates.push({ ...withoutNotifications, ...pausedAtPatch }); - if (hasPausedAtPatch) { - updateCandidates.push(withoutNotifications); - } - } - - let medication; - let lastUpdateErr: unknown; - for (const candidate of updateCandidates) { + // Recompute each rollup day the tombstoned slots sat on, now that the + // tombstones are committed. Best-effort like every other rollup + // write-hook: a failure is annotated, never fails the edit. + for (const dayKey of staleRollupDays) { try { - medication = await prisma.medication.update({ - where: { id }, - data: candidate, - include: { schedules: true }, + await recomputeMedicationComplianceForDay( + user.id, + id, + dayKey, + user.timezone || DEFAULT_TIMEZONE, + ); + } catch (err) { + annotate({ + meta: { + medication_compliance_rollup_failed: true, + medication_compliance_rollup_error: + err instanceof Error ? err.message : String(err), + }, }); - break; - } catch (updateErr) { - lastUpdateErr = updateErr; } } - if (!medication) throw lastUpdateErr; - - // v1.25 H-MED1 — durable pause intervals. The `pausedAt` column is a - // single live marker that resume clears, so the paused window is - // irrecoverable once resumed and every paused day's slot collapses to - // "missed" in the compliance denominator. Mirror the live transition - // onto an additive era record the rate paths read: a pause opens an era - // (resumedAt null), a resume closes the latest open era. `userId` comes - // from the authenticated session; the data object is built field-by-field. - if (active === false && existing.active) { - await prisma.medicationPauseEra.create({ - data: { - medicationId: id, - userId: user.id, - pausedAt: new Date(), - resumedAt: null, - }, - }); - } else if (active === true && !existing.active) { - await prisma.medicationPauseEra.updateMany({ - where: { medicationId: id, userId: user.id, resumedAt: null }, - data: { resumedAt: new Date() }, - }); - } // Apply the v1.5.5 primary-schedule grace bridge after the // medication update so a failure here doesn't roll the medication @@ -1055,29 +1097,6 @@ export const PUT = apiHandler( } } - // v1.40 (#1024) — write the window onto the latest course and project it - // back, then serve the projected row. - if (touchesWindow) { - let projected; - try { - projected = await setCurrentWindow({ - userId: user.id, - medicationId: id, - timeZone: courseTz, - ...nextWindow, - }); - } catch (err) { - // A concurrent course write landed between the check above and this - // transaction. Nothing of the window was written. - if (err instanceof CourseWriteError) { - const r = courseRefusalResponse(err.refusal); - return apiError(r.message, r.status, { errorCode: r.errorCode }); - } - throw err; - } - medication = { ...medication, ...projected }; - } - // v1.40 (#1041) — an iPhone app build that predates custom categories // decodes an unknown category as OTHER and sends it back on every edit, // which would overwrite the person's own category with OTHER. A native diff --git a/src/lib/medications/courses.ts b/src/lib/medications/courses.ts index 760c5baee..4abd7b7d0 100644 --- a/src/lib/medications/courses.ts +++ b/src/lib/medications/courses.ts @@ -323,57 +323,69 @@ function windowAsCourse( * Clearing both on a medication with one course deletes it (chronic again, * the pre-course meaning of no dates); on one with several it is refused, * because it would silently drop the history. + * + * `tx` runs the write inside the caller's transaction, so the PUT's row + * update and this course write commit or roll back together: a refusal here + * then leaves nothing of the PUT behind. */ export async function setCurrentWindow( args: WriteContext & { startsOn: Date | null; endsOn: Date | null }, + tx?: Tx, +): Promise { + return tx + ? writeCurrentWindow(tx, args) + : prisma.$transaction((own) => writeCurrentWindow(own, args)); +} + +async function writeCurrentWindow( + tx: Tx, + args: WriteContext & { startsOn: Date | null; endsOn: Date | null }, ): Promise { const todayKey = userDayKey(args.now ?? new Date(), args.timeZone); - return prisma.$transaction(async (tx) => { - await lockMedication(tx, args.medicationId); - const { medication, courses } = await loadOwned( - tx, - args.userId, - args.medicationId, - ); - const latest = sortCourses(courses).at(-1); + await lockMedication(tx, args.medicationId); + const { medication, courses } = await loadOwned( + tx, + args.userId, + args.medicationId, + ); + const latest = sortCourses(courses).at(-1); - if (args.startsOn === null && args.endsOn === null) { - if (!latest) return project(tx, args.medicationId); - if (courses.length > 1) throw new CourseWriteError("windowRequired"); - await tx.medicationCourse.delete({ where: { id: latest.id } }); - return project(tx, args.medicationId); - } - - const next = windowAsCourse( - args.startsOn, - args.endsOn, - latest, - medication.createdAt, - args.timeZone, - ); - - if (latest) { - refuseIfInvalid( - courses.map((c) => (c.id === latest.id ? next : c)), - todayKey, - medication.oneShot, - ); - await tx.medicationCourse.update({ - where: { id: latest.id }, - data: next, - }); - } else { - refuseIfInvalid([next], todayKey, medication.oneShot); - await tx.medicationCourse.create({ - data: { - medicationId: args.medicationId, - userId: args.userId, - ...next, - }, - }); - } + if (args.startsOn === null && args.endsOn === null) { + if (!latest) return project(tx, args.medicationId); + if (courses.length > 1) throw new CourseWriteError("windowRequired"); + await tx.medicationCourse.delete({ where: { id: latest.id } }); return project(tx, args.medicationId); - }); + } + + const next = windowAsCourse( + args.startsOn, + args.endsOn, + latest, + medication.createdAt, + args.timeZone, + ); + + if (latest) { + refuseIfInvalid( + courses.map((c) => (c.id === latest.id ? next : c)), + todayKey, + medication.oneShot, + ); + await tx.medicationCourse.update({ + where: { id: latest.id }, + data: next, + }); + } else { + refuseIfInvalid([next], todayKey, medication.oneShot); + await tx.medicationCourse.create({ + data: { + medicationId: args.medicationId, + userId: args.userId, + ...next, + }, + }); + } + return project(tx, args.medicationId); } /** diff --git a/tests/integration/medication-put-course-race.test.ts b/tests/integration/medication-put-course-race.test.ts new file mode 100644 index 000000000..5b6410394 --- /dev/null +++ b/tests/integration/medication-put-course-race.test.ts @@ -0,0 +1,152 @@ +/** + * A medication PUT the course writer refuses changes nothing. + * + * The PUT checks the window against the other courses first, then writes the + * row, then writes the course. A course written by another request between + * the check and the course write makes the course write refuse. If the row + * write had already committed, the client got an error while the name, the + * schedule and `oneShot` were changed anyway: a one-time medication with a + * window that spans more than its one day. + * + * The race is reproduced by letting the up-front check pass (it is the check + * the other request outran), so the real course writer meets the conflicting + * courses and refuses, through the shipped route and a real Postgres. + * + * Mutation: move the row update back out of the transaction the course write + * runs in, and the name and `oneShot` assertions turn red. + */ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +const OWNER = "user-med-put-race"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +// The check the concurrent request outran: it saw no conflict. +vi.mock("@/lib/medications/courses", async (importOriginal) => ({ + ...(await importOriginal()), + checkCurrentWindow: vi.fn(async () => null), +})); + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + cookieJar.clear(); + headerJar.clear(); + await prisma.user.create({ + data: { + id: OWNER, + username: OWNER, + email: `${OWNER}@example.test`, + timezone: "UTC", + }, + }); + const session = await prisma.session.create({ + data: { userId: OWNER, expiresAt: new Date(Date.now() + 60 * 60 * 1000) }, + }); + cookieJar.set("healthlog_session", session.id); +}); + +function req(method: string, path: string, body?: unknown) { + return new NextRequest(`http://localhost${path}`, { + method, + headers: { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); +} + +describe("a medication PUT refused by the course writer", () => { + it("leaves the medication row, its schedule and its courses as they were", async () => { + const prisma = getPrismaClient(); + const { POST } = await import("@/app/api/medications/route"); + const created = await POST( + req("POST", "/api/medications", { + name: "Tamiflu", + dose: "75 mg", + startsOn: "2026-03-01", + endsOn: "2026-03-07", + schedules: [{ windowStart: "08:00", windowEnd: "09:00" }], + }), + ); + expect(created.status).toBe(201); + const { id } = (await created.json()).data as { id: string }; + + // The course the other request wrote. + const { POST: postCourse } = + await import("@/app/api/medications/[id]/courses/route"); + const course = await postCourse( + req("POST", `/api/medications/${id}/courses`, { + startsOn: "2026-06-10", + endsOn: "2026-06-16", + }), + { params: Promise.resolve({ id }) }, + ); + expect(course.status).toBe(201); + + const before = await prisma.medication.findUniqueOrThrow({ + where: { id }, + include: { schedules: true, courses: { orderBy: { startsOn: "asc" } } }, + }); + const revisionsBefore = await prisma.medicationScheduleRevision.count({ + where: { medicationId: id }, + }); + + // A one-time medication cannot carry two courses: the writer refuses. + const { PUT } = await import("@/app/api/medications/[id]/route"); + const res = await PUT( + req("PUT", `/api/medications/${id}`, { + name: "Oseltamivir", + oneShot: true, + startsOn: "2026-03-05", + schedules: [{ windowStart: "20:00", windowEnd: "21:00" }], + }), + { params: Promise.resolve({ id }) }, + ); + expect(res.status).toBe(422); + expect((await res.json()).meta.errorCode).toMatch(/^medications\.course\./); + + const after = await prisma.medication.findUniqueOrThrow({ + where: { id }, + include: { schedules: true, courses: { orderBy: { startsOn: "asc" } } }, + }); + expect(after.name).toBe("Tamiflu"); + expect(after.oneShot).toBe(false); + expect(after.startsOn).toEqual(before.startsOn); + expect(after.endsOn).toEqual(before.endsOn); + expect(after.updatedAt).toEqual(before.updatedAt); + expect(after.schedules.map((s) => s.id)).toEqual( + before.schedules.map((s) => s.id), + ); + expect(after.schedules.map((s) => s.windowStart)).toEqual(["08:00"]); + expect(after.courses.map((c) => [c.startsOn, c.endsOn])).toEqual( + before.courses.map((c) => [c.startsOn, c.endsOn]), + ); + expect( + await prisma.medicationScheduleRevision.count({ + where: { medicationId: id }, + }), + ).toBe(revisionsBefore); + }); +}); From b3e18739fb902e1624954245c5ebdb0ccd2d0ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 08:24:07 +0200 Subject: [PATCH 04/21] docs(readme): name the people whose reports shaped HealthLog --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f5a20e17e..54917888f 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,9 @@ Actively developed — new releases roughly weekly, issue reports and PRs welcom HealthLog started as something I built for myself. That it has turned into a project other people use, care about and help shape still surprises me, and I am grateful for every part of it. -Thank you to everyone who sent code that shipped: [@TimonBed](https://github.com/TimonBed), [@Antiheld86](https://github.com/Antiheld86), [@BernhardBuckel](https://github.com/BernhardBuckel), [@balajiv113](https://github.com/balajiv113), [@aucun6352](https://github.com/aucun6352), [@mathewcsims](https://github.com/mathewcsims) and [@muhdusama](https://github.com/muhdusama). A special thanks to [@BernhardBuckel](https://github.com/BernhardBuckel), who has gone far beyond one fix: careful reports with the code paths already traced, pull requests with tests and a plan before every change, and fixes that everyone using HealthLog benefits from, from the lab screens to how charts pick a source and average a day. +Thank you to everyone who sent code that shipped: [@TimonBed](https://github.com/TimonBed), [@Antiheld86](https://github.com/Antiheld86), [@BernhardBuckel](https://github.com/BernhardBuckel), [@balajiv113](https://github.com/balajiv113), [@aucun6352](https://github.com/aucun6352), [@mathewcsims](https://github.com/mathewcsims) and [@muhdusama](https://github.com/muhdusama). A special thanks to [@BernhardBuckel](https://github.com/BernhardBuckel), who has been working through the lab screens one careful pull request at a time, with tests and a plan before every change. -Thank you to the people whose reports shaped HealthLog, again and again: [@lutzkind](https://github.com/lutzkind), [@HercoGC](https://github.com/HercoGC), [@mills1975](https://github.com/mills1975), [@sreeramachandramurthy](https://github.com/sreeramachandramurthy), [@CritLoren](https://github.com/CritLoren), [@mbreitkreuz](https://github.com/mbreitkreuz), [@k-nacion](https://github.com/k-nacion), [@NebuPookins](https://github.com/NebuPookins), [@Nazza01](https://github.com/Nazza01), [@Cnote43](https://github.com/Cnote43) and [@juliaanvaneijndt](https://github.com/juliaanvaneijndt). And to everyone else who opened an issue, asked a question in the discussions or wrote me an email. Most of what got better started with someone describing what went wrong for them, often with a log, a screenshot and a second try when the first fix was not enough. +Thank you to everyone who opened an issue, asked a question in the discussions or wrote me an email, and especially to [@lutzkind](https://github.com/lutzkind), [@HercoGC](https://github.com/HercoGC), [@mills1975](https://github.com/mills1975), [@sreeramachandramurthy](https://github.com/sreeramachandramurthy), [@CritLoren](https://github.com/CritLoren), [@mbreitkreuz](https://github.com/mbreitkreuz), [@k-nacion](https://github.com/k-nacion), [@NebuPookins](https://github.com/NebuPookins), [@Nazza01](https://github.com/Nazza01), [@Cnote43](https://github.com/Cnote43) and [@juliaanvaneijndt](https://github.com/juliaanvaneijndt), whose reports shaped HealthLog again and again. Most of what got better in the last months started with someone describing what went wrong for them, often with a log, a screenshot and a second try when the first fix was not enough. And thank you to everyone who supported the project with a coffee or by buying the iPhone app. I don't name anyone here, but I read every message that comes with it. It pays for the servers and the Apple developer account, and it tells me that HealthLog is worth the evenings. From 715df7d5d33cbc9281c385b1a3269866e8d6761f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 08:29:43 +0200 Subject: [PATCH 05/21] docs(readme): keep the contributor list without a separate line for one person --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 54917888f..a757a5c08 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ Actively developed — new releases roughly weekly, issue reports and PRs welcom HealthLog started as something I built for myself. That it has turned into a project other people use, care about and help shape still surprises me, and I am grateful for every part of it. -Thank you to everyone who sent code that shipped: [@TimonBed](https://github.com/TimonBed), [@Antiheld86](https://github.com/Antiheld86), [@BernhardBuckel](https://github.com/BernhardBuckel), [@balajiv113](https://github.com/balajiv113), [@aucun6352](https://github.com/aucun6352), [@mathewcsims](https://github.com/mathewcsims) and [@muhdusama](https://github.com/muhdusama). A special thanks to [@BernhardBuckel](https://github.com/BernhardBuckel), who has been working through the lab screens one careful pull request at a time, with tests and a plan before every change. +Thank you to everyone who sent code that shipped: [@TimonBed](https://github.com/TimonBed), [@Antiheld86](https://github.com/Antiheld86), [@BernhardBuckel](https://github.com/BernhardBuckel), [@balajiv113](https://github.com/balajiv113), [@aucun6352](https://github.com/aucun6352), [@mathewcsims](https://github.com/mathewcsims) and [@muhdusama](https://github.com/muhdusama). Thank you to everyone who opened an issue, asked a question in the discussions or wrote me an email, and especially to [@lutzkind](https://github.com/lutzkind), [@HercoGC](https://github.com/HercoGC), [@mills1975](https://github.com/mills1975), [@sreeramachandramurthy](https://github.com/sreeramachandramurthy), [@CritLoren](https://github.com/CritLoren), [@mbreitkreuz](https://github.com/mbreitkreuz), [@k-nacion](https://github.com/k-nacion), [@NebuPookins](https://github.com/NebuPookins), [@Nazza01](https://github.com/Nazza01), [@Cnote43](https://github.com/Cnote43) and [@juliaanvaneijndt](https://github.com/juliaanvaneijndt), whose reports shaped HealthLog again and again. Most of what got better in the last months started with someone describing what went wrong for them, often with a log, a screenshot and a second try when the first fix was not enough. From 43a53255c94f727625ec7958667f9534e61ac6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 08:31:11 +0200 Subject: [PATCH 06/21] Carry custom mood labels readable in a portable backup The labels of the mood tags and mood categories an account created travelled as the stored ciphertext in a portable file, so a restore onto a host with another key refused the file or wrote labels nobody could open. They now follow the cycle note: readable in a portable file, sealed under the receiving host's key on restore, ciphertext kept only where this host opens it and otherwise reported as moodLabelCiphertext. The sealing helpers move to one module both sections share. --- docs/api/openapi.yaml | 8 + messages/de.json | 3 +- messages/en.json | 3 +- messages/es.json | 3 +- messages/fr.json | 3 +- messages/it.json | 3 +- messages/ko.json | 3 +- messages/pl.json | 3 +- src/components/admin/backups-section.tsx | 3 + src/lib/cycle/backup.ts | 60 +--- src/lib/export/backup-key-ids.ts | 7 +- src/lib/export/full-backup-payload.ts | 27 +- src/lib/export/restore-backup.ts | 23 +- src/lib/export/restore-skips.ts | 8 +- src/lib/export/sealed-text.ts | 70 +++++ src/lib/validations/backup.ts | 10 +- ...ackup-mood-label-cross-key-restore.test.ts | 266 ++++++++++++++++++ 17 files changed, 431 insertions(+), 72 deletions(-) create mode 100644 src/lib/export/sealed-text.ts create mode 100644 tests/integration/backup-mood-label-cross-key-restore.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index ffdc1ead6..9f672d83b 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -36796,6 +36796,10 @@ components: anyOf: - type: string - type: "null" + label: + anyOf: + - type: string + - type: "null" required: - id - key @@ -36853,6 +36857,10 @@ components: anyOf: - type: string - type: "null" + label: + anyOf: + - type: string + - type: "null" scaleMin: default: 1 type: integer diff --git a/messages/de.json b/messages/de.json index 5390c5fb4..d50600800 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7555,7 +7555,8 @@ "restoreFailureForeignReference": "Die Datei verweist auf Einträge, die zu einem anderen Konto gehören. Es wurde nichts geschrieben.", "previewKeyMissing": "Diese Sicherung lässt sich auf diesem Server nicht wiederherstellen: Sie braucht den Verschlüsselungsschlüssel {keys}. Nimm den Schlüssel zuerst wieder in ENCRYPTION_KEYS auf und starte neu.", "restoreSkippedSymptomEpisodeReference": "Symptom-Eintrag mit Verknüpfung zu einer Erkrankung, die die Datei nicht enthält (ohne Verknüpfung behalten)", - "restoreSkippedCycleCiphertext": "Zyklus-Notiz oder -Bezeichnung mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (Tag oder Symptom ohne sie behalten)" + "restoreSkippedCycleCiphertext": "Zyklus-Notiz oder -Bezeichnung mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (Tag oder Symptom ohne sie behalten)", + "restoreSkippedMoodLabelCiphertext": "Name eines Stimmungs-Tags oder einer Gruppe mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (ohne Namen behalten)" }, "danger-zone": { "title": "Gefahrenzone", diff --git a/messages/en.json b/messages/en.json index dd65185b3..b2a3d92c6 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7555,7 +7555,8 @@ "restoreFailureForeignReference": "The file points at records that belong to another account. Nothing was written.", "previewKeyMissing": "This backup cannot be restored on this server: it needs encryption key {keys}. Add the key back to ENCRYPTION_KEYS and restart first.", "restoreSkippedSymptomEpisodeReference": "Symptom entry linked to an illness the file does not contain (kept without the link)", - "restoreSkippedCycleCiphertext": "Cycle note or label sealed with a key this server does not hold (day or symptom kept without it)" + "restoreSkippedCycleCiphertext": "Cycle note or label sealed with a key this server does not hold (day or symptom kept without it)", + "restoreSkippedMoodLabelCiphertext": "Mood tag or group name sealed with a key this server does not hold (kept without its name)" }, "danger-zone": { "title": "Danger Zone", diff --git a/messages/es.json b/messages/es.json index 8ec71bc7a..daf60d074 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7555,7 +7555,8 @@ "restoreFailureForeignReference": "El archivo apunta a registros que pertenecen a otra cuenta. No se escribió nada.", "previewKeyMissing": "Esta copia no se puede restaurar en este servidor: necesita la clave de cifrado {keys}. Vuelve a añadir la clave a ENCRYPTION_KEYS y reinicia primero.", "restoreSkippedSymptomEpisodeReference": "Entrada de síntoma vinculada a una enfermedad que el archivo no contiene (se conserva sin el vínculo)", - "restoreSkippedCycleCiphertext": "Nota o etiqueta del ciclo cifrada con una clave que este servidor no tiene (el día o el síntoma se conserva sin ella)" + "restoreSkippedCycleCiphertext": "Nota o etiqueta del ciclo cifrada con una clave que este servidor no tiene (el día o el síntoma se conserva sin ella)", + "restoreSkippedMoodLabelCiphertext": "Nombre de etiqueta o grupo de ánimo cifrado con una clave que este servidor no tiene (se conserva sin el nombre)" }, "danger-zone": { "title": "Zona de peligro", diff --git a/messages/fr.json b/messages/fr.json index e9d1163a5..29378a5a1 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7555,7 +7555,8 @@ "restoreFailureForeignReference": "Le fichier renvoie à des éléments qui appartiennent à un autre compte. Rien n'a été écrit.", "previewKeyMissing": "Cette sauvegarde ne peut pas être restaurée sur ce serveur : elle a besoin de la clé de chiffrement {keys}. Remettez d'abord la clé dans ENCRYPTION_KEYS et redémarrez.", "restoreSkippedSymptomEpisodeReference": "Entrée de symptôme liée à une maladie absente du fichier (conservée sans le lien)", - "restoreSkippedCycleCiphertext": "Note ou libellé du cycle chiffré avec une clé que ce serveur ne possède pas (jour ou symptôme conservé sans)" + "restoreSkippedCycleCiphertext": "Note ou libellé du cycle chiffré avec une clé que ce serveur ne possède pas (jour ou symptôme conservé sans)", + "restoreSkippedMoodLabelCiphertext": "Nom d’étiquette ou de groupe d’humeur chiffré avec une clé que ce serveur ne possède pas (conservé sans son nom)" }, "danger-zone": { "title": "Zone de danger", diff --git a/messages/it.json b/messages/it.json index d1538b30b..00a8957cf 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7555,7 +7555,8 @@ "restoreFailureForeignReference": "Il file rimanda a voci che appartengono a un altro account. Non è stato scritto nulla.", "previewKeyMissing": "Questo backup non può essere ripristinato su questo server: richiede la chiave di cifratura {keys}. Aggiungi prima di nuovo la chiave a ENCRYPTION_KEYS e riavvia.", "restoreSkippedSymptomEpisodeReference": "Voce di sintomo collegata a una malattia assente dal file (mantenuta senza collegamento)", - "restoreSkippedCycleCiphertext": "Nota o etichetta del ciclo cifrata con una chiave che questo server non possiede (giorno o sintomo mantenuto senza)" + "restoreSkippedCycleCiphertext": "Nota o etichetta del ciclo cifrata con una chiave che questo server non possiede (giorno o sintomo mantenuto senza)", + "restoreSkippedMoodLabelCiphertext": "Nome di tag o gruppo dell’umore cifrato con una chiave che questo server non possiede (mantenuto senza nome)" }, "danger-zone": { "title": "Zona pericolosa", diff --git a/messages/ko.json b/messages/ko.json index a61e278d9..7a1991f25 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7534,7 +7534,8 @@ "restoreFailureForeignReference": "파일이 다른 계정에 속한 기록을 가리키고 있습니다. 아무것도 기록되지 않았습니다.", "previewKeyMissing": "이 서버에서는 이 백업을 복원할 수 없습니다. 암호화 키 {keys}가 필요합니다. 먼저 키를 ENCRYPTION_KEYS에 다시 추가하고 재시작하세요.", "restoreSkippedSymptomEpisodeReference": "파일에 없는 질병과 연결된 증상 기록(연결 없이 유지됨)", - "restoreSkippedCycleCiphertext": "이 서버에 없는 키로 암호화된 주기 메모 또는 이름(해당 날짜나 증상은 그 내용 없이 유지됨)" + "restoreSkippedCycleCiphertext": "이 서버에 없는 키로 암호화된 주기 메모 또는 이름(해당 날짜나 증상은 그 내용 없이 유지됨)", + "restoreSkippedMoodLabelCiphertext": "이 서버에 없는 키로 암호화된 기분 태그 또는 그룹 이름(이름 없이 유지됨)" }, "danger-zone": { "title": "위험 구역", diff --git a/messages/pl.json b/messages/pl.json index 6a51e0eeb..d8cbfe8b5 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7555,7 +7555,8 @@ "restoreFailureForeignReference": "Plik wskazuje na wpisy należące do innego konta. Nic nie zostało zapisane.", "previewKeyMissing": "Tej kopii nie da się przywrócić na tym serwerze: wymaga klucza szyfrowania {keys}. Najpierw dodaj klucz z powrotem do ENCRYPTION_KEYS i uruchom ponownie.", "restoreSkippedSymptomEpisodeReference": "Wpis objawu powiązany z chorobą, której nie ma w pliku (zachowany bez powiązania)", - "restoreSkippedCycleCiphertext": "Notatka lub etykieta cyklu zaszyfrowana kluczem, którego ten serwer nie ma (dzień lub objaw zachowany bez niej)" + "restoreSkippedCycleCiphertext": "Notatka lub etykieta cyklu zaszyfrowana kluczem, którego ten serwer nie ma (dzień lub objaw zachowany bez niej)", + "restoreSkippedMoodLabelCiphertext": "Nazwa tagu lub grupy nastroju zaszyfrowana kluczem, którego ten serwer nie ma (zachowana bez nazwy)" }, "danger-zone": { "title": "Strefa zagrożenia", diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index 9a295f8a3..96f34bdd9 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -464,6 +464,9 @@ function catalogueLabel( if (catalogue === "symptomEpisodeReference") { return t("admin.section.backups.restoreSkippedSymptomEpisodeReference"); } + if (catalogue === "moodLabelCiphertext") { + return t("admin.section.backups.restoreSkippedMoodLabelCiphertext"); + } if (catalogue === "cycleCiphertext") { return t("admin.section.backups.restoreSkippedCycleCiphertext"); } diff --git a/src/lib/cycle/backup.ts b/src/lib/cycle/backup.ts index 4862a41a3..c04859d87 100644 --- a/src/lib/cycle/backup.ts +++ b/src/lib/cycle/backup.ts @@ -40,9 +40,12 @@ import type { PrismaClient, SecondarySymptom, } from "@/generated/prisma/client"; -import { decrypt, encrypt } from "@/lib/crypto"; +import { + openSealedForExport, + sealForRestore, + UNOPENED, +} from "@/lib/export/sealed-text"; import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker"; -import { getEvent } from "@/lib/logging/context"; import type { BackupPayload } from "@/lib/validations/backup"; import { recordUnknownKeys, @@ -189,26 +192,12 @@ export interface CycleSensitiveFields { contraceptive?: string | null; } -/** Open a sealed value for a portable file, or the unreadable marker. */ -function openForExport(sealed: string, field: string): string { - try { - return decrypt(sealed); - } catch (err) { - getEvent()?.addWarning( - `cycle ${field} decrypt failed: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - return UNREADABLE_EXPORT_MARKER; - } -} - /** The sensitive envelope opened for a portable file, or the marker. */ function openSensitiveForExport( sealed: string | null, ): CycleSensitiveFields | string | null { if (!sealed) return null; - const opened = openForExport(sealed, "sensitive envelope"); + const opened = openSealedForExport(sealed, "cycle sensitive envelope"); if (opened === UNREADABLE_EXPORT_MARKER) return opened; try { return JSON.parse(opened) as CycleSensitiveFields; @@ -330,7 +319,7 @@ export async function buildCycleBackupSection( : { sensitive: openSensitiveForExport(d.sensitiveEncrypted), note: d.notesEncrypted - ? openForExport(d.notesEncrypted, "day-log note") + ? openSealedForExport(d.notesEncrypted, "cycle day-log note") : null, }), source: d.source, @@ -353,7 +342,7 @@ export async function buildCycleBackupSection( ? { labelEncrypted: sym.labelEncrypted } : { label: sym.labelEncrypted - ? openForExport(sym.labelEncrypted, "symptom label") + ? openSealedForExport(sym.labelEncrypted, "cycle symptom label") : null, }), })), @@ -447,39 +436,6 @@ function severityOrNull(value: number | null | undefined): number | null { : null; } -/** A readable value the file could not supply: the writer could not open it. */ -const UNOPENED = Symbol("unopened"); - -/** - * The sealed column value for one free-text field from the file. - * - * Ciphertext is written back only when this host opens it — a value sealed - * under a key this host does not hold (or the same key id over other key - * material) would be a row nobody can read. A readable value is sealed under - * this host's active key. Anything kept out is recorded under `path`. - */ -function sealForRestore( - sealed: string | null | undefined, - readable: string | null | undefined | typeof UNOPENED, - path: string, - unopened: string[], -): string | null { - if (sealed) { - try { - decrypt(sealed); - return sealed; - } catch { - unopened.push(path); - return null; - } - } - if (readable === UNOPENED) { - unopened.push(path); - return null; - } - return readable ? encrypt(readable) : null; -} - function enumOrNull( value: string | null | undefined, allow: ReadonlySet, diff --git a/src/lib/export/backup-key-ids.ts b/src/lib/export/backup-key-ids.ts index aead1cb94..c4113125e 100644 --- a/src/lib/export/backup-key-ids.ts +++ b/src/lib/export/backup-key-ids.ts @@ -101,8 +101,9 @@ export function isEncryptedMember(name: string): boolean { * Sections whose restore opens every inner value with this host's keys itself, * and keeps out (and names in the skip report) any value it cannot open. * - * The cycle day-logs and custom cycle symptoms are here because a portable - * file written before v1.40 carried their free text as the stored ciphertext, + * The cycle day-logs, custom cycle symptoms and custom mood tags and mood + * categories are here because a portable file written before v1.40 carried + * their free text as the stored ciphertext, * not as readable text. Such a file is still restorable onto another host: * what that host can open comes back, what it cannot stays out of the row and * is reported. Refusing the whole file over a note would cost every other @@ -114,6 +115,8 @@ export function isEncryptedMember(name: string): boolean { export const RESTORE_SELF_VERIFIED_SECTIONS: ReadonlySet = new Set([ "cycleDayLogs", "customSymptoms", + "customMoodTags", + "customMoodTagCategories", ]); interface KeyUse { diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index 9bd1b84d7..f96f79935 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -25,6 +25,7 @@ import { dayKeyOfDate } from "@/lib/medications/course-window"; import { Buffer } from "node:buffer"; import type { Prisma, PrismaClient } from "@/generated/prisma/client"; import { readNote } from "@/lib/crypto/note-cipher"; +import { openSealedForExport } from "@/lib/export/sealed-text"; import { iterateMeasurementPages } from "@/lib/export/paged-measurements"; import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup"; import { @@ -1284,8 +1285,18 @@ export async function buildFullBackupPayload( icon: category.icon, sortOrder: category.sortOrder, isActive: category.isActive, - // Ciphertext verbatim, like the tag label beside it. - labelEncrypted: category.labelEncrypted, + // The person's own words: ciphertext in a disaster-recovery payload, + // readable in a portable one (see `sealed-text.ts`). + ...(disasterRecovery + ? { labelEncrypted: category.labelEncrypted } + : { + label: category.labelEncrypted + ? openSealedForExport( + category.labelEncrypted, + "mood category label", + ) + : null, + }), })), hiddenMoodTags: hiddenMoodTags.map((hidden) => ({ key: hidden.moodTag.key, @@ -1300,9 +1311,15 @@ export async function buildFullBackupPayload( isActive: tag.isActive, icon: tag.icon, sortOrder: tag.sortOrder, - // Ciphertext verbatim, like every other *Encrypted column in a DR - // payload: the same instance's key reads it back unchanged. - labelEncrypted: tag.labelEncrypted, + // Ciphertext in a disaster-recovery payload (the same instance's key + // reads it back), readable in a portable one (see `sealed-text.ts`). + ...(disasterRecovery + ? { labelEncrypted: tag.labelEncrypted } + : { + label: tag.labelEncrypted + ? openSealedForExport(tag.labelEncrypted, "mood tag label") + : null, + }), scaleMin: tag.scaleMin, scaleMax: tag.scaleMax, inverse: tag.inverse, diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index 0ac7e1a68..edf38d815 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -66,6 +66,7 @@ import { listTenantEdges, } from "@/lib/export/tenant-boundary"; import { encryptNote } from "@/lib/crypto/note-cipher"; +import { sealForRestore } from "@/lib/export/sealed-text"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { encryptContextToBytes } from "@/lib/labs/biomarker-store"; import { encryptNoteToBytes } from "@/lib/labs/store"; @@ -1533,6 +1534,8 @@ export async function restoreBackup( // Upserted by key for the same reason the tags below are: `key` is // globally unique, so a seeded category that later claims the key // must be left alone rather than overwritten by a restore. + // Labels this host's keys do not open, by file path. + const unopenedMoodLabels: string[] = []; for (const category of payload.customMoodTagCategories) { await tx.moodTagCategory.upsert({ where: { key: category.key }, @@ -1544,7 +1547,12 @@ export async function restoreBackup( icon: category.icon ?? null, sortOrder: category.sortOrder ?? 0, isActive: category.isActive ?? true, - labelEncrypted: category.labelEncrypted ?? null, + labelEncrypted: sealForRestore( + category.labelEncrypted, + category.label, + `customMoodTagCategories.${category.key}.labelEncrypted`, + unopenedMoodLabels, + ), }, update: {}, }); @@ -1567,7 +1575,12 @@ export async function restoreBackup( isActive: tag.isActive, icon: tag.icon ?? null, sortOrder: tag.sortOrder, - labelEncrypted: tag.labelEncrypted ?? null, + labelEncrypted: sealForRestore( + tag.labelEncrypted, + tag.label, + `customMoodTags.${tag.key}.labelEncrypted`, + unopenedMoodLabels, + ), scaleMin: tag.scaleMin, scaleMax: tag.scaleMax, inverse: tag.inverse, @@ -1577,6 +1590,12 @@ export async function restoreBackup( update: {}, }); } + recordUnknownKeys( + skips, + "moodLabelCiphertext", + unopenedMoodLabels, + unopenedMoodLabels, + ); // Which tags the account hid. Delete-then-recreate like every // other section, and resolved by KEY: what people hide is almost diff --git a/src/lib/export/restore-skips.ts b/src/lib/export/restore-skips.ts index 2b2ca30fc..e292b78e4 100644 --- a/src/lib/export/restore-skips.ts +++ b/src/lib/export/restore-skips.ts @@ -160,6 +160,11 @@ * Writing it back would store a value no reader can open, so the FIELD is kept * out and the day or the symptom restores without it. Its key is the path in * the file (`cycleDayLogs..notesEncrypted`), never the value. + * + * The seventeenth, `moodLabelCiphertext`, is the same for the label of a mood + * tag or mood category the account created + * (`customMoodTags..labelEncrypted`): the tag or category restores, under + * its key, without the person's own name for it. */ export type SkippedCatalogue = | "cycleSymptom" @@ -181,7 +186,8 @@ export type SkippedCatalogue = | "checkupClosure" | "accountSetting" | "symptomEpisodeReference" - | "cycleCiphertext"; + | "cycleCiphertext" + | "moodLabelCiphertext"; /** One key this instance does not know, and the links it cost. */ export interface SkippedCatalogueKey { diff --git a/src/lib/export/sealed-text.ts b/src/lib/export/sealed-text.ts new file mode 100644 index 000000000..c1e28a3c7 --- /dev/null +++ b/src/lib/export/sealed-text.ts @@ -0,0 +1,70 @@ +/** + * Both ends of a sealed text column stored with the string codec + * (`.`), for the backup sections that carry one: the cycle + * day-log note and sensitive envelope, custom cycle symptom labels, and the + * custom mood tag and mood category labels. + * + * The split is the one every other section uses. A disaster-recovery file + * carries the stored ciphertext, which the same instance's key reads back. A + * portable file carries the readable value (or the unreadable marker when + * the writing host could not open it), and the restore seals it under the + * receiving host's active key, because ciphertext from one host is noise on + * another. + * + * Portable files written before v1.40 carried these columns as ciphertext. + * They still restore: a ciphertext value is written back only when this + * host's keys open it. One that does not open is kept out of the row and its + * file path is recorded, so the caller can name it in the skip report rather + * than store a value nobody can read. That per-value check is why the key + * preflight leaves these sections out (`RESTORE_SELF_VERIFIED_SECTIONS`). + */ +import { decrypt, encrypt } from "@/lib/crypto"; +import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker"; +import { getEvent } from "@/lib/logging/context"; + +/** Open a sealed value for a portable file, or the unreadable marker. */ +export function openSealedForExport(sealed: string, field: string): string { + try { + return decrypt(sealed); + } catch (err) { + getEvent()?.addWarning( + `${field} decrypt failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return UNREADABLE_EXPORT_MARKER; + } +} + +/** A readable value the file could not supply: the writer could not open it. */ +export const UNOPENED: unique symbol = Symbol("unopened"); + +/** + * The sealed column value for one free-text field from the file. + * + * Ciphertext is written back only when this host opens it — a value sealed + * under a key this host does not hold (or the same key id over other key + * material) would be a row nobody can read. A readable value is sealed under + * this host's active key. Anything kept out is recorded under `path`. + */ +export function sealForRestore( + sealed: string | null | undefined, + readable: string | null | undefined | typeof UNOPENED, + path: string, + unopened: string[], +): string | null { + if (sealed) { + try { + decrypt(sealed); + return sealed; + } catch { + unopened.push(path); + return null; + } + } + if (readable === UNOPENED) { + unopened.push(path); + return null; + } + return readable ? encrypt(readable) : null; +} diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index ad0334bcc..a96fa893a 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -717,10 +717,12 @@ const customMoodTagSchema = z isActive: z.boolean().default(true), icon: z.string().nullable().optional(), sortOrder: z.number().int().default(0), - // The user's own words for the tag, encrypted at rest and carried - // verbatim. Without it the tag comes back as a bare key and the person - // who named it "Migräne" gets `custom:cm3x9…` instead. + // The user's own words for the tag. Without it the tag comes back as a + // bare key and the person who named it "Migräne" gets `custom:cm3x9…` + // instead. Ciphertext on a disaster-recovery file, readable `label` on a + // portable one; the restore seals the readable value under its own key. labelEncrypted: z.string().nullable().optional(), + label: z.string().nullable().optional(), // The scale a RATED factor was recorded on. `inverse` marks a factor // where a HIGH value is bad (stress, conflict). Dropping it does not lose // a label, it silently reverses the meaning of every rating already @@ -1642,7 +1644,9 @@ const customMoodTagCategorySchema = z icon: z.string().nullable().optional(), sortOrder: z.number().int().optional(), isActive: z.boolean().optional(), + // Same split as the custom tag's label. labelEncrypted: z.string().nullable().optional(), + label: z.string().nullable().optional(), }) .passthrough(); diff --git a/tests/integration/backup-mood-label-cross-key-restore.test.ts b/tests/integration/backup-mood-label-cross-key-restore.test.ts new file mode 100644 index 000000000..e2bb51883 --- /dev/null +++ b/tests/integration/backup-mood-label-cross-key-restore.test.ts @@ -0,0 +1,266 @@ +/** + * A portable file's custom mood labels survive a move to a host with other + * keys. + * + * The labels of the mood tags and mood tag categories a person created are + * sealed under the writing host's key. A portable file used to carry them as + * the stored ciphertext, so a restore onto a host with a different key + * refused the file at the key check, or with a matching key id over other key + * material wrote labels nobody could open. + * + * Both ends and the pipe: built by the real `buildFullBackupPayload` under key + * A, restored through the real route under key B, opened under key B. The + * second case is a file written before the fix: a label this host opens comes + * back, one it cannot open stays out and is named in the skip report. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { _resetCryptoCacheForTests, decrypt, encrypt } from "@/lib/crypto"; +import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { parseBackupPayload } from "@/lib/validations/backup"; +import { POST } from "./restore-job-driver"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserData: vi.fn(), +})); + +const saved = { + key: process.env.ENCRYPTION_KEY, + keys: process.env.ENCRYPTION_KEYS, + active: process.env.ENCRYPTION_ACTIVE_KEY_ID, +}; + +/** Make this process the host holding exactly `keys`, writing under `active`. */ +function useKeys(keys: Record, active: string) { + process.env.ENCRYPTION_KEY = ""; + process.env.ENCRYPTION_KEYS = JSON.stringify(keys); + process.env.ENCRYPTION_ACTIVE_KEY_ID = active; + _resetCryptoCacheForTests(); +} + +const HOST_A = { a: "11".repeat(32) }; +const HOST_B = { b: "22".repeat(32) }; +/** Same key id as host B, different key material. */ +const HOST_B_IMPOSTOR = { b: "33".repeat(32) }; + +const CATEGORY_LABEL = "Evenings"; +const TAG_LABEL = "Migraine"; + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +afterEach(() => { + process.env.ENCRYPTION_KEY = saved.key ?? ""; + process.env.ENCRYPTION_KEYS = saved.keys ?? ""; + process.env.ENCRYPTION_ACTIVE_KEY_ID = saved.active ?? ""; + _resetCryptoCacheForTests(); +}); + +function makeRequest(id: string) { + return new Request(`http://localhost/api/admin/backups/${id}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }); +} + +async function seedOwner() { + const prisma = getPrismaClient(); + const owner = await prisma.user.create({ + data: { + username: "mood-cross-key-owner", + email: "mood-cross-key-owner@example.test", + role: "ADMIN", + }, + }); + const session = await prisma.session.create({ + data: { userId: owner.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return owner; +} + +/** Store `payload` as a backup on this host and run the real restore. */ +async function restore(ownerId: string, payload: unknown) { + const prisma = getPrismaClient(); + const backup = await prisma.dataBackup.create({ + data: { + userId: ownerId, + type: "MOOD_CROSS_KEY_RESTORE", + data: encrypt(JSON.stringify(payload)), + }, + }); + return POST(makeRequest(backup.id) as unknown as Parameters[0], { + params: Promise.resolve({ id: backup.id }), + }); +} + +/** + * One mood entry. The restore writes the account's own tags and categories + * inside its mood-entry branch, so a file without entries never reaches them. + */ +async function seedMoodEntry(userId: string) { + await getPrismaClient().moodEntry.create({ + data: { + userId, + date: "2026-07-01", + mood: "GUT", + score: 4, + source: "MOODLOG", + moodLoggedAt: new Date("2026-07-01T20:00:00.000Z"), + }, + }); +} + +describe("custom mood labels in a portable file restored under another key", () => { + it("come back readable under the receiving host's key", async () => { + const prisma = getPrismaClient(); + const owner = await seedOwner(); + + useKeys(HOST_A, "a"); + await seedMoodEntry(owner.id); + const category = await prisma.moodTagCategory.create({ + data: { + userId: owner.id, + key: "customcat:cross-key", + labelKey: "mood.category.custom", + labelEncrypted: encrypt(CATEGORY_LABEL), + }, + }); + await prisma.moodTag.create({ + data: { + userId: owner.id, + categoryId: category.id, + key: "custom:cross-key", + labelKey: "mood.tag.custom", + kind: "BINARY", + labelEncrypted: encrypt(TAG_LABEL), + }, + }); + const { payload: built } = await buildFullBackupPayload(prisma, owner.id); + const file = JSON.parse(JSON.stringify(built)) as unknown; + + // Host B is another database: the rows host A wrote are not there. + useKeys(HOST_B, "b"); + await prisma.moodTag.deleteMany({ where: { userId: owner.id } }); + await prisma.moodTagCategory.deleteMany({ where: { userId: owner.id } }); + const response = await restore(owner.id, file); + expect(response.status).toBe(200); + + const restoredCategory = await prisma.moodTagCategory.findUniqueOrThrow({ + where: { key: "customcat:cross-key" }, + }); + expect(decrypt(restoredCategory.labelEncrypted!)).toBe(CATEGORY_LABEL); + const restoredTag = await prisma.moodTag.findUniqueOrThrow({ + where: { key: "custom:cross-key" }, + }); + expect(decrypt(restoredTag.labelEncrypted!)).toBe(TAG_LABEL); + + const body = (await response.json()) as { + data: { skipped: { links: number } }; + }; + expect(body.data.skipped.links).toBe(0); + }); + + it("restores a legacy file's label ciphertext only where this host can open it", async () => { + const prisma = getPrismaClient(); + const owner = await seedOwner(); + + useKeys(HOST_B, "b"); + const ownLabel = encrypt(TAG_LABEL); + useKeys(HOST_B_IMPOSTOR, "b"); + const impostorLabel = encrypt(CATEGORY_LABEL); + + // The shape a portable file had before the fix: labels as ciphertext. + useKeys(HOST_B, "b"); + await seedMoodEntry(owner.id); + const { payload: built } = await buildFullBackupPayload(prisma, owner.id); + const file = JSON.parse(JSON.stringify(built)) as Record; + file.customMoodTagCategories = [ + { + id: "legacy-category-id", + key: "customcat:legacy", + labelKey: "mood.category.custom", + icon: null, + sortOrder: 0, + isActive: true, + labelEncrypted: impostorLabel, + }, + ]; + file.customMoodTags = [ + { + key: "custom:legacy", + labelKey: "mood.tag.custom", + categoryId: "legacy-category-id", + kind: "BINARY", + isActive: true, + icon: null, + sortOrder: 0, + labelEncrypted: ownLabel, + scaleMin: 1, + scaleMax: 5, + inverse: false, + }, + ]; + parseBackupPayload(JSON.stringify(file)); + + const response = await restore(owner.id, file); + expect(response.status).toBe(200); + + const category = await prisma.moodTagCategory.findUniqueOrThrow({ + where: { key: "customcat:legacy" }, + }); + expect(category.labelEncrypted).toBeNull(); + const tag = await prisma.moodTag.findUniqueOrThrow({ + where: { key: "custom:legacy" }, + }); + expect(decrypt(tag.labelEncrypted!)).toBe(TAG_LABEL); + + const body = (await response.json()) as { + data: { + skipped: { + links: number; + catalogueKeys: Array<{ + catalogue: string; + key: string; + links: number; + }>; + }; + }; + }; + expect(body.data.skipped.catalogueKeys).toEqual([ + { + catalogue: "moodLabelCiphertext", + key: "customMoodTagCategories.customcat:legacy.labelEncrypted", + links: 1, + }, + ]); + }); +}); From c7a2eec078d48084e39b876aa37e7297299a961b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 08:31:11 +0200 Subject: [PATCH 07/21] Drop the column fallback around the medication update The update retried without pausedAt and notificationsEnabled for a database that lacked the columns. Migrations guarantee both, and inside the transaction a failed statement aborts it, so the retry could not work anyway. --- src/app/api/medications/[id]/route.ts | 41 ++++----------------------- 1 file changed, 5 insertions(+), 36 deletions(-) diff --git a/src/app/api/medications/[id]/route.ts b/src/app/api/medications/[id]/route.ts index f7e62e198..3587a4dda 100644 --- a/src/app/api/medications/[id]/route.ts +++ b/src/app/api/medications/[id]/route.ts @@ -969,42 +969,11 @@ export const PUT = apiHandler( }), }; - const withoutNotifications = { ...baseUpdateData } as Record< - string, - unknown - >; - delete withoutNotifications.notificationsEnabled; - const hasPausedAtPatch = Object.keys(pausedAtPatch).length > 0; - const hasNotificationsPatch = notificationsEnabled !== undefined; - - const updateCandidates: Array> = [ - { ...baseUpdateData, ...pausedAtPatch }, - ]; - if (hasPausedAtPatch) { - updateCandidates.push(baseUpdateData); - } - if (hasNotificationsPatch) { - updateCandidates.push({ ...withoutNotifications, ...pausedAtPatch }); - if (hasPausedAtPatch) { - updateCandidates.push(withoutNotifications); - } - } - - let updated; - let lastUpdateErr: unknown; - for (const candidate of updateCandidates) { - try { - updated = await tx.medication.update({ - where: { id }, - data: candidate, - include: { schedules: true }, - }); - break; - } catch (updateErr) { - lastUpdateErr = updateErr; - } - } - if (!updated) throw lastUpdateErr; + const updated = await tx.medication.update({ + where: { id }, + data: { ...baseUpdateData, ...pausedAtPatch }, + include: { schedules: true }, + }); // v1.25 H-MED1 — durable pause intervals. The `pausedAt` column is a // single live marker that resume clears, so the paused window is From b55850967dab9f7c8dff53175f25170e48a73483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:14:22 +0200 Subject: [PATCH 08/21] fix(idempotency): replay a key only for the same request body The replay cell was keyed on (user, key, method, path). A client that sends two different writes under one key got the first write's cached response for the second, and the second was never saved. Native clients up to 1.1.0 post the two halves of a manual blood pressure exactly that way, so the diastolic value was silently dropped. Each cell now stores a SHA-256 fingerprint of the request body (JSON with recursively sorted keys, otherwise the raw bytes; scoped by method and path; at most the first MiB is read). Lookup replays, or refuses as in flight, only when the fingerprint matches. A different body, or a row written before the column existed, runs the handler without claiming or caching and leaves the existing cell for its own retries; a lost claim race re-checks the winner's fingerprint the same way. The body itself is never stored. Migration 0371 adds the nullable idempotency_keys.request_fingerprint column. The OpenAPI parameter description, the client notes and the Home Assistant guide say what a replay now requires. docs/ops gains a read-only query that lists systolic readings left without their diastolic half, held to the schema by an integration test. --- CHANGELOG.md | 15 ++ docs/CONTRIBUTING-CLIENTS.md | 2 +- docs/api/openapi.yaml | 14 +- docs/integrations/home-assistant.md | 5 +- docs/ops/blood-pressure-missing-diastolic.md | 60 +++++ .../migration.sql | 8 + prisma/schema.prisma | 22 +- .../__tests__/idempotency-conflict.test.ts | 11 +- src/lib/__tests__/idempotency.test.ts | 208 +++++++++++++++ src/lib/idempotency.ts | 184 ++++++++++++- src/lib/openapi/routes/shared.ts | 5 +- .../idempotency-body-fingerprint.test.ts | 249 ++++++++++++++++++ 12 files changed, 761 insertions(+), 22 deletions(-) create mode 100644 docs/ops/blood-pressure-missing-diastolic.md create mode 100644 prisma/migrations/0371_idempotency_request_fingerprint/migration.sql create mode 100644 tests/integration/idempotency-body-fingerprint.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 53cb335cb..5654e7f10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [Unreleased] + +### Fixed + +- **Blood pressure saved without its diastolic value.** The iPhone app (up to + 1.0.3, and TestFlight 1.1.0) sends the two halves of a manual reading as two + requests under one `Idempotency-Key`, and the server answered the second + with the first one's cached response instead of saving it. The replay cache + now stores a SHA-256 fingerprint of the request body (migration 0371, + `idempotency_keys.request_fingerprint`) and replays only the same body; a + different body under the same key is saved normally. Readings saved before + this fix stay incomplete: delete the incomplete reading and enter it again + with both values and the original time. Operators can list affected + readings with the query in `docs/ops/blood-pressure-missing-diastolic.md`. + ## [1.39.9] — 2026-10-03 Workouts can be logged by hand, the body fat tile opens its own page, and diff --git a/docs/CONTRIBUTING-CLIENTS.md b/docs/CONTRIBUTING-CLIENTS.md index 2f5660022..e2f627e8c 100644 --- a/docs/CONTRIBUTING-CLIENTS.md +++ b/docs/CONTRIBUTING-CLIENTS.md @@ -19,7 +19,7 @@ Every JSON route answers the same shape: `{ data, error }`, with `meta` on the e - Refresh tokens rotate per device and are one-time-use. Reusing an old one revokes that device's whole token family, on purpose. Persist the newest pair atomically and never retry a refresh with a token that already succeeded once. - Bearer token scopes are fail-closed. A token minted for one purpose does not open unrelated routes, and a route that does not declare a scope refuses narrow tokens outright. Expect `403` to be a normal, recoverable answer, not an anomaly. - Admin surfaces are cookie-session only, by construction. A native client cannot be an admin client, whatever scopes its token carries. Do not build admin features. -- Mutations accept an `Idempotency-Key` header. Replays answer with the original result plus `X-Idempotent-Replay`, so retrying on a dropped connection is safe. Use it for anything a user would notice twice. +- Mutations accept an `Idempotency-Key` header. A repeat of the same key with the same body answers with the original result plus `X-Idempotent-Replay`, so retrying on a dropped connection is safe. A different body under the same key is treated as a new request, not a replay; still, give each distinct write its own key. Use it for anything a user would notice twice. ## Data semantics worth knowing before you sync diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 25431a0bd..50a2926cb 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -162,12 +162,14 @@ paths: pattern: ^[A-Za-z0-9_\-:.]{8,128}$ minLength: 8 maxLength: 128 - description: "Makes this write safe to retry. Send the same key again and the first response is replayed rather than the - work repeated, marked with `X-Idempotent-Replay: true`. A key that does not match the pattern is IGNORED - rather than refused — the write then proceeds unprotected and looks exactly like a request that carried no - key, so validate the key on your side. Scoped per user and per (method, path): the same key on a different - route is a different key. While a first request under this key is still in flight, a second is refused with - 409 rather than queued." + description: "Makes this write safe to retry. Send the same key with the same body again and the first response is + replayed rather than the work repeated, marked with `X-Idempotent-Replay: true`. The body is compared as + JSON with key order ignored; a different body under the same key is not a retry and runs as a new write, + while the first response stays stored for retries of the first request. A key that does not match the + pattern is IGNORED rather than refused — the write then proceeds unprotected and looks exactly like a + request that carried no key, so validate the key on your side. Scoped per user and per (method, path): the + same key on a different route is a different key. While a first request under this key is still in flight, a + second with the same body is refused with 409 rather than queued." requestBody: required: true content: diff --git a/docs/integrations/home-assistant.md b/docs/integrations/home-assistant.md index c521a8c74..71d679ccc 100644 --- a/docs/integrations/home-assistant.md +++ b/docs/integrations/home-assistant.md @@ -171,8 +171,9 @@ tells you which rows to retry. ## Retries and duplicates Send an `Idempotency-Key` header with a value your automation can reproduce for -the same reading. A retry with the same key returns the original response -instead of writing a second row: +the same reading. A retry with the same key and the same body returns the +original response instead of writing a second row. A different body under the +same key is a different reading and is written: ```yaml headers: diff --git a/docs/ops/blood-pressure-missing-diastolic.md b/docs/ops/blood-pressure-missing-diastolic.md new file mode 100644 index 000000000..1509a6ade --- /dev/null +++ b/docs/ops/blood-pressure-missing-diastolic.md @@ -0,0 +1,60 @@ +# Blood pressure readings missing their diastolic half + +## What happened + +The iPhone app (every App Store build up to 1.0.3 and TestFlight 1.1.0) +saves a manually entered blood pressure as two requests to +`POST /api/measurements`: one `BLOOD_PRESSURE_SYS`, one +`BLOOD_PRESSURE_DIA`, both carrying the same `Idempotency-Key`. Until the +replay cache learned to compare request bodies, the server treated the second +request as a retry of the first. It answered with the first request's cached +`201` (header `X-Idempotent-Replay: true`) and never wrote the diastolic +value. The app showed success, so nobody noticed. + +The fix makes the cache replay only a request with the same body. A different +body under the same key now runs normally. Nothing recovers the values that +were never sent to the database: they exist only on the person's phone, if at +all. + +## Finding the affected readings + +Read-only. Run it against the HealthLog database (with the bundled compose +file: `docker compose exec db psql -U healthlog healthlog`). It lists every live systolic reading that has no +live diastolic reading for the same account at the same instant. + + + +```sql +SELECT u.username, + m.id, + m.measured_at, + m.value AS systolic, + m.source +FROM measurements m +JOIN users u ON u.id = m.user_id +WHERE m.type = 'BLOOD_PRESSURE_SYS' + AND m.deleted_at IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM measurements d + WHERE d.user_id = m.user_id + AND d.measured_at = m.measured_at + AND d.type = 'BLOOD_PRESSURE_DIA' + AND d.deleted_at IS NULL + ) +ORDER BY u.username, m.measured_at; +``` + + + +Readings from the phone app carry `source = 'MANUAL'`. Rows from a synced +source (Withings, Apple Health) that show up here are a different gap, not +this one: those sources write both halves themselves. + +## What to tell users + +There is nothing the server can repair: the diastolic value never reached +it. A person whose reading appears in the list can re-enter it: delete the +incomplete reading, then add the blood pressure again with both values and +the original date and time. From this release on, both halves are saved on +the first try, from any app version. diff --git a/prisma/migrations/0371_idempotency_request_fingerprint/migration.sql b/prisma/migrations/0371_idempotency_request_fingerprint/migration.sql new file mode 100644 index 000000000..dcb360852 --- /dev/null +++ b/prisma/migrations/0371_idempotency_request_fingerprint/migration.sql @@ -0,0 +1,8 @@ +-- What request a replay-cache cell answered, as a SHA-256 of the canonical +-- request body scoped by method and path. A request that reuses a key with a +-- different body is executed instead of being handed the first body's cached +-- response. The body itself is never stored. Nullable: rows written before +-- this release carry none, and a null never matches, so they stop replaying +-- for anything but their own retries already in flight (they expire within +-- 24 hours either way). +ALTER TABLE "idempotency_keys" ADD COLUMN IF NOT EXISTS "request_fingerprint" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6a05a5659..9250aef3f 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5545,15 +5545,19 @@ model ProviderHealth { /// (userId, key, method, path) so retries within the TTL replay the /// original outcome instead of producing a duplicate side-effect. model IdempotencyKey { - id String @id @default(cuid()) - userId String @map("user_id") - key String - method String - path String - responseStatus Int @map("response_status") - responseBody String @map("response_body") // serialized JSON envelope - expiresAt DateTime @map("expires_at") - createdAt DateTime @default(now()) @map("created_at") + id String @id @default(cuid()) + userId String @map("user_id") + key String + method String + path String + responseStatus Int @map("response_status") + responseBody String @map("response_body") // serialized JSON envelope + /// SHA-256 of the canonical request body, scoped by method and path. A + /// lookup replays only when it matches; a different body under the same key + /// runs normally. Null on rows written before it existed, which never match. + requestFingerprint String? @map("request_fingerprint") + expiresAt DateTime @map("expires_at") + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) diff --git a/src/app/api/anamnesis/facts/__tests__/idempotency-conflict.test.ts b/src/app/api/anamnesis/facts/__tests__/idempotency-conflict.test.ts index adb31bbdc..dc93fd8e2 100644 --- a/src/app/api/anamnesis/facts/__tests__/idempotency-conflict.test.ts +++ b/src/app/api/anamnesis/facts/__tests__/idempotency-conflict.test.ts @@ -58,10 +58,13 @@ vi.mock("@/lib/profile/health-facts", () => ({ import { NextRequest } from "next/server"; import { POST } from "../route"; +import { requestFingerprint } from "@/lib/idempotency"; /** The pending sentinel `findCached` recognises as "another call is running". */ const PENDING_STATUS = 0; +const BODY = JSON.stringify({ kind: "SMOKING_STATUS", value: "FORMER" }); + function request(): NextRequest { return new NextRequest("http://localhost/api/anamnesis/facts", { method: "POST", @@ -69,7 +72,7 @@ function request(): NextRequest { "content-type": "application/json", "idempotency-key": "key-in-flight", }, - body: JSON.stringify({ kind: "SMOKING_STATUS", value: "FORMER" }), + body: BODY, }); } @@ -99,6 +102,12 @@ describe("idempotency 409 envelope, over the real route", () => { it("serves `error` as a string when a request is already in flight", async () => { mocks.findUnique.mockResolvedValue({ id: "idem-1", + // The same request in flight: a different body would run instead. + requestFingerprint: requestFingerprint( + "POST", + "/api/anamnesis/facts", + new TextEncoder().encode(BODY), + ), responseStatus: PENDING_STATUS, responseBody: "", expiresAt: new Date(Date.now() + 60_000), diff --git a/src/lib/__tests__/idempotency.test.ts b/src/lib/__tests__/idempotency.test.ts index 61494372e..a93a23ee1 100644 --- a/src/lib/__tests__/idempotency.test.ts +++ b/src/lib/__tests__/idempotency.test.ts @@ -49,6 +49,8 @@ import { withIdempotency, defaultUserIdResolver, isCachableStatus, + canonicalJson, + requestFingerprint, } from "../idempotency"; import { prisma } from "@/lib/db"; import { getSession } from "@/lib/auth/session"; @@ -81,6 +83,13 @@ function makeRequest( }); } +/** The fingerprint `makeRequest("POST", …)` produces. */ +const OK_BODY_FINGERPRINT = requestFingerprint( + "POST", + "/api/example", + new TextEncoder().encode(JSON.stringify({ ok: true })), +); + beforeEach(() => { vi.resetAllMocks(); vi.mocked(prisma.idempotencyKey.findUnique).mockResolvedValue(null); @@ -144,6 +153,11 @@ describe("withIdempotency", () => { data: { responseBody: string; responseStatus: number }; } ).data; + const claimedFingerprint = ( + vi.mocked(prisma.idempotencyKey.create).mock.calls[0][0] as { + data: { requestFingerprint: string }; + } + ).data.requestFingerprint; // Second call: cache hit returns persisted envelope. vi.mocked(prisma.idempotencyKey.findUnique).mockResolvedValueOnce({ @@ -152,6 +166,7 @@ describe("withIdempotency", () => { key: "abc-12345678", method: "POST", path: "/api/example", + requestFingerprint: claimedFingerprint, responseStatus: persistedBody.responseStatus, responseBody: persistedBody.responseBody, expiresAt: new Date(Date.now() + 86_400_000), @@ -296,6 +311,7 @@ describe("withIdempotency concurrency claim (A3)", () => { key: "abc-12345678", method: "POST", path: "/api/example", + requestFingerprint: OK_BODY_FINGERPRINT, responseStatus: 0, responseBody: "", expiresAt: new Date(Date.now() + 60_000), @@ -928,3 +944,195 @@ describe("the record-session fence sits above the replay cache", () => { expect(handler).toHaveBeenCalledTimes(1); }); }); + +describe("request fingerprint", () => { + const enc = (text: string) => new TextEncoder().encode(text); + + it("sorts object keys recursively and keeps array order", () => { + expect(canonicalJson({ b: 1, a: { d: [3, 1], c: null } })).toBe( + '{"a":{"c":null,"d":[3,1]},"b":1}', + ); + expect(canonicalJson([{ y: 1, x: 2 }, "s"])).toBe('[{"x":2,"y":1},"s"]'); + }); + + it("treats the same JSON object in a different key order as one request", () => { + const a = requestFingerprint("POST", "/p", enc('{"type":"X","value":1}')); + const b = requestFingerprint( + "POST", + "/p", + enc('{ "value": 1, "type": "X" }'), + ); + expect(a).toBe(b); + expect(a).toMatch(/^[0-9a-f]{64}$/); + }); + + it("tells nested values and array order apart", () => { + const base = requestFingerprint("POST", "/p", enc('{"a":{"b":[1,2]}}')); + expect(requestFingerprint("POST", "/p", enc('{"a":{"b":[2,1]}}'))).not.toBe( + base, + ); + expect(requestFingerprint("POST", "/p", enc('{"a":{"b":[1,3]}}'))).not.toBe( + base, + ); + }); + + it("tells the two halves of a blood pressure apart", () => { + const sys = requestFingerprint( + "POST", + "/api/measurements", + enc('{"type":"BLOOD_PRESSURE_SYS","value":120}'), + ); + const dia = requestFingerprint( + "POST", + "/api/measurements", + enc('{"type":"BLOOD_PRESSURE_DIA","value":80}'), + ); + expect(sys).not.toBe(dia); + }); + + it("hashes a non-JSON or empty body as its raw bytes", () => { + const text = requestFingerprint("POST", "/p", enc("not json")); + expect(text).toBe(requestFingerprint("POST", "/p", enc("not json"))); + expect(text).not.toBe(requestFingerprint("POST", "/p", enc("not json!"))); + const empty = requestFingerprint("DELETE", "/p", new Uint8Array()); + expect(empty).toBe(requestFingerprint("DELETE", "/p", new Uint8Array())); + expect(empty).not.toBe(text); + // Invalid UTF-8 is raw bytes, not an error. + expect(() => + requestFingerprint("POST", "/p", new Uint8Array([0xff, 0xfe])), + ).not.toThrow(); + }); + + it("is scoped by method and path", () => { + const body = enc('{"a":1}'); + const base = requestFingerprint("POST", "/p", body); + expect(requestFingerprint("PUT", "/p", body)).not.toBe(base); + expect(requestFingerprint("POST", "/q", body)).not.toBe(base); + }); +}); + +describe("withIdempotency with a different body under one key", () => { + function postWith(body: unknown): NextRequest { + return new NextRequest("http://localhost/api/example", { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": "abc-12345678", + }, + body: JSON.stringify(body), + }); + } + + function liveRow(fingerprint: string | null, responseStatus: number) { + return { + id: "idem-1", + userId: "u-1", + key: "abc-12345678", + method: "POST", + path: "/api/example", + requestFingerprint: fingerprint, + responseStatus, + responseBody: "", + expiresAt: new Date(Date.now() + 60_000), + createdAt: new Date(), + } as never; + } + + it.each([ + ["a completed row for another body", "f".repeat(64), 201], + ["an in-flight claim for another body", "f".repeat(64), 0], + ["a legacy row without a fingerprint", null, 201], + ])( + "runs the handler without claiming or caching over %s", + async (_label, fingerprint, status) => { + vi.mocked(prisma.idempotencyKey.findUnique).mockResolvedValueOnce( + liveRow(fingerprint, status), + ); + const handler = vi.fn(async () => + NextResponse.json({ data: "dia", error: null }, { status: 201 }), + ); + const wrapped = withIdempotency<[NextRequest]>( + handler, + async () => "u-1", + ); + const res = await wrapped(postWith({ type: "BLOOD_PRESSURE_DIA" })); + + expect(res.status).toBe(201); + expect(res.headers.get("X-Idempotent-Replay")).toBeNull(); + expect(handler).toHaveBeenCalledTimes(1); + expect(prisma.idempotencyKey.create).not.toHaveBeenCalled(); + expect(prisma.idempotencyKey.updateMany).not.toHaveBeenCalled(); + expect(prisma.idempotencyKey.deleteMany).not.toHaveBeenCalled(); + expect(prisma.idempotencyKey.delete).not.toHaveBeenCalled(); + }, + ); + + it("runs the handler when a racing claim for another body wins the insert", async () => { + vi.mocked(prisma.idempotencyKey.create).mockRejectedValueOnce( + Object.assign(new Error("unique"), { code: "P2002" }), + ); + // First lookup: nothing yet. Second, after the collision: the other body. + vi.mocked(prisma.idempotencyKey.findUnique) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(liveRow("f".repeat(64), 0)); + const handler = vi.fn(async () => + NextResponse.json({ data: "dia", error: null }, { status: 201 }), + ); + const wrapped = withIdempotency<[NextRequest]>(handler, async () => "u-1"); + const res = await wrapped(postWith({ type: "BLOOD_PRESSURE_DIA" })); + + expect(res.status).toBe(201); + expect(handler).toHaveBeenCalledTimes(1); + expect(prisma.idempotencyKey.updateMany).not.toHaveBeenCalled(); + }); + + it("stores the fingerprint, never the body, on the claim", async () => { + const handler = vi.fn(async () => + NextResponse.json({ data: "ok", error: null }, { status: 201 }), + ); + const wrapped = withIdempotency<[NextRequest]>(handler, async () => "u-1"); + await wrapped(postWith({ type: "BLOOD_PRESSURE_SYS", value: 123 })); + + const claim = ( + vi.mocked(prisma.idempotencyKey.create).mock.calls[0][0] as { + data: Record; + } + ).data; + expect(claim.requestFingerprint).toBe( + requestFingerprint( + "POST", + "/api/example", + new TextEncoder().encode( + JSON.stringify({ type: "BLOOD_PRESSURE_SYS", value: 123 }), + ), + ), + ); + expect(JSON.stringify(claim)).not.toContain("BLOOD_PRESSURE_SYS"); + }); + + it("leaves the body readable for the handler", async () => { + const handler = vi.fn(async (req: NextRequest) => + NextResponse.json({ data: await req.json(), error: null }), + ); + const wrapped = withIdempotency<[NextRequest]>(handler, async () => "u-1"); + const res = await wrapped(postWith({ type: "WEIGHT", value: 70 })); + expect((await res.json()).data).toEqual({ type: "WEIGHT", value: 70 }); + }); + + it("reads at most a bounded prefix of a large body and still hands the whole body on", async () => { + const big = "x".repeat(3 * 1024 * 1024); + const handler = vi.fn(async (req: NextRequest) => + NextResponse.json({ data: (await req.text()).length, error: null }), + ); + const wrapped = withIdempotency<[NextRequest]>(handler, async () => "u-1"); + const res = await wrapped( + new NextRequest("http://localhost/api/example", { + method: "POST", + headers: { "idempotency-key": "abc-12345678" }, + body: big, + }), + ); + expect((await res.json()).data).toBe(big.length); + expect(prisma.idempotencyKey.create).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/idempotency.ts b/src/lib/idempotency.ts index 70e2ddcbc..82d2cc0b6 100644 --- a/src/lib/idempotency.ts +++ b/src/lib/idempotency.ts @@ -11,6 +11,19 @@ * the record the request claims is folded into `key` (see `cellKey`), so the * tuple identifies a request to one person's record rather than a request by * one person. A caller acting only as themselves is unaffected, byte for byte. + * + * A cell also remembers WHICH request it answered: a SHA-256 of the canonical + * request body (`requestFingerprint`). A key names one request, not a slot, and + * a client that reuses a key for a second, different request must not be + * handed the first request's response. Native clients up to 1.1.0 sent the two + * halves of a manual blood pressure as two POSTs under one key; the second half + * replayed the first's 201 and was never written. A lookup therefore replays, + * or reports in-flight, only for the same body. A different body — or a row + * written before fingerprints existed — runs the handler normally without + * claiming or caching anything, and the existing cell stays for its own + * retries. This is deliberately not the IETF draft's 422: refusing would turn + * every affected client's second write into a visible error with no way for + * the person to recover the value. */ import type { NextRequest } from "next/server"; import { headers } from "next/headers"; @@ -109,6 +122,122 @@ export interface IdempotencyContext { key: string; method: string; path: string; + /** See `requestFingerprint`. Never the body itself. */ + fingerprint: string; +} + +/** + * How much of a body the fingerprint reads before it stops. The wrapper runs + * before the handler's own bounded readers and upload slots, so it must not + * buffer an arbitrarily large body on their behalf: reading the clone holds + * what it has read in the original's queue until the handler consumes it. + * Every JSON write the cache fronts is far below this; the one multipart route + * (document upload) is above it, and there the prefix plus the declared length + * identifies a retry of the same upload, with the route's own content hash as + * the duplicate check behind it. + */ +const FINGERPRINT_MAX_BYTES = 1024 * 1024; + +/** + * JSON with every object's keys sorted, recursively. Arrays keep their order — + * it is part of the request. Exported for its unit tests. + */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((v) => canonicalJson(v)).join(",")}]`; + } + if (value !== null && typeof value === "object") { + const entries = Object.keys(value) + .sort() + .map( + (k) => + `${JSON.stringify(k)}:${canonicalJson((value as Record)[k])}`, + ); + return `{${entries.join(",")}}`; + } + return JSON.stringify(value); +} + +/** + * The fingerprint of a request: SHA-256 hex over method, path and the body in + * canonical form. A body that parses as JSON is hashed as `canonicalJson`, so + * the same object serialised with a different key order is the same request; + * anything else (multipart, plain text, an empty body) is hashed as its raw + * bytes. `truncatedAt` marks a body cut at `FINGERPRINT_MAX_BYTES`, hashed as + * that raw prefix together with the declared length. Exported for unit tests. + */ +export function requestFingerprint( + method: string, + path: string, + body: Uint8Array, + truncatedAt: string | null = null, +): string { + const hash = createHash("sha256") + .update(method) + .update("\0") + .update(path) + .update("\0"); + if (truncatedAt !== null) { + return hash.update(`partial:${truncatedAt}\0`).update(body).digest("hex"); + } + if (body.length > 0) { + try { + const text = new TextDecoder("utf-8", { fatal: true }).decode(body); + return hash + .update("json:") + .update(canonicalJson(JSON.parse(text))) + .digest("hex"); + } catch { + // Not UTF-8 or not JSON — fall through to the raw bytes. + } + } + return hash.update("raw:").update(body).digest("hex"); +} + +/** + * Read the request's body from a clone, at most `FINGERPRINT_MAX_BYTES`, and + * fingerprint it. The original request keeps its body for the handler. + */ +async function fingerprintRequest( + request: Request | NextRequest, + path: string, +): Promise { + const stream = request.clone().body; + if (!stream) + return requestFingerprint(request.method, path, new Uint8Array()); + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + let truncated = false; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (total + value.byteLength > FINGERPRINT_MAX_BYTES) { + chunks.push(value.subarray(0, FINGERPRINT_MAX_BYTES - total)); + total = FINGERPRINT_MAX_BYTES; + truncated = true; + // Cancelling this branch of the tee stops it from queueing the rest of + // the body, and leaves the handler's branch intact. Not awaited: a tee + // branch's cancel settles only once BOTH branches are done, which is + // after the handler has run. + void reader.cancel().catch(() => {}); + break; + } + chunks.push(value); + total += value.byteLength; + } + const body = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return requestFingerprint( + request.method, + path, + body, + truncated ? (request.headers.get("content-length") ?? "unknown") : null, + ); } function getIdempotencyKey(request: Request | NextRequest): string | null { @@ -125,10 +254,16 @@ function getIdempotencyKey(request: Request | NextRequest): string | null { * - `{ kind: "pending" }` — another request holds an in-flight claim * for this key; the caller must NOT run the * handler again. + * - `{ kind: "mismatch" }` — the live row answered a different body (or + * predates fingerprints); the caller runs the + * handler without claiming or caching. * - `null` — no live row; the caller may claim + run. */ type CacheLookup = - { kind: "replay"; response: NextResponse } | { kind: "pending" } | null; + | { kind: "replay"; response: NextResponse } + | { kind: "pending" } + | { kind: "mismatch" } + | null; /** * Look up the state of a (userId, key, method, path) tuple. Distinguishes @@ -157,6 +292,13 @@ async function findCached(ctx: IdempotencyContext): Promise { return null; } + // Checked before both the pending and the replay arm: a different request + // under the same key is neither in flight nor answered. A legacy row's null + // never equals a fingerprint. + if (row.requestFingerprint !== ctx.fingerprint) { + return { kind: "mismatch" }; + } + if (row.responseStatus === PENDING_STATUS) { // A concurrent request claimed this key and is still running its // side-effect. Signal the caller to refuse rather than double-execute. @@ -208,6 +350,7 @@ async function claimKey(ctx: IdempotencyContext): Promise { key: ctx.key, method: ctx.method, path: ctx.path, + requestFingerprint: ctx.fingerprint, responseStatus: PENDING_STATUS, responseBody: "", expiresAt: new Date(Date.now() + PENDING_TTL_MS), @@ -233,6 +376,7 @@ async function releaseClaim(ctx: IdempotencyContext): Promise { key: ctx.key, method: ctx.method, path: ctx.path, + requestFingerprint: ctx.fingerprint, responseStatus: PENDING_STATUS, }, }) @@ -312,6 +456,7 @@ async function persistCached( key: ctx.key, method: ctx.method, path: ctx.path, + requestFingerprint: ctx.fingerprint, }, data: { responseStatus: response.status, @@ -502,6 +647,23 @@ async function narrowTokenFacet(): Promise { return apiToken.permissions.includes("*") ? null : `t:${apiToken.id}`; } +/** + * Run a request whose key is already taken by a different body: the handler + * runs exactly as it would without a key, and nothing is claimed or cached, so + * the existing cell keeps answering its own retries. + */ +function runUncached( + ctx: IdempotencyContext, + handler: (...args: Args) => Promise, + args: Args, +): Promise { + annotate({ + action: { name: "idempotency.key.body_mismatch" }, + meta: { method: ctx.method, path: ctx.path }, + }); + return handler(...args); +} + export function withIdempotency< Args extends [Request | NextRequest, ...unknown[]], >( @@ -589,14 +751,27 @@ export function withIdempotency< if (tokenFacet !== null) facets.push(tokenFacet); const url = new URL(request.url); + const fingerprint = await fingerprintRequest(request, url.pathname).catch( + () => null, + ); + if (fingerprint === null) { + // The body could not be read for the fingerprint (already consumed, or + // the stream failed). Without knowing which request this is, no cell + // may answer for it; the handler meets the body problem itself. + return handler(...args); + } const ctx: IdempotencyContext = { userId, key: cellKey(key, claimedRecord, facets), method: request.method, path: url.pathname, + fingerprint, }; const cached = await findCached(ctx); + if (cached?.kind === "mismatch") { + return runUncached(ctx, handler, args); + } if (cached?.kind === "replay") { return cached.response; } @@ -617,6 +792,13 @@ export function withIdempotency< // one caller may execute the side-effect for a given key. const won = await claimKey(ctx); if (!won) { + // The racing claim may be a different request under the same key — + // the two halves of one reading posted together. That one is not a + // duplicate of this one and must still run. + const raced = await findCached(ctx); + if (raced?.kind === "mismatch") { + return runUncached(ctx, handler, args); + } annotate({ action: { name: "idempotency.inflight_conflict" }, meta: { method: ctx.method, path: ctx.path }, diff --git a/src/lib/openapi/routes/shared.ts b/src/lib/openapi/routes/shared.ts index c96613bc0..831933630 100644 --- a/src/lib/openapi/routes/shared.ts +++ b/src/lib/openapi/routes/shared.ts @@ -391,7 +391,8 @@ export const profileEmailRateLimitResponse = { * * Scope, so nobody reads more into it than is there: only POST, PUT, PATCH and * DELETE are covered (`SUPPORTED_METHODS`), the key is scoped per user and per - * `(method, path)`, and `isCachableStatus` refuses to cache 5xx or the + * `(method, path)`, a replay needs the same body (a different body under the + * same key runs as a new write and is neither replayed nor refused), and `isCachableStatus` refuses to cache 5xx or the * transient 401 / 403 / 408 / 429 — a retry after one of those runs for real. * * `src/__tests__/idempotency-contract-publication.test.ts` holds this to the @@ -410,7 +411,7 @@ export const idempotencyKeyParameter = { maxLength: 128, }, description: - "Makes this write safe to retry. Send the same key again and the first response is replayed rather than the work repeated, marked with `X-Idempotent-Replay: true`. A key that does not match the pattern is IGNORED rather than refused — the write then proceeds unprotected and looks exactly like a request that carried no key, so validate the key on your side. Scoped per user and per (method, path): the same key on a different route is a different key. While a first request under this key is still in flight, a second is refused with 409 rather than queued.", + "Makes this write safe to retry. Send the same key with the same body again and the first response is replayed rather than the work repeated, marked with `X-Idempotent-Replay: true`. The body is compared as JSON with key order ignored; a different body under the same key is not a retry and runs as a new write, while the first response stays stored for retries of the first request. A key that does not match the pattern is IGNORED rather than refused — the write then proceeds unprotected and looks exactly like a request that carried no key, so validate the key on your side. Scoped per user and per (method, path): the same key on a different route is a different key. While a first request under this key is still in flight, a second with the same body is refused with 409 rather than queued.", }; /** diff --git a/tests/integration/idempotency-body-fingerprint.test.ts b/tests/integration/idempotency-body-fingerprint.test.ts new file mode 100644 index 000000000..c0d04d93c --- /dev/null +++ b/tests/integration/idempotency-body-fingerprint.test.ts @@ -0,0 +1,249 @@ +/** + * One `Idempotency-Key`, two different requests. + * + * Native clients up to 1.1.0 post a manual blood pressure as two + * `POST /api/measurements` calls — the systolic half, then the diastolic half — + * under the same key. While the replay cell was keyed on + * `(user, key, method, path)` alone, the second call was answered with the + * first call's cached 201 and the diastolic value was never written. The cell + * now carries a fingerprint of the request body: only the same body replays. + * + * Everything here is real — the shipped route, the shipped wrapper, the + * session resolver and the `idempotency_keys` unique index. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { POST as postMeasurement } from "@/app/api/measurements/route"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const USER_A = "idem-fingerprint-a"; +const USER_B = "idem-fingerprint-b"; +const KEY = "bp-entry-5c1e2a7f"; +const MEASURED_AT = "2026-09-30T07:15:00.000Z"; + +const SYS = { type: "BLOOD_PRESSURE_SYS", value: 128, measuredAt: MEASURED_AT }; +const DIA = { type: "BLOOD_PRESSURE_DIA", value: 84, measuredAt: MEASURED_AT }; + +const sessions = new Map(); + +function post(body: unknown, key = KEY): Promise { + return postMeasurement( + new NextRequest("http://localhost/api/measurements", { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": key, + }, + body: JSON.stringify(body), + }), + ); +} + +function signInAs(userId: string) { + cookieJar.set("healthlog_session", sessions.get(userId)!); +} + +async function rowsOf(userId: string) { + const rows = await getPrismaClient().measurement.findMany({ + where: { userId, deletedAt: null }, + select: { type: true, value: true }, + }); + // By name, not by the enum's declaration order. + return rows.sort((a, b) => a.type.localeCompare(b.type)); +} + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + cookieJar.clear(); + headerJar.clear(); + sessions.clear(); + for (const id of [USER_A, USER_B]) { + await prisma.user.create({ + data: { + id, + username: id, + email: `${id}@example.test`, + timezone: "UTC", + }, + }); + const session = await prisma.session.create({ + data: { userId: id, expiresAt: new Date(Date.now() + 60 * 60 * 1000) }, + }); + sessions.set(id, session.id); + } + signInAs(USER_A); +}); + +describe("Idempotency-Key reused for a different body (real Postgres)", () => { + it("writes both halves of a blood pressure posted under one key", async () => { + const first = await post(SYS); + expect(first.status).toBe(201); + + const second = await post(DIA); + expect(second.status).toBe(201); + expect(second.headers.get("X-Idempotent-Replay")).toBeNull(); + + expect(await rowsOf(USER_A)).toEqual([ + { type: "BLOOD_PRESSURE_DIA", value: 84 }, + { type: "BLOOD_PRESSURE_SYS", value: 128 }, + ]); + }); + + it("still replays a retry of the same body and writes one row", async () => { + const first = await post(SYS); + expect(first.status).toBe(201); + const firstBody = await first.json(); + + // Same object, different key order: the same request. + const retry = await post({ + measuredAt: MEASURED_AT, + value: 128, + type: "BLOOD_PRESSURE_SYS", + }); + expect(retry.status).toBe(201); + expect(retry.headers.get("X-Idempotent-Replay")).toBe("true"); + expect(await retry.json()).toEqual(firstBody); + + expect(await rowsOf(USER_A)).toEqual([ + { type: "BLOOD_PRESSURE_SYS", value: 128 }, + ]); + }); + + it("keeps the first cell answering its own retries after a different body ran", async () => { + await post(SYS); + await post(DIA); + const retry = await post(SYS); + expect(retry.headers.get("X-Idempotent-Replay")).toBe("true"); + expect(await rowsOf(USER_A)).toHaveLength(2); + const cells = await getPrismaClient().idempotencyKey.findMany({ + where: { userId: USER_A }, + }); + expect(cells).toHaveLength(1); + expect(cells[0]?.requestFingerprint).toMatch(/^[0-9a-f]{64}$/); + }); + + it("writes both halves when they are in flight at the same time", async () => { + const [a, b] = await Promise.all([post(SYS), post(DIA)]); + expect([a.status, b.status]).toEqual([201, 201]); + expect(await rowsOf(USER_A)).toEqual([ + { type: "BLOOD_PRESSURE_DIA", value: 84 }, + { type: "BLOOD_PRESSURE_SYS", value: 128 }, + ]); + }); + + it("still lets only one of two identical bodies in flight run", async () => { + const [a, b] = await Promise.all([post(SYS), post(SYS)]); + expect([a.status, b.status].sort()).toEqual([201, 409]); + expect(await rowsOf(USER_A)).toHaveLength(1); + }); + + it("does not replay a row written before fingerprints existed", async () => { + await getPrismaClient().idempotencyKey.create({ + data: { + userId: USER_A, + key: KEY, + method: "POST", + path: "/api/measurements", + responseStatus: 201, + responseBody: JSON.stringify({ data: { id: "legacy" }, error: null }), + expiresAt: new Date(Date.now() + 60 * 60 * 1000), + }, + }); + + const res = await post(DIA); + expect(res.status).toBe(201); + expect(res.headers.get("X-Idempotent-Replay")).toBeNull(); + expect(await rowsOf(USER_A)).toEqual([ + { type: "BLOOD_PRESSURE_DIA", value: 84 }, + ]); + // The legacy row is left alone, not claimed over. + const cell = await getPrismaClient().idempotencyKey.findFirstOrThrow({ + where: { userId: USER_A, key: KEY }, + }); + expect(cell.requestFingerprint).toBeNull(); + }); + + it("never hands one user's cached response to another using the same key", async () => { + const first = await post(SYS); + const firstBody = await first.json(); + + signInAs(USER_B); + const other = await post(SYS); + expect(other.status).toBe(201); + expect(other.headers.get("X-Idempotent-Replay")).toBeNull(); + expect(await other.json()).not.toEqual(firstBody); + + expect(await rowsOf(USER_A)).toHaveLength(1); + expect(await rowsOf(USER_B)).toHaveLength(1); + }); + + it("the documented operator query finds a systolic reading left without its diastolic half", async () => { + const doc = readFileSync( + join(process.cwd(), "docs/ops/blood-pressure-missing-diastolic.md"), + "utf8", + ); + const block = + /([\s\S]*?)/.exec( + doc, + ); + const sql = block?.[1] + .replace(/```sql|```/g, "") + .trim() + .replace(/;$/, ""); + expect(sql).toBeTruthy(); + + // A complete reading through the fixed path, and the damage the old + // cache left behind: a systolic half on its own. + await post(SYS); + await post(DIA); + const lone = await getPrismaClient().measurement.create({ + data: { + userId: USER_A, + type: "BLOOD_PRESSURE_SYS", + value: 141, + unit: "mmHg", + measuredAt: new Date("2026-09-29T19:40:00.000Z"), + }, + }); + + const found = await getPrismaClient().$queryRawUnsafe< + Array<{ id: string; systolic: number }> + >(sql!); + expect(found.map((r) => [r.id, r.systolic])).toEqual([[lone.id, 141]]); + }); +}); From 2798cc953fc5d32497ad178c7cb4a1e19e6fd878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:18:16 +0200 Subject: [PATCH 09/21] refactor(workouts): drop the unused clock arguments of the blank draft --- src/components/workouts/manual-workout-form.tsx | 2 +- src/lib/workouts/__tests__/manual-entry.test.ts | 2 +- src/lib/workouts/manual-entry.ts | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/components/workouts/manual-workout-form.tsx b/src/components/workouts/manual-workout-form.tsx index c849056e4..4455c83ae 100644 --- a/src/components/workouts/manual-workout-form.tsx +++ b/src/components/workouts/manual-workout-form.tsx @@ -116,7 +116,7 @@ export function ManualWorkoutForm({ // a submit after an edit updates the row the first submit stored. const [externalId] = useState(newManualWorkoutExternalId); const [draft, setDraft] = useState(() => - emptyManualWorkoutDraft(new Date(), timezone), + emptyManualWorkoutDraft(), ); const [errors, setErrors] = useState< Partial> diff --git a/src/lib/workouts/__tests__/manual-entry.test.ts b/src/lib/workouts/__tests__/manual-entry.test.ts index 0f728fb10..001142455 100644 --- a/src/lib/workouts/__tests__/manual-entry.test.ts +++ b/src/lib/workouts/__tests__/manual-entry.test.ts @@ -242,7 +242,7 @@ describe("buildManualWorkoutEntry — validation", () => { describe("the form's defaults", () => { it("opens with a blank start; the field caps at the current minute", () => { expect(wallClockNow(NOW, "Europe/Berlin")).toBe("2026-09-15T14:00"); - expect(emptyManualWorkoutDraft(NOW, "Asia/Seoul")).toMatchObject({ + expect(emptyManualWorkoutDraft()).toMatchObject({ sportType: "", start: "", hours: "", diff --git a/src/lib/workouts/manual-entry.ts b/src/lib/workouts/manual-entry.ts index 6be35a294..d672ae8ea 100644 --- a/src/lib/workouts/manual-entry.ts +++ b/src/lib/workouts/manual-entry.ts @@ -118,10 +118,7 @@ export function wallClockNow(now: Date, timezone: string): string { return `${p.year}-${pad(p.month)}-${pad(p.day)}T${pad(p.hour)}:${pad(p.minute)}`; } -export function emptyManualWorkoutDraft( - now: Date, - timezone: string, -): ManualWorkoutDraft { +export function emptyManualWorkoutDraft(): ManualWorkoutDraft { return { sportType: "", // Blank on purpose (#1085): a blank start means "it just ended", so the From 8ac3fddc3b9b6e9feafbff616b7835d87aee8824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:19:31 +0200 Subject: [PATCH 10/21] Carry visit and vaccination free text readable in a portable backup A practitioner's note, a visit's reason, outcome and body site and a vaccination's note travelled as stored ciphertext in a portable file, the same defect the cycle and mood sections had. They now go through the shared sealed-text helpers: readable in a portable file, sealed under the receiving host's key on restore, ciphertext kept only where this host opens it and otherwise reported as visitCiphertext or vaccinationCiphertext. The practitioner address and phone ciphertext of a disaster-recovery file gets the same check, since the key preflight now leaves these sections to it. --- docs/api/openapi.yaml | 20 ++ messages/de.json | 4 +- messages/en.json | 4 +- messages/es.json | 4 +- messages/fr.json | 4 +- messages/it.json | 4 +- messages/ko.json | 4 +- messages/pl.json | 4 +- src/components/admin/backups-section.tsx | 6 + src/lib/export/backup-key-ids.ts | 10 +- src/lib/export/restore-skips.ts | 10 +- src/lib/export/sealed-text.ts | 72 ++++- src/lib/export/vaccinations-backup.ts | 48 +-- src/lib/export/visits-backup.ts | 140 ++++++--- src/lib/validations/backup.ts | 12 +- .../backup-visits-cross-key-restore.test.ts | 296 ++++++++++++++++++ .../integration/encounter-procedures.test.ts | 5 +- 17 files changed, 559 insertions(+), 88 deletions(-) create mode 100644 tests/integration/backup-visits-cross-key-restore.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 9f672d83b..59f2766f1 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -38448,6 +38448,10 @@ components: - type: string minLength: 1 - type: "null" + note: + anyOf: + - type: string + - type: "null" createdAt: type: string minLength: 1 @@ -38515,6 +38519,18 @@ components: - type: string minLength: 1 - type: "null" + reason: + anyOf: + - type: string + - type: "null" + outcome: + anyOf: + - type: string + - type: "null" + bodySite: + anyOf: + - type: string + - type: "null" laterality: anyOf: - type: string @@ -38671,6 +38687,10 @@ components: - type: string minLength: 1 - type: "null" + note: + anyOf: + - type: string + - type: "null" createdAt: type: string minLength: 1 diff --git a/messages/de.json b/messages/de.json index d50600800..11af9ce58 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7556,7 +7556,9 @@ "previewKeyMissing": "Diese Sicherung lässt sich auf diesem Server nicht wiederherstellen: Sie braucht den Verschlüsselungsschlüssel {keys}. Nimm den Schlüssel zuerst wieder in ENCRYPTION_KEYS auf und starte neu.", "restoreSkippedSymptomEpisodeReference": "Symptom-Eintrag mit Verknüpfung zu einer Erkrankung, die die Datei nicht enthält (ohne Verknüpfung behalten)", "restoreSkippedCycleCiphertext": "Zyklus-Notiz oder -Bezeichnung mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (Tag oder Symptom ohne sie behalten)", - "restoreSkippedMoodLabelCiphertext": "Name eines Stimmungs-Tags oder einer Gruppe mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (ohne Namen behalten)" + "restoreSkippedMoodLabelCiphertext": "Name eines Stimmungs-Tags oder einer Gruppe mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (ohne Namen behalten)", + "restoreSkippedVisitCiphertext": "Text zu Arztbesuch oder Praxis mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (ohne ihn behalten)", + "restoreSkippedVaccinationCiphertext": "Impf-Notiz mit einem Schlüssel verschlüsselt, den dieser Server nicht hat (Impfung ohne sie behalten)" }, "danger-zone": { "title": "Gefahrenzone", diff --git a/messages/en.json b/messages/en.json index b2a3d92c6..e1d6f6fed 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7556,7 +7556,9 @@ "previewKeyMissing": "This backup cannot be restored on this server: it needs encryption key {keys}. Add the key back to ENCRYPTION_KEYS and restart first.", "restoreSkippedSymptomEpisodeReference": "Symptom entry linked to an illness the file does not contain (kept without the link)", "restoreSkippedCycleCiphertext": "Cycle note or label sealed with a key this server does not hold (day or symptom kept without it)", - "restoreSkippedMoodLabelCiphertext": "Mood tag or group name sealed with a key this server does not hold (kept without its name)" + "restoreSkippedMoodLabelCiphertext": "Mood tag or group name sealed with a key this server does not hold (kept without its name)", + "restoreSkippedVisitCiphertext": "Visit or practitioner text sealed with a key this server does not hold (kept without it)", + "restoreSkippedVaccinationCiphertext": "Vaccination note sealed with a key this server does not hold (dose kept without it)" }, "danger-zone": { "title": "Danger Zone", diff --git a/messages/es.json b/messages/es.json index daf60d074..a5b8de087 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7556,7 +7556,9 @@ "previewKeyMissing": "Esta copia no se puede restaurar en este servidor: necesita la clave de cifrado {keys}. Vuelve a añadir la clave a ENCRYPTION_KEYS y reinicia primero.", "restoreSkippedSymptomEpisodeReference": "Entrada de síntoma vinculada a una enfermedad que el archivo no contiene (se conserva sin el vínculo)", "restoreSkippedCycleCiphertext": "Nota o etiqueta del ciclo cifrada con una clave que este servidor no tiene (el día o el síntoma se conserva sin ella)", - "restoreSkippedMoodLabelCiphertext": "Nombre de etiqueta o grupo de ánimo cifrado con una clave que este servidor no tiene (se conserva sin el nombre)" + "restoreSkippedMoodLabelCiphertext": "Nombre de etiqueta o grupo de ánimo cifrado con una clave que este servidor no tiene (se conserva sin el nombre)", + "restoreSkippedVisitCiphertext": "Texto de visita o profesional cifrado con una clave que este servidor no tiene (se conserva sin él)", + "restoreSkippedVaccinationCiphertext": "Nota de vacunación cifrada con una clave que este servidor no tiene (la dosis se conserva sin ella)" }, "danger-zone": { "title": "Zona de peligro", diff --git a/messages/fr.json b/messages/fr.json index 29378a5a1..7e023dc5a 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7556,7 +7556,9 @@ "previewKeyMissing": "Cette sauvegarde ne peut pas être restaurée sur ce serveur : elle a besoin de la clé de chiffrement {keys}. Remettez d'abord la clé dans ENCRYPTION_KEYS et redémarrez.", "restoreSkippedSymptomEpisodeReference": "Entrée de symptôme liée à une maladie absente du fichier (conservée sans le lien)", "restoreSkippedCycleCiphertext": "Note ou libellé du cycle chiffré avec une clé que ce serveur ne possède pas (jour ou symptôme conservé sans)", - "restoreSkippedMoodLabelCiphertext": "Nom d’étiquette ou de groupe d’humeur chiffré avec une clé que ce serveur ne possède pas (conservé sans son nom)" + "restoreSkippedMoodLabelCiphertext": "Nom d’étiquette ou de groupe d’humeur chiffré avec une clé que ce serveur ne possède pas (conservé sans son nom)", + "restoreSkippedVisitCiphertext": "Texte de consultation ou de praticien chiffré avec une clé que ce serveur ne possède pas (conservé sans)", + "restoreSkippedVaccinationCiphertext": "Note de vaccination chiffrée avec une clé que ce serveur ne possède pas (dose conservée sans)" }, "danger-zone": { "title": "Zone de danger", diff --git a/messages/it.json b/messages/it.json index 00a8957cf..9e4a387e9 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7556,7 +7556,9 @@ "previewKeyMissing": "Questo backup non può essere ripristinato su questo server: richiede la chiave di cifratura {keys}. Aggiungi prima di nuovo la chiave a ENCRYPTION_KEYS e riavvia.", "restoreSkippedSymptomEpisodeReference": "Voce di sintomo collegata a una malattia assente dal file (mantenuta senza collegamento)", "restoreSkippedCycleCiphertext": "Nota o etichetta del ciclo cifrata con una chiave che questo server non possiede (giorno o sintomo mantenuto senza)", - "restoreSkippedMoodLabelCiphertext": "Nome di tag o gruppo dell’umore cifrato con una chiave che questo server non possiede (mantenuto senza nome)" + "restoreSkippedMoodLabelCiphertext": "Nome di tag o gruppo dell’umore cifrato con una chiave che questo server non possiede (mantenuto senza nome)", + "restoreSkippedVisitCiphertext": "Testo di visita o professionista cifrato con una chiave che questo server non possiede (mantenuto senza)", + "restoreSkippedVaccinationCiphertext": "Nota di vaccinazione cifrata con una chiave che questo server non possiede (dose mantenuta senza)" }, "danger-zone": { "title": "Zona pericolosa", diff --git a/messages/ko.json b/messages/ko.json index 7a1991f25..9ef99ba16 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7535,7 +7535,9 @@ "previewKeyMissing": "이 서버에서는 이 백업을 복원할 수 없습니다. 암호화 키 {keys}가 필요합니다. 먼저 키를 ENCRYPTION_KEYS에 다시 추가하고 재시작하세요.", "restoreSkippedSymptomEpisodeReference": "파일에 없는 질병과 연결된 증상 기록(연결 없이 유지됨)", "restoreSkippedCycleCiphertext": "이 서버에 없는 키로 암호화된 주기 메모 또는 이름(해당 날짜나 증상은 그 내용 없이 유지됨)", - "restoreSkippedMoodLabelCiphertext": "이 서버에 없는 키로 암호화된 기분 태그 또는 그룹 이름(이름 없이 유지됨)" + "restoreSkippedMoodLabelCiphertext": "이 서버에 없는 키로 암호화된 기분 태그 또는 그룹 이름(이름 없이 유지됨)", + "restoreSkippedVisitCiphertext": "이 서버에 없는 키로 암호화된 진료 또는 의료진 텍스트(해당 내용 없이 유지됨)", + "restoreSkippedVaccinationCiphertext": "이 서버에 없는 키로 암호화된 예방접종 메모(접종 기록은 메모 없이 유지됨)" }, "danger-zone": { "title": "위험 구역", diff --git a/messages/pl.json b/messages/pl.json index d8cbfe8b5..d3c92bc44 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7556,7 +7556,9 @@ "previewKeyMissing": "Tej kopii nie da się przywrócić na tym serwerze: wymaga klucza szyfrowania {keys}. Najpierw dodaj klucz z powrotem do ENCRYPTION_KEYS i uruchom ponownie.", "restoreSkippedSymptomEpisodeReference": "Wpis objawu powiązany z chorobą, której nie ma w pliku (zachowany bez powiązania)", "restoreSkippedCycleCiphertext": "Notatka lub etykieta cyklu zaszyfrowana kluczem, którego ten serwer nie ma (dzień lub objaw zachowany bez niej)", - "restoreSkippedMoodLabelCiphertext": "Nazwa tagu lub grupy nastroju zaszyfrowana kluczem, którego ten serwer nie ma (zachowana bez nazwy)" + "restoreSkippedMoodLabelCiphertext": "Nazwa tagu lub grupy nastroju zaszyfrowana kluczem, którego ten serwer nie ma (zachowana bez nazwy)", + "restoreSkippedVisitCiphertext": "Tekst wizyty lub lekarza zaszyfrowany kluczem, którego ten serwer nie ma (zachowany bez niego)", + "restoreSkippedVaccinationCiphertext": "Notatka szczepienia zaszyfrowana kluczem, którego ten serwer nie ma (dawka zachowana bez niej)" }, "danger-zone": { "title": "Strefa zagrożenia", diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index 96f34bdd9..22b557d71 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -464,6 +464,12 @@ function catalogueLabel( if (catalogue === "symptomEpisodeReference") { return t("admin.section.backups.restoreSkippedSymptomEpisodeReference"); } + if (catalogue === "visitCiphertext") { + return t("admin.section.backups.restoreSkippedVisitCiphertext"); + } + if (catalogue === "vaccinationCiphertext") { + return t("admin.section.backups.restoreSkippedVaccinationCiphertext"); + } if (catalogue === "moodLabelCiphertext") { return t("admin.section.backups.restoreSkippedMoodLabelCiphertext"); } diff --git a/src/lib/export/backup-key-ids.ts b/src/lib/export/backup-key-ids.ts index c4113125e..88bb09ad6 100644 --- a/src/lib/export/backup-key-ids.ts +++ b/src/lib/export/backup-key-ids.ts @@ -101,9 +101,10 @@ export function isEncryptedMember(name: string): boolean { * Sections whose restore opens every inner value with this host's keys itself, * and keeps out (and names in the skip report) any value it cannot open. * - * The cycle day-logs, custom cycle symptoms and custom mood tags and mood - * categories are here because a portable file written before v1.40 carried - * their free text as the stored ciphertext, + * The cycle day-logs, custom cycle symptoms, custom mood tags and mood + * categories, practitioners, visits and vaccinations are here because a + * portable file written before v1.40 carried their free text as the stored + * ciphertext, * not as readable text. Such a file is still restorable onto another host: * what that host can open comes back, what it cannot stays out of the row and * is reported. Refusing the whole file over a note would cost every other @@ -117,6 +118,9 @@ export const RESTORE_SELF_VERIFIED_SECTIONS: ReadonlySet = new Set([ "customSymptoms", "customMoodTags", "customMoodTagCategories", + "practitioners", + "encounters", + "vaccinations", ]); interface KeyUse { diff --git a/src/lib/export/restore-skips.ts b/src/lib/export/restore-skips.ts index e292b78e4..c8bb1fd92 100644 --- a/src/lib/export/restore-skips.ts +++ b/src/lib/export/restore-skips.ts @@ -165,6 +165,12 @@ * tag or mood category the account created * (`customMoodTags..labelEncrypted`): the tag or category restores, under * its key, without the person's own name for it. + * + * The eighteenth and nineteenth, `visitCiphertext` and `vaccinationCiphertext`, + * are the same again for the free text of the visits record (a practitioner's + * note, address and phone; a visit's reason, outcome and body site) and of a + * vaccination's note (`vaccinations..noteEncrypted`). The row restores + * without the field. */ export type SkippedCatalogue = | "cycleSymptom" @@ -187,7 +193,9 @@ export type SkippedCatalogue = | "accountSetting" | "symptomEpisodeReference" | "cycleCiphertext" - | "moodLabelCiphertext"; + | "moodLabelCiphertext" + | "visitCiphertext" + | "vaccinationCiphertext"; /** One key this instance does not know, and the links it cost. */ export interface SkippedCatalogueKey { diff --git a/src/lib/export/sealed-text.ts b/src/lib/export/sealed-text.ts index c1e28a3c7..4773d6186 100644 --- a/src/lib/export/sealed-text.ts +++ b/src/lib/export/sealed-text.ts @@ -1,8 +1,9 @@ /** - * Both ends of a sealed text column stored with the string codec - * (`.`), for the backup sections that carry one: the cycle - * day-log note and sensitive envelope, custom cycle symptom labels, and the - * custom mood tag and mood category labels. + * Both ends of a sealed free-text column in a backup section: the string + * codec (`.` in a text column — the cycle day-log note and + * sensitive envelope, custom cycle symptom and mood labels) and the shared + * bytes codec (the practitioner note, the visit's reason, outcome and body + * site, the vaccination note). * * The split is the one every other section uses. A disaster-recovery file * carries the stored ciphertext, which the same instance's key reads back. A @@ -18,6 +19,9 @@ * than store a value nobody can read. That per-value check is why the key * preflight leaves these sections out (`RESTORE_SELF_VERIFIED_SECTIONS`). */ +import { Buffer } from "node:buffer"; + +import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { decrypt, encrypt } from "@/lib/crypto"; import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker"; import { getEvent } from "@/lib/logging/context"; @@ -68,3 +72,63 @@ export function sealForRestore( } return readable ? encrypt(readable) : null; } + +/** + * The `Bytes` twin of {@link openSealedForExport}, for columns written with + * the shared bytes codec (`encryptToBytes`): the practitioner note, the + * visit's reason, outcome and body site, the vaccination note. + */ +export function openSealedBytesForExport( + sealed: Uint8Array | null, + field: string, +): string | null { + if (!sealed || sealed.byteLength === 0) return null; + try { + return decryptFromBytes(sealed); + } catch (err) { + getEvent()?.addWarning( + `${field} decrypt failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return UNREADABLE_EXPORT_MARKER; + } +} + +/** The stored `Bytes` ciphertext as a disaster-recovery file carries it. */ +export function encodeSealedBytes(value: Uint8Array | null): string | null { + if (!value || value.byteLength === 0) return null; + return Buffer.from(value).toString("base64"); +} + +function decodeBase64(encoded: string): Uint8Array { + const decoded = Buffer.from(encoded, "base64"); + const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength)); + bytes.set(decoded); + return bytes; +} + +/** + * The `Bytes` twin of {@link sealForRestore}. `sealed` is the base64 a + * disaster-recovery (or pre-v1.40 portable) file carries; it is written back + * only when this host opens it. A readable value is sealed under this host's + * active key. + */ +export function sealBytesForRestore( + sealed: string | null | undefined, + readable: string | null | undefined, + path: string, + unopened: string[], +): Uint8Array | null { + if (sealed) { + const bytes = decodeBase64(sealed); + try { + decryptFromBytes(bytes); + return bytes; + } catch { + unopened.push(path); + return null; + } + } + return readable ? encryptToBytes(readable) : null; +} diff --git a/src/lib/export/vaccinations-backup.ts b/src/lib/export/vaccinations-backup.ts index f88bcc151..2e15242b0 100644 --- a/src/lib/export/vaccinations-backup.ts +++ b/src/lib/export/vaccinations-backup.ts @@ -35,7 +35,11 @@ * guarantee `src/lib/vaccinations/__tests__/catalog-integrity.test.ts` * holds. */ -import { Buffer } from "node:buffer"; +import { + encodeSealedBytes, + openSealedBytesForExport, + sealBytesForRestore, +} from "@/lib/export/sealed-text"; import type { Prisma, PrismaClient } from "@/generated/prisma/client"; import type { VaccinationSite } from "@/generated/prisma/client"; @@ -72,8 +76,10 @@ export interface VaccinationBackupEntry { * lacks the reminder. */ reminderId: string | null; - /** Base64 ciphertext, carried verbatim — never decrypted into the file. */ - noteEncrypted: string | null; + /** Base64 ciphertext; disaster-recovery payloads only. */ + noteEncrypted?: string | null; + /** The note readable; portable payloads only. */ + note?: string | null; createdAt: string; updatedAt: string; deletedAt?: string | null; @@ -96,12 +102,6 @@ export interface VaccinationsBackupCounts { vaccinationLinks: number; } -/** Base64 for a `Bytes` ciphertext column, or null when the column is empty. */ -function encodeCiphertext(value: Uint8Array | null): string | null { - if (!value || value.byteLength === 0) return null; - return Buffer.from(value).toString("base64"); -} - /** * Named select constants, one per model — a structural matcher binds a model * to the literal beside its delegate call, and an inline object inside a @@ -172,7 +172,14 @@ export async function buildVaccinationsBackupSection( practitionerId: row.practitionerId, encounterId: row.encounterId, reminderId: row.reminderId, - noteEncrypted: encodeCiphertext(row.noteEncrypted), + ...(disasterRecovery + ? { noteEncrypted: encodeSealedBytes(row.noteEncrypted) } + : { + note: openSealedBytesForExport( + row.noteEncrypted, + "vaccination note", + ), + }), createdAt: row.createdAt.toISOString(), updatedAt: row.updatedAt.toISOString(), ...(disasterRecovery @@ -240,16 +247,10 @@ export type RestoredVaccination = Pick< | "encounterId" | "reminderId" | "noteEncrypted" + | "note" > > & { deletedAt?: string | null }; -function decodeCiphertext(encoded: string): Uint8Array { - const decoded = Buffer.from(encoded, "base64"); - const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength)); - bytes.set(decoded); - return bytes; -} - /** * Re-create the account's immunization log and its document links. * @@ -307,6 +308,8 @@ export async function restoreVaccinationsData( const droppedPractitioners: string[] = []; const droppedEncounters: string[] = []; const droppedReminders: string[] = []; + // Sealed notes this host's keys do not open, by file path. + const unopened: string[] = []; if (payload.vaccinations.length > 0) { await tx.vaccinationRecord.createMany({ @@ -349,10 +352,12 @@ export async function restoreVaccinationsData( practitionerId, encounterId, reminderId, - noteEncrypted: - entry.noteEncrypted == null - ? null - : decodeCiphertext(entry.noteEncrypted), + noteEncrypted: sealBytesForRestore( + entry.noteEncrypted, + entry.note, + `vaccinations.${entry.id}.noteEncrypted`, + unopened, + ), createdAt: new Date(entry.createdAt), updatedAt: new Date(entry.updatedAt), deletedAt: entry.deletedAt ? new Date(entry.deletedAt) : null, @@ -379,6 +384,7 @@ export async function restoreVaccinationsData( [...new Set(droppedReminders)], droppedReminders, ); + recordUnknownKeys(skips, "vaccinationCiphertext", unopened, unopened); const restoredVaccinations = new Set( payload.vaccinations.map((entry) => entry.id), diff --git a/src/lib/export/visits-backup.ts b/src/lib/export/visits-backup.ts index 32ebca5f4..2957bb947 100644 --- a/src/lib/export/visits-backup.ts +++ b/src/lib/export/visits-backup.ts @@ -33,7 +33,6 @@ * result ids, so a lab link in one cannot resolve; a disaster-recovery * payload carries every id and they all do. */ -import { Buffer } from "node:buffer"; import type { Prisma, PrismaClient } from "@/generated/prisma/client"; import type { EncounterKind, @@ -42,6 +41,11 @@ import type { } from "@/generated/prisma/client"; import { encryptNote } from "@/lib/crypto/note-cipher"; +import { + encodeSealedBytes, + openSealedBytesForExport, + sealBytesForRestore, +} from "@/lib/export/sealed-text"; import { readPractitionerContact } from "@/lib/practitioners/dto"; import { recordUnknownKeys, @@ -71,8 +75,10 @@ export interface PractitionerBackupEntry { locationEncrypted?: string | null; /** Base64 ciphertext of the phone number, disaster-recovery payloads only. */ phoneEncrypted?: string | null; - /** Base64 ciphertext, carried verbatim — never decrypted into the file. */ - noteEncrypted: string | null; + /** Base64 ciphertext; disaster-recovery payloads only. */ + noteEncrypted?: string | null; + /** The note readable; portable payloads only. */ + note?: string | null; createdAt: string; updatedAt: string; deletedAt?: string | null; @@ -87,10 +93,15 @@ export interface EncounterBackupEntry { status: EncounterStatus; kind: EncounterKind; practitionerId: string | null; - reasonEncrypted: string | null; - outcomeEncrypted: string | null; + /** Base64 ciphertext; disaster-recovery payloads only. */ + reasonEncrypted?: string | null; + outcomeEncrypted?: string | null; /** The procedure's body site, ciphertext like the two above (v1.39.1). */ - bodySiteEncrypted: string | null; + bodySiteEncrypted?: string | null; + /** The three readable; portable payloads only. */ + reason?: string | null; + outcome?: string | null; + bodySite?: string | null; laterality: Laterality | null; /** * The appointment's reminder row, which restores in the same run since @@ -124,12 +135,6 @@ export interface VisitsBackupCounts { encounterLinks: number; } -/** Base64 for a `Bytes` ciphertext column, or null when the column is empty. */ -function encodeCiphertext(value: Uint8Array | null): string | null { - if (!value || value.byteLength === 0) return null; - return Buffer.from(value).toString("base64"); -} - /** * Named select constants, one per model. * @@ -256,10 +261,10 @@ export async function buildVisitsBackupSection( ? { location: null, phone: null, - locationEncrypted: encodeCiphertext( + locationEncrypted: encodeSealedBytes( row.locationEncrypted ?? encryptNote(row.location), ), - phoneEncrypted: encodeCiphertext( + phoneEncrypted: encodeSealedBytes( row.phoneEncrypted ?? encryptNote(row.phone), ), } @@ -275,7 +280,14 @@ export async function buildVisitsBackupSection( "phone", ), }), - noteEncrypted: encodeCiphertext(row.noteEncrypted), + ...(disasterRecovery + ? { noteEncrypted: encodeSealedBytes(row.noteEncrypted) } + : { + note: openSealedBytesForExport( + row.noteEncrypted, + "practitioner note", + ), + }), createdAt: row.createdAt.toISOString(), updatedAt: row.updatedAt.toISOString(), ...(disasterRecovery @@ -288,9 +300,26 @@ export async function buildVisitsBackupSection( status: row.status, kind: row.kind, practitionerId: row.practitionerId, - reasonEncrypted: encodeCiphertext(row.reasonEncrypted), - outcomeEncrypted: encodeCiphertext(row.outcomeEncrypted), - bodySiteEncrypted: encodeCiphertext(row.bodySiteEncrypted), + ...(disasterRecovery + ? { + reasonEncrypted: encodeSealedBytes(row.reasonEncrypted), + outcomeEncrypted: encodeSealedBytes(row.outcomeEncrypted), + bodySiteEncrypted: encodeSealedBytes(row.bodySiteEncrypted), + } + : { + reason: openSealedBytesForExport( + row.reasonEncrypted, + "visit reason", + ), + outcome: openSealedBytesForExport( + row.outcomeEncrypted, + "visit outcome", + ), + bodySite: openSealedBytesForExport( + row.bodySiteEncrypted, + "visit body site", + ), + }), laterality: row.laterality, reminderId: row.reminderId, createdAt: row.createdAt.toISOString(), @@ -380,6 +409,7 @@ export type RestoredPractitioner = Pick< | "locationEncrypted" | "phoneEncrypted" | "noteEncrypted" + | "note" > > & { deletedAt?: string | null }; @@ -394,18 +424,14 @@ export type RestoredEncounter = Pick< | "reasonEncrypted" | "outcomeEncrypted" | "bodySiteEncrypted" + | "reason" + | "outcome" + | "bodySite" | "laterality" | "reminderId" > > & { deletedAt?: string | null }; -function decodeCiphertext(encoded: string): Uint8Array { - const decoded = Buffer.from(encoded, "base64"); - const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength)); - bytes.set(decoded); - return bytes; -} - /** * Re-create the account's visits, address book and link rows. * @@ -440,6 +466,9 @@ export async function restoreVisitsData( where: { userId: ownerId }, }); + // Sealed values this host's keys do not open, by file path. + const unopened: string[] = []; + if (payload.practitioners.length > 0) { await tx.practitioner.createMany({ data: payload.practitioners.map((entry) => ({ @@ -451,18 +480,24 @@ export async function restoreVisitsData( // v1.39.4 — ciphertext wins when a disaster-recovery file carried it; // a portable file's (or an older file's) readable value is sealed // under this instance's key. The readable columns are never written. - locationEncrypted: - entry.locationEncrypted != null - ? decodeCiphertext(entry.locationEncrypted) - : encryptNote(entry.location), - phoneEncrypted: - entry.phoneEncrypted != null - ? decodeCiphertext(entry.phoneEncrypted) - : encryptNote(entry.phone), - noteEncrypted: - entry.noteEncrypted == null - ? null - : decodeCiphertext(entry.noteEncrypted), + locationEncrypted: sealBytesForRestore( + entry.locationEncrypted, + entry.location, + `practitioners.${entry.id}.locationEncrypted`, + unopened, + ), + phoneEncrypted: sealBytesForRestore( + entry.phoneEncrypted, + entry.phone, + `practitioners.${entry.id}.phoneEncrypted`, + unopened, + ), + noteEncrypted: sealBytesForRestore( + entry.noteEncrypted, + entry.note, + `practitioners.${entry.id}.noteEncrypted`, + unopened, + ), createdAt: new Date(entry.createdAt), updatedAt: new Date(entry.updatedAt), deletedAt: entry.deletedAt ? new Date(entry.deletedAt) : null, @@ -517,20 +552,26 @@ export async function restoreVisitsData( status: entry.status, kind: entry.kind, practitionerId, - reasonEncrypted: - entry.reasonEncrypted == null - ? null - : decodeCiphertext(entry.reasonEncrypted), - outcomeEncrypted: - entry.outcomeEncrypted == null - ? null - : decodeCiphertext(entry.outcomeEncrypted), + reasonEncrypted: sealBytesForRestore( + entry.reasonEncrypted, + entry.reason, + `encounters.${entry.id}.reasonEncrypted`, + unopened, + ), + outcomeEncrypted: sealBytesForRestore( + entry.outcomeEncrypted, + entry.outcome, + `encounters.${entry.id}.outcomeEncrypted`, + unopened, + ), // Absent in a file written before v1.39.1: the visit comes back // with no site, which is what it had. - bodySiteEncrypted: - entry.bodySiteEncrypted == null - ? null - : decodeCiphertext(entry.bodySiteEncrypted), + bodySiteEncrypted: sealBytesForRestore( + entry.bodySiteEncrypted, + entry.bodySite, + `encounters.${entry.id}.bodySiteEncrypted`, + unopened, + ), laterality: entry.laterality ?? null, reminderId, createdAt: new Date(entry.createdAt), @@ -553,6 +594,7 @@ export async function restoreVisitsData( [...new Set(droppedReminders)], droppedReminders, ); + recordUnknownKeys(skips, "visitCiphertext", unopened, unopened); const restoredEncounters = new Set(payload.encounters.map((e) => e.id)); diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index a96fa893a..f0a31c31f 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -1207,8 +1207,9 @@ const symptomDefinitionBackupSchema = z * * `id` is required on both records, unlike the lab result above: a link * addresses an encounter and its far side by id and nothing else on either row - * is unique enough to rebuild the reference from. The ciphertext columns ride - * verbatim as base64 — a visit note is never decrypted into the file. + * is unique enough to rebuild the reference from. Free text follows the note + * contract: ciphertext as base64 on a disaster-recovery file, the readable + * value on a portable one (sealed under the receiving host's key on restore). */ const practitionerBackupSchema = z .object({ @@ -1222,6 +1223,7 @@ const practitionerBackupSchema = z locationEncrypted: base64BytesSchema.nullable().optional(), phoneEncrypted: base64BytesSchema.nullable().optional(), noteEncrypted: base64BytesSchema.nullable().optional(), + note: z.string().nullable().optional(), createdAt: isoDateTime, updatedAt: isoDateTime, deletedAt: isoDateTime.nullable().optional(), @@ -1240,6 +1242,10 @@ const encounterBackupSchema = z // v1.39.1 — the procedure's body site and side. Optional so a file written // before they existed still parses; the visit restores with neither. bodySiteEncrypted: base64BytesSchema.nullable().optional(), + // The readable reason, outcome and body site of a portable file. + reason: z.string().nullable().optional(), + outcome: z.string().nullable().optional(), + bodySite: z.string().nullable().optional(), laterality: z.enum(Laterality).nullable().optional(), // Remapped against the restored reminders (they travel since v1.37.20); // dropped to NULL, with the drop named, only when the file lacks the row. @@ -1281,7 +1287,9 @@ const vaccinationBackupSchema = z // Remapped against the restored reminders (they travel since v1.37.20); // dropped to NULL, with the drop named, only when the file lacks the row. reminderId: z.string().nullable().optional(), + // Ciphertext on a disaster-recovery file, readable on a portable one. noteEncrypted: base64BytesSchema.nullable().optional(), + note: z.string().nullable().optional(), createdAt: isoDateTime, updatedAt: isoDateTime, deletedAt: isoDateTime.nullable().optional(), diff --git a/tests/integration/backup-visits-cross-key-restore.test.ts b/tests/integration/backup-visits-cross-key-restore.test.ts new file mode 100644 index 000000000..234f6c136 --- /dev/null +++ b/tests/integration/backup-visits-cross-key-restore.test.ts @@ -0,0 +1,296 @@ +/** + * A portable file's visit and vaccination free text survives a move to a host + * with other keys. + * + * A practitioner's note, a visit's reason, outcome and body site and a + * vaccination's note are sealed under the writing host's key. A portable file + * used to carry them as the stored ciphertext, so a restore onto a host with + * a different key refused the file at the key check, or with a matching key id + * over other key material wrote values nobody could open. + * + * Both ends and the pipe: built by the real `buildFullBackupPayload` under key + * A, restored through the real route under key B, opened under key B. The + * second case is a file written before the fix: a value this host opens comes + * back, one it cannot open stays out and is named in the skip report. + */ +import { Buffer } from "node:buffer"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec"; +import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto"; +import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { parseBackupPayload } from "@/lib/validations/backup"; +import { POST } from "./restore-job-driver"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserData: vi.fn(), +})); + +const saved = { + key: process.env.ENCRYPTION_KEY, + keys: process.env.ENCRYPTION_KEYS, + active: process.env.ENCRYPTION_ACTIVE_KEY_ID, +}; + +/** Make this process the host holding exactly `keys`, writing under `active`. */ +function useKeys(keys: Record, active: string) { + process.env.ENCRYPTION_KEY = ""; + process.env.ENCRYPTION_KEYS = JSON.stringify(keys); + process.env.ENCRYPTION_ACTIVE_KEY_ID = active; + _resetCryptoCacheForTests(); +} + +const HOST_A = { a: "11".repeat(32) }; +const HOST_B = { b: "22".repeat(32) }; +/** Same key id as host B, different key material. */ +const HOST_B_IMPOSTOR = { b: "33".repeat(32) }; + +const b64 = (bytes: Uint8Array) => Buffer.from(bytes).toString("base64"); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +afterEach(() => { + process.env.ENCRYPTION_KEY = saved.key ?? ""; + process.env.ENCRYPTION_KEYS = saved.keys ?? ""; + process.env.ENCRYPTION_ACTIVE_KEY_ID = saved.active ?? ""; + _resetCryptoCacheForTests(); +}); + +function makeRequest(id: string) { + return new Request(`http://localhost/api/admin/backups/${id}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }); +} + +async function seedOwner() { + const prisma = getPrismaClient(); + const owner = await prisma.user.create({ + data: { + username: "visits-cross-key-owner", + email: "visits-cross-key-owner@example.test", + role: "ADMIN", + }, + }); + const session = await prisma.session.create({ + data: { userId: owner.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return owner; +} + +/** Store `payload` as a backup on this host and run the real restore. */ +async function restore(ownerId: string, payload: unknown) { + const prisma = getPrismaClient(); + const backup = await prisma.dataBackup.create({ + data: { + userId: ownerId, + type: "VISITS_CROSS_KEY_RESTORE", + data: encrypt(JSON.stringify(payload)), + }, + }); + return POST(makeRequest(backup.id) as unknown as Parameters[0], { + params: Promise.resolve({ id: backup.id }), + }); +} + +describe("visit and vaccination free text restored under another key", () => { + it("comes back readable under the receiving host's key", async () => { + const prisma = getPrismaClient(); + const owner = await seedOwner(); + + useKeys(HOST_A, "a"); + const practitioner = await prisma.practitioner.create({ + data: { + userId: owner.id, + name: "Practice", + noteEncrypted: encryptToBytes("ring the upper bell"), + locationEncrypted: encryptToBytes("Main street 1"), + }, + }); + const encounter = await prisma.encounter.create({ + data: { + userId: owner.id, + occurredAt: new Date("2026-06-30T08:00:00.000Z"), + status: "DONE", + kind: "ROUTINE", + practitionerId: practitioner.id, + reasonEncrypted: encryptToBytes("annual check"), + outcomeEncrypted: encryptToBytes("all fine"), + bodySiteEncrypted: encryptToBytes("left knee"), + }, + }); + const vaccination = await prisma.vaccinationRecord.create({ + data: { + userId: owner.id, + occurredAt: new Date("2026-05-14T00:00:00.000Z"), + vaccineName: "Tetanus", + noteEncrypted: encryptToBytes("sore arm for a day"), + }, + }); + const { payload: built } = await buildFullBackupPayload(prisma, owner.id); + const file = JSON.parse(JSON.stringify(built)) as unknown; + + useKeys(HOST_B, "b"); + const response = await restore(owner.id, file); + expect(response.status).toBe(200); + + const p = await prisma.practitioner.findUniqueOrThrow({ + where: { id: practitioner.id }, + }); + expect(decryptFromBytes(p.noteEncrypted!)).toBe("ring the upper bell"); + expect(decryptFromBytes(p.locationEncrypted!)).toBe("Main street 1"); + const e = await prisma.encounter.findUniqueOrThrow({ + where: { id: encounter.id }, + }); + expect(decryptFromBytes(e.reasonEncrypted!)).toBe("annual check"); + expect(decryptFromBytes(e.outcomeEncrypted!)).toBe("all fine"); + expect(decryptFromBytes(e.bodySiteEncrypted!)).toBe("left knee"); + const v = await prisma.vaccinationRecord.findUniqueOrThrow({ + where: { id: vaccination.id }, + }); + expect(decryptFromBytes(v.noteEncrypted!)).toBe("sore arm for a day"); + + const body = (await response.json()) as { + data: { skipped: { links: number } }; + }; + expect(body.data.skipped.links).toBe(0); + }); + + it("restores a legacy file's ciphertext only where this host can open it", async () => { + const prisma = getPrismaClient(); + const owner = await seedOwner(); + + useKeys(HOST_B, "b"); + const ownReason = b64(encryptToBytes("annual check")); + useKeys(HOST_A, "a"); + const foreignNote = b64(encryptToBytes("written under key a")); + useKeys(HOST_B_IMPOSTOR, "b"); + const impostorOutcome = b64(encryptToBytes("same id, other key")); + const impostorVaccination = b64(encryptToBytes("same id, other key")); + + // The shape a portable file had before the fix: ciphertext, no text. + useKeys(HOST_B, "b"); + const { payload: built } = await buildFullBackupPayload(prisma, owner.id); + const file = JSON.parse(JSON.stringify(built)) as Record; + const at = "2026-06-30T08:00:00.000Z"; + file.practitioners = [ + { + id: "legacy-practitioner", + name: "Practice", + specialty: null, + practice: null, + location: null, + phone: null, + noteEncrypted: foreignNote, + createdAt: at, + updatedAt: at, + }, + ]; + file.encounters = [ + { + id: "legacy-encounter", + occurredAt: at, + status: "DONE", + kind: "ROUTINE", + practitionerId: "legacy-practitioner", + reasonEncrypted: ownReason, + outcomeEncrypted: impostorOutcome, + bodySiteEncrypted: null, + laterality: null, + reminderId: null, + createdAt: at, + updatedAt: at, + }, + ]; + file.vaccinations = [ + { + id: "legacy-vaccination", + occurredAt: at, + vaccineName: "Tetanus", + noteEncrypted: impostorVaccination, + createdAt: at, + updatedAt: at, + }, + ]; + parseBackupPayload(JSON.stringify(file)); + + const response = await restore(owner.id, file); + expect(response.status).toBe(200); + + const p = await prisma.practitioner.findUniqueOrThrow({ + where: { id: "legacy-practitioner" }, + }); + expect(p.noteEncrypted).toBeNull(); + const e = await prisma.encounter.findUniqueOrThrow({ + where: { id: "legacy-encounter" }, + }); + expect(decryptFromBytes(e.reasonEncrypted!)).toBe("annual check"); + expect(e.outcomeEncrypted).toBeNull(); + const v = await prisma.vaccinationRecord.findUniqueOrThrow({ + where: { id: "legacy-vaccination" }, + }); + expect(v.vaccineName).toBe("Tetanus"); + expect(v.noteEncrypted).toBeNull(); + + const body = (await response.json()) as { + data: { + skipped: { + links: number; + catalogueKeys: Array<{ + catalogue: string; + key: string; + links: number; + }>; + }; + }; + }; + expect(body.data.skipped.catalogueKeys).toEqual([ + { + catalogue: "vaccinationCiphertext", + key: "vaccinations.legacy-vaccination.noteEncrypted", + links: 1, + }, + { + catalogue: "visitCiphertext", + key: "encounters.legacy-encounter.outcomeEncrypted", + links: 1, + }, + { + catalogue: "visitCiphertext", + key: "practitioners.legacy-practitioner.noteEncrypted", + links: 1, + }, + ]); + }); +}); diff --git a/tests/integration/encounter-procedures.test.ts b/tests/integration/encounter-procedures.test.ts index 31198c652..036aa2416 100644 --- a/tests/integration/encounter-procedures.test.ts +++ b/tests/integration/encounter-procedures.test.ts @@ -343,7 +343,10 @@ describe("the backup", () => { const entry = section.encounters.find((row) => row.id === created.id); expect(entry?.kind).toBe("PROCEDURE"); expect(entry?.laterality).toBe("BOTH"); - expect(entry?.bodySiteEncrypted).toEqual(expect.any(String)); + // A portable section carries the site readable, never the ciphertext; the + // restore seals it under this host's key. + expect(entry?.bodySite).toBe("Hüfte"); + expect(entry).not.toHaveProperty("bodySiteEncrypted"); // Through the file schema, as a restore reads it: a field the schema does // not know would be dropped here and come back as null. From b36190b2dfd9c31bf8aadc8af430c84148071615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:19:32 +0200 Subject: [PATCH 11/21] Restore custom mood tags without mood entries The account's own mood categories, tags and hidden set were written only inside the restore's mood-entry branch, so a vocabulary set up before the first entry did not come back. They are written whether or not the file carries an entry. --- src/lib/export/restore-backup.ts | 244 +++++++++--------- ...ackup-mood-label-cross-key-restore.test.ts | 23 +- 2 files changed, 128 insertions(+), 139 deletions(-) diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index edf38d815..e0bd34eef 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -1515,6 +1515,130 @@ export async function restoreBackup( } reportSection("mood"); + // The account's own categories, tags and hidden set are written + // whether or not the file carries a mood entry: an account that set + // up its mood vocabulary before logging a single entry has them too, + // and they used to be written only inside the entries branch. + // The account's own CATEGORIES go in before its tags. A custom + // tag's `categoryId` is a real foreign key, so writing the tag + // first threw `mood_tags_category_id_fkey` and rolled the entire + // restore back — an account that had grouped its own mood factors + // could not be restored at all, and the failure named a constraint + // rather than the thing that was missing. + // + // Upserted by key for the same reason the tags below are: `key` is + // globally unique, so a seeded category that later claims the key + // must be left alone rather than overwritten by a restore. + // Labels this host's keys do not open, by file path. + const unopenedMoodLabels: string[] = []; + for (const category of payload.customMoodTagCategories) { + await tx.moodTagCategory.upsert({ + where: { key: category.key }, + create: { + id: category.id, + userId: ownerId, + key: category.key, + labelKey: category.labelKey, + icon: category.icon ?? null, + sortOrder: category.sortOrder ?? 0, + isActive: category.isActive ?? true, + labelEncrypted: sealForRestore( + category.labelEncrypted, + category.label, + `customMoodTagCategories.${category.key}.labelEncrypted`, + unopenedMoodLabels, + ), + }, + update: {}, + }); + } + + // Re-create the account's own tag definitions first. The lookup + // below used to ask only for the seeded catalogue (`userId: null`), + // so a single custom rated tag made the whole restore throw + // "Unknown mood factor keys" — the file was complete and unusable. + for (const tag of payload.customMoodTags) { + await tx.moodTag.upsert({ + where: { key: tag.key }, + create: { + ...(tag.id ? { id: tag.id } : {}), + userId: ownerId, + key: tag.key, + labelKey: tag.labelKey, + categoryId: tag.categoryId, + kind: tag.kind, + isActive: tag.isActive, + icon: tag.icon ?? null, + sortOrder: tag.sortOrder, + labelEncrypted: sealForRestore( + tag.labelEncrypted, + tag.label, + `customMoodTags.${tag.key}.labelEncrypted`, + unopenedMoodLabels, + ), + scaleMin: tag.scaleMin, + scaleMax: tag.scaleMax, + inverse: tag.inverse, + }, + // `key` is globally unique, so a seeded key would collide. + // Leave the catalogue row alone; the links resolve either way. + update: {}, + }); + } + recordUnknownKeys( + skips, + "moodLabelCiphertext", + unopenedMoodLabels, + unopenedMoodLabels, + ); + + // Which tags the account hid. Delete-then-recreate like every + // other section, and resolved by KEY: what people hide is almost + // always a seeded tag, whose id differs on every instance. + // + // A key this instance no longer knows costs one hidden row and is + // REPORTED. Dropping it silently would un-hide a factor the person + // had deliberately taken out of their mood editor, and they would + // have no way to know why it came back. + await tx.moodTagHidden.deleteMany({ where: { userId: ownerId } }); + if (payload.hiddenMoodTags.length > 0) { + const hiddenKeys = payload.hiddenMoodTags.map((h) => h.key); + const hiddenRows = await tx.moodTag.findMany({ + where: { + key: { in: [...new Set(hiddenKeys)] }, + OR: [{ userId: null }, { userId: ownerId }], + }, + select: { id: true, key: true }, + }); + const hiddenByKey = new Map( + hiddenRows.map((row) => [row.key, row.id]), + ); + const unresolvedHidden: string[] = []; + const writableHidden = payload.hiddenMoodTags.filter((entry) => { + if (hiddenByKey.has(entry.key)) return true; + unresolvedHidden.push(entry.key); + return false; + }); + if (writableHidden.length > 0) { + await tx.moodTagHidden.createMany({ + data: writableHidden.map((entry) => ({ + userId: ownerId, + moodTagId: hiddenByKey.get(entry.key)!, + ...(entry.createdAt + ? { createdAt: new Date(entry.createdAt) } + : {}), + })), + skipDuplicates: true, + }); + } + recordUnknownKeys( + skips, + "moodTag", + [...new Set(unresolvedHidden)], + unresolvedHidden, + ); + } + if (payload.moodEntries.length > 0) { const referencedFactorKeys = payload.moodEntries.flatMap((entry) => entry.factors.map((factor) => factor.key), @@ -1524,126 +1648,6 @@ export async function restoreBackup( (entry) => entry.structuredTags, ); const tagKeys = [...new Set(referencedTagKeys)]; - // The account's own CATEGORIES go in before its tags. A custom - // tag's `categoryId` is a real foreign key, so writing the tag - // first threw `mood_tags_category_id_fkey` and rolled the entire - // restore back — an account that had grouped its own mood factors - // could not be restored at all, and the failure named a constraint - // rather than the thing that was missing. - // - // Upserted by key for the same reason the tags below are: `key` is - // globally unique, so a seeded category that later claims the key - // must be left alone rather than overwritten by a restore. - // Labels this host's keys do not open, by file path. - const unopenedMoodLabels: string[] = []; - for (const category of payload.customMoodTagCategories) { - await tx.moodTagCategory.upsert({ - where: { key: category.key }, - create: { - id: category.id, - userId: ownerId, - key: category.key, - labelKey: category.labelKey, - icon: category.icon ?? null, - sortOrder: category.sortOrder ?? 0, - isActive: category.isActive ?? true, - labelEncrypted: sealForRestore( - category.labelEncrypted, - category.label, - `customMoodTagCategories.${category.key}.labelEncrypted`, - unopenedMoodLabels, - ), - }, - update: {}, - }); - } - - // Re-create the account's own tag definitions first. The lookup - // below used to ask only for the seeded catalogue (`userId: null`), - // so a single custom rated tag made the whole restore throw - // "Unknown mood factor keys" — the file was complete and unusable. - for (const tag of payload.customMoodTags) { - await tx.moodTag.upsert({ - where: { key: tag.key }, - create: { - ...(tag.id ? { id: tag.id } : {}), - userId: ownerId, - key: tag.key, - labelKey: tag.labelKey, - categoryId: tag.categoryId, - kind: tag.kind, - isActive: tag.isActive, - icon: tag.icon ?? null, - sortOrder: tag.sortOrder, - labelEncrypted: sealForRestore( - tag.labelEncrypted, - tag.label, - `customMoodTags.${tag.key}.labelEncrypted`, - unopenedMoodLabels, - ), - scaleMin: tag.scaleMin, - scaleMax: tag.scaleMax, - inverse: tag.inverse, - }, - // `key` is globally unique, so a seeded key would collide. - // Leave the catalogue row alone; the links resolve either way. - update: {}, - }); - } - recordUnknownKeys( - skips, - "moodLabelCiphertext", - unopenedMoodLabels, - unopenedMoodLabels, - ); - - // Which tags the account hid. Delete-then-recreate like every - // other section, and resolved by KEY: what people hide is almost - // always a seeded tag, whose id differs on every instance. - // - // A key this instance no longer knows costs one hidden row and is - // REPORTED. Dropping it silently would un-hide a factor the person - // had deliberately taken out of their mood editor, and they would - // have no way to know why it came back. - await tx.moodTagHidden.deleteMany({ where: { userId: ownerId } }); - if (payload.hiddenMoodTags.length > 0) { - const hiddenKeys = payload.hiddenMoodTags.map((h) => h.key); - const hiddenRows = await tx.moodTag.findMany({ - where: { - key: { in: [...new Set(hiddenKeys)] }, - OR: [{ userId: null }, { userId: ownerId }], - }, - select: { id: true, key: true }, - }); - const hiddenByKey = new Map( - hiddenRows.map((row) => [row.key, row.id]), - ); - const unresolvedHidden: string[] = []; - const writableHidden = payload.hiddenMoodTags.filter((entry) => { - if (hiddenByKey.has(entry.key)) return true; - unresolvedHidden.push(entry.key); - return false; - }); - if (writableHidden.length > 0) { - await tx.moodTagHidden.createMany({ - data: writableHidden.map((entry) => ({ - userId: ownerId, - moodTagId: hiddenByKey.get(entry.key)!, - ...(entry.createdAt - ? { createdAt: new Date(entry.createdAt) } - : {}), - })), - skipDuplicates: true, - }); - } - recordUnknownKeys( - skips, - "moodTag", - [...new Set(unresolvedHidden)], - unresolvedHidden, - ); - } - const factorRows = factorKeys.length === 0 ? [] diff --git a/tests/integration/backup-mood-label-cross-key-restore.test.ts b/tests/integration/backup-mood-label-cross-key-restore.test.ts index e2bb51883..f1a0b6719 100644 --- a/tests/integration/backup-mood-label-cross-key-restore.test.ts +++ b/tests/integration/backup-mood-label-cross-key-restore.test.ts @@ -122,30 +122,16 @@ async function restore(ownerId: string, payload: unknown) { }); } -/** - * One mood entry. The restore writes the account's own tags and categories - * inside its mood-entry branch, so a file without entries never reaches them. - */ -async function seedMoodEntry(userId: string) { - await getPrismaClient().moodEntry.create({ - data: { - userId, - date: "2026-07-01", - mood: "GUT", - score: 4, - source: "MOODLOG", - moodLoggedAt: new Date("2026-07-01T20:00:00.000Z"), - }, - }); -} - describe("custom mood labels in a portable file restored under another key", () => { + // Neither case logs a mood entry: an account's own tags and categories + // travel whether or not it has entries (they used to be written only inside + // the restore's entries branch, and a vocabulary set up before the first + // entry did not come back). it("come back readable under the receiving host's key", async () => { const prisma = getPrismaClient(); const owner = await seedOwner(); useKeys(HOST_A, "a"); - await seedMoodEntry(owner.id); const category = await prisma.moodTagCategory.create({ data: { userId: owner.id, @@ -200,7 +186,6 @@ describe("custom mood labels in a portable file restored under another key", () // The shape a portable file had before the fix: labels as ciphertext. useKeys(HOST_B, "b"); - await seedMoodEntry(owner.id); const { payload: built } = await buildFullBackupPayload(prisma, owner.id); const file = JSON.parse(JSON.stringify(built)) as Record; file.customMoodTagCategories = [ From bf35779db77ae269bfd2eef6f548a06f85e65ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:19:32 +0200 Subject: [PATCH 12/21] Fail the round trip when a portable file carries ciphertext With every sealed column filled, the column round trip now also builds the portable file of the same account and fails on any section that carries a sealed column as stored ciphertext, unless the section is named with a reason. The disaster-recovery file of the same account is the floor, so a matcher that finds nothing cannot pass the check. --- tests/integration/backup-round-trip.test.ts | 59 +++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/integration/backup-round-trip.test.ts b/tests/integration/backup-round-trip.test.ts index 2911cadfc..cd3018e99 100644 --- a/tests/integration/backup-round-trip.test.ts +++ b/tests/integration/backup-round-trip.test.ts @@ -77,6 +77,7 @@ import { } from "@/lib/labs/biomarker-store"; import { legacyStreamedBlobFrom } from "@/__tests__/helpers/legacy-backup-blob"; import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; +import { BackupKeyIdCollector } from "@/lib/export/backup-key-ids"; import { TWO_ENDED_MODELS, USER_COLUMN_BACKUP_CLASS, @@ -3694,6 +3695,12 @@ function lostColumns( return { lost: lost.sort(), compared }; } +/** + * Sections a portable file may carry stored ciphertext in, each with the + * reason. Empty: every sealed column travels readable in a portable file. + */ +const PORTABLE_CIPHERTEXT_ALLOWED: ReadonlyMap = new Map(); + describe("every column of every two-ended model survives a real restore", () => { it("brings each column back with the value it left with", async () => { const prisma = getPrismaClient(); @@ -3758,6 +3765,58 @@ describe("every column of every two-ended model survives a real restore", () => }, { purpose: "disaster-recovery", exportedAt }, ); + + // ── No ciphertext in a portable file ────────────────────────────────── + // + // Every sealed column is filled at this point, so the portable file of + // this account is the one place a column still exported as stored + // ciphertext shows up. A portable file is read and restored on hosts + // with other keys; ciphertext in it is noise there, and a restore either + // refuses the file or writes values nobody can open. That class was + // fixed one section at a time (cycle, mood labels, visits, vaccinations) + // and kept coming back, because each builder decides its own encoding. + // This reads what the builders actually wrote, rather than how their + // source is spelt. Limit: it sees members named `…Encrypted` only, the + // naming the key preflight relies on too. + const sealedIn = (text: string) => { + const collector = new BackupKeyIdCollector(); + collector.visit(JSON.parse(text)); + const sections = new Map(); + for (const [, use] of collector.entries()) { + for (const section of use.sections) { + sections.set(section, (sections.get(section) ?? 0) + use.count); + } + } + return sections; + }; + // The floor: the same matcher over the disaster-recovery file of the same + // account, which carries every sealed column as ciphertext. A matcher + // that finds nothing there would make the portable check pass vacuously. + const drSealed = sealedIn(json); + expect( + [...drSealed.values()].reduce((a, b) => a + b, 0), + ).toBeGreaterThanOrEqual(40); + expect(drSealed.size).toBeGreaterThanOrEqual(15); + let portableJson = ""; + await streamFullBackupJson( + prisma, + OWNER_ID, + (chunk) => { + portableJson += chunk; + }, + { purpose: "portable-export", exportedAt }, + ); + const portableSealed = [...sealedIn(portableJson).keys()].filter( + (section) => !PORTABLE_CIPHERTEXT_ALLOWED.has(section), + ); + expect( + portableSealed, + "a portable file carries these sections' sealed columns as stored " + + "ciphertext. Carry them readable (`src/lib/export/sealed-text.ts`) " + + "and seal them again on restore, or name the section in " + + "PORTABLE_CIPHERTEXT_ALLOWED with the reason", + ).toEqual([]); + await prisma.user.delete({ where: { id: OWNER_ID } }); await createOwner(prisma); const backup = await prisma.dataBackup.create({ From b63fa317f388038566b246f2a27be2d4496f2364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:37:23 +0200 Subject: [PATCH 13/21] fix(docker): hoist the canvas binary of the pinned version Refs #1124. The store holds two @napi-rs/canvas versions, the pinned one and an older one that pdf-parse's own pdfjs-dist pulls in. The runner stage took the musl binary with `find | head -1`, got the old one, and copied it into the new loader's directory, where the loader looks first. Rendering still worked, but `loadImage` calls `Image.decode`, which the old binary lacks, so every preview thumbnail failed with "image.decode is not a function" on both architectures. The builder now records the resolved canvas version and the runner hoists that version by name, as it already does for pdfjs-dist. The build stops when the binary it copies belongs to another version. --- Dockerfile | 51 +++++++++++---- .../canvas-binary-hoist-guard.test.ts | 65 +++++++++++++++++++ 2 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 src/__tests__/canvas-binary-hoist-guard.test.ts diff --git a/Dockerfile b/Dockerfile index 6b871db84..20fe839da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,6 +54,9 @@ RUN NODE_OPTIONS=--max-old-space-size=8192 pnpm build # name; see the note there for why guessing was wrong. RUN node -p "require('/app/node_modules/pdfjs-dist/package.json').version" \ > /app/.pdfjs-version +# The same for @napi-rs/canvas, whose native binary must match its JS loader. +RUN node -p "require('/app/node_modules/@napi-rs/canvas/package.json').version" \ + > /app/.canvas-version # ── Stage 3: Production runner ───────────────────────────── FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2 AS runner @@ -120,21 +123,41 @@ RUN node -e "require.resolve('pg-boss'); require.resolve('pg')" # @napi-rs/canvas" → "Cannot polyfill DOMMatrix" → rasterize ReferenceError); # (2) @napi-rs/canvas's own loader then resolves its platform binary. Hoist BOTH # the canvas package and the musl binary package to /app/node_modules/@napi-rs -# via symlink into the .pnpm store, and also drop the prebuilt .node into the -# canvas dir so the loader's local-file fallback (`require('./skia..node')`) -# is belt-and-suspenders. Per-arch buildx installs only the matching musl triple. +# via symlink into the .pnpm store, and drop the prebuilt .node into the canvas +# dir: the loader tries that local file (`require('./skia..node')`) +# before the platform package, so it decides which binary runs. +# +# Hoist the version the app pins, by name, exactly like pdfjs-dist below. The +# store holds a second, older canvas that pdf-parse's own pdfjs-dist pulls in, +# and this step used to take whichever `find | head -1` returned first. It +# returned the old binary, which then sat next to the new loader: rendering +# still worked, but `loadImage` called an `Image.decode` the old binary does +# not have, and every preview thumbnail failed with "image.decode is not a +# function". Name the version, and fail the build when the binary that lands +# in the canvas dir is not that version's. +COPY --from=builder /app/.canvas-version /tmp/.canvas-version RUN set -e; \ - CANVAS_DIR="$(find /app/node_modules/.pnpm -maxdepth 4 -type d -path '*@napi-rs+canvas@*/node_modules/@napi-rs/canvas' 2>/dev/null | head -1)"; \ - MUSL_DIR="$(find /app/node_modules/.pnpm -maxdepth 4 -type d -path '*@napi-rs+canvas-linux-*-musl@*/node_modules/@napi-rs/canvas-linux-*-musl' 2>/dev/null | head -1)"; \ - NODE_BIN="$(find /app/node_modules/.pnpm -maxdepth 5 -name 'skia.linux-*-musl.node' 2>/dev/null | head -1)"; \ - if [ -n "$CANVAS_DIR" ]; then \ - mkdir -p /app/node_modules/@napi-rs; \ - ln -sfn "$CANVAS_DIR" /app/node_modules/@napi-rs/canvas; \ - [ -n "$MUSL_DIR" ] && ln -sfn "$MUSL_DIR" "/app/node_modules/@napi-rs/$(basename "$MUSL_DIR")"; \ - [ -n "$NODE_BIN" ] && cp "$NODE_BIN" "$CANVAS_DIR/"; \ - chown -R nextjs:nodejs /app/node_modules/@napi-rs; \ - echo "canvas hoisted for pdfjs: $CANVAS_DIR (musl=$MUSL_DIR)"; \ - else echo "WARN: @napi-rs/canvas not found; PDF rasterization will degrade to local text"; fi + CANVAS_VERSION="$(cat /tmp/.canvas-version)"; \ + rm -f /tmp/.canvas-version; \ + CANVAS_DIR="/app/node_modules/.pnpm/@napi-rs+canvas@${CANVAS_VERSION}/node_modules/@napi-rs/canvas"; \ + MUSL_DIR="$(find /app/node_modules/.pnpm -maxdepth 4 -type d -path "*@napi-rs+canvas-linux-*-musl@${CANVAS_VERSION}/node_modules/@napi-rs/canvas-linux-*-musl" 2>/dev/null | head -1)"; \ + if [ ! -d "$CANVAS_DIR" ] || [ -z "$MUSL_DIR" ]; then \ + echo "ERROR: pinned @napi-rs/canvas@${CANVAS_VERSION} or its musl binary did not reach the traced image."; \ + echo "Present instead:"; \ + find /app/node_modules/.pnpm -maxdepth 1 -name '@napi-rs+canvas*' 2>/dev/null; \ + exit 1; \ + fi; \ + NODE_BIN="$(find "$MUSL_DIR" -maxdepth 1 -name 'skia.linux-*-musl.node' | head -1)"; \ + [ -n "$NODE_BIN" ] || { echo "ERROR: no skia binary in $MUSL_DIR"; exit 1; }; \ + mkdir -p /app/node_modules/@napi-rs; \ + ln -sfn "$CANVAS_DIR" /app/node_modules/@napi-rs/canvas; \ + ln -sfn "$MUSL_DIR" "/app/node_modules/@napi-rs/$(basename "$MUSL_DIR")"; \ + cp "$NODE_BIN" "$CANVAS_DIR/"; \ + chown -R nextjs:nodejs /app/node_modules/@napi-rs; \ + BIN_VERSION="$(node -p "require('$MUSL_DIR/package.json').version")"; \ + [ "$BIN_VERSION" = "$CANVAS_VERSION" ] || { echo "ERROR: canvas binary is $BIN_VERSION, loader is $CANVAS_VERSION"; exit 1; }; \ + cmp -s "$NODE_BIN" "$CANVAS_DIR/$(basename "$NODE_BIN")" || { echo "ERROR: the canvas dir holds a different binary"; exit 1; }; \ + echo "canvas hoisted for pdfjs: $CANVAS_DIR (binary $BIN_VERSION from $MUSL_DIR)" # pdfjs-dist is `serverExternalPackages` (NOT bundled — the bundled copy's render # path breaks in the standalone image), so the server does a runtime bare diff --git a/src/__tests__/canvas-binary-hoist-guard.test.ts b/src/__tests__/canvas-binary-hoist-guard.test.ts new file mode 100644 index 000000000..d230fab26 --- /dev/null +++ b/src/__tests__/canvas-binary-hoist-guard.test.ts @@ -0,0 +1,65 @@ +/** + * Structural guard: the runtime image puts the @napi-rs/canvas native binary + * of the version the app pins next to that version's loader. + * + * ## The failure this freezes (GitHub #1124) + * + * The pnpm store holds two canvas versions: the one in package.json and an + * older one that pdf-parse's own pdfjs-dist pulls in. The runner stage picked + * the musl binary with `find | head -1`, got the old one, and copied it into + * the new loader's directory, where the loader looks first. PDF rendering + * still worked, so nothing looked wrong; `loadImage` called an + * `Image.decode` the old binary lacks, and every preview thumbnail failed + * with "image.decode is not a function" on both architectures. + * + * ## What it proves and what it does not + * + * The builder records the resolved canvas version, the runner hoists by that + * name, and the build stops when the copied binary's package is a different + * version. Whether the binary loads is proved when the image runs; this file + * reads the Dockerfile only. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +const DOCKERFILE = readFileSync(join(process.cwd(), "Dockerfile"), "utf8"); + +function stage(name: string): string { + const start = DOCKERFILE.search(new RegExp(`^FROM .* AS ${name}$`, "m")); + expect(start, `the Dockerfile has a ${name} stage`).toBeGreaterThan(0); + const next = DOCKERFILE.slice(start + 1).search(/^FROM /m); + return next < 0 + ? DOCKERFILE.slice(start) + : DOCKERFILE.slice(start, start + 1 + next); +} + +describe("canvas binary hoist", () => { + it("records the resolved canvas version in the builder", () => { + expect(stage("builder")).toMatch( + /require\('\/app\/node_modules\/@napi-rs\/canvas\/package\.json'\)\.version"[\s\\]*>\s*\/app\/\.canvas-version/, + ); + }); + + it("hoists the loader and the musl binary of that version only", () => { + const runner = stage("runner"); + expect(runner).toContain( + "COPY --from=builder /app/.canvas-version /tmp/.canvas-version", + ); + expect(runner).toContain( + 'CANVAS_DIR="/app/node_modules/.pnpm/@napi-rs+canvas@${CANVAS_VERSION}/', + ); + expect(runner).toContain("canvas-linux-*-musl@${CANVAS_VERSION}/"); + // A version-blind pick is the defect: any `canvas…@*` glob decides by + // readdir order which of the store's copies wins. + expect(runner).not.toMatch(/@napi-rs\+canvas[^"'\s]*@\*/); + }); + + it("fails the build when the copied binary is another version", () => { + const runner = stage("runner"); + expect(runner).toMatch( + /\[ "\$BIN_VERSION" = "\$CANVAS_VERSION" \] \|\| \{[^}]*exit 1; \}/, + ); + }); +}); From 4b73099e75bdc87a4e36cb331a748861bf50aae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:37:24 +0200 Subject: [PATCH 14/21] fix(documents): render PDFs with pdfjs font data and refuse blank pages Refs #1124. The rasterizer opened pdfjs without its standard font, CMap and wasm directories and told it to use system fonts. The runtime image has none, so any text pdfjs could not draw from an embedded program (a non-embedded standard font, or an embedded one it cannot parse) came out as nothing, and an image-only provider was sent the page background. A workstation hides this because its own fonts stand in. pdfjs now gets the directories from its own package, resolved at runtime from the working directory, and Liberation Sans from that package is registered as the canvas' sans-serif, serif and monospace faces for the text pdfjs draws with a substitute. A document whose every rendered page is blank now reports render-failed, so the read records raster-failed instead of indexing a transcript of an empty page as a success. --- .../prepare-vision-raster-failure.test.ts | 15 ++ .../__tests__/rasterize-pdf-cleanup.test.ts | 9 +- .../__tests__/rasterize-pdf-font-data.test.ts | 155 ++++++++++++++ src/lib/documents/rasterize-pdf.ts | 189 +++++++++++++++++- tests/fixtures/pdf/fallback-fonts.pdf | Bin 0 -> 1801 bytes tests/fixtures/pdf/invisible-text-only.pdf | Bin 0 -> 747 bytes .../pdf/ocrmypdf11-two-images-type1c.pdf | Bin 0 -> 43948 bytes 7 files changed, 358 insertions(+), 10 deletions(-) create mode 100644 src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts create mode 100644 tests/fixtures/pdf/fallback-fonts.pdf create mode 100644 tests/fixtures/pdf/invisible-text-only.pdf create mode 100644 tests/fixtures/pdf/ocrmypdf11-two-images-type1c.pdf diff --git a/src/lib/documents/__tests__/prepare-vision-raster-failure.test.ts b/src/lib/documents/__tests__/prepare-vision-raster-failure.test.ts index 05af83fdc..3e01f2d75 100644 --- a/src/lib/documents/__tests__/prepare-vision-raster-failure.test.ts +++ b/src/lib/documents/__tests__/prepare-vision-raster-failure.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { Buffer } from "node:buffer"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; /** * Refs #776 — the raster-failed chain, end to end through the REAL @@ -68,4 +70,17 @@ describe("raster-failed end to end (#776)", () => { const vision = await prepareVisionInput(DOC as never, true); expect(vision).toMatchObject({ ok: true }); }); + + it("a PDF whose pages all render blank is rasterFailed, not an image of nothing (#1124)", async () => { + // Text in render mode 3 only: pdfjs opens it and draws an empty page. + // Before the blank-page guard this went to the provider as one white + // JPEG and came back as a near-empty transcript indexed as a success. + vi.mocked(decryptDocumentContent).mockReturnValue( + readFileSync( + join(process.cwd(), "tests/fixtures/pdf/invisible-text-only.pdf"), + ) as never, + ); + const vision = await prepareVisionInput(DOC as never, false); + expect(vision).toEqual({ ok: false, reason: "rasterFailed" }); + }); }); diff --git a/src/lib/documents/__tests__/rasterize-pdf-cleanup.test.ts b/src/lib/documents/__tests__/rasterize-pdf-cleanup.test.ts index 4e0d2cdba..bdafb0bfa 100644 --- a/src/lib/documents/__tests__/rasterize-pdf-cleanup.test.ts +++ b/src/lib/documents/__tests__/rasterize-pdf-cleanup.test.ts @@ -18,9 +18,16 @@ vi.mock("pdfjs-dist/legacy/build/pdf.mjs", () => ({ canvasFactory: { create: () => ({ canvas: { + width: 1, + height: 1, toBuffer: () => Buffer.from([0xff, 0xd8]), }, - context: {}, + // One black pixel: an inked page, so the blank-page guard passes. + context: { + getImageData: () => ({ + data: new Uint8ClampedArray([0, 0, 0, 255]), + }), + }, }), }, getPage: async () => ({ diff --git a/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts b/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts new file mode 100644 index 000000000..86fd7c5fe --- /dev/null +++ b/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts @@ -0,0 +1,155 @@ +import { Buffer } from "node:buffer"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { createCanvas, loadImage } from "@napi-rs/canvas"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * GitHub #1124 — PDFs from older OCRmyPDF archives got no thumbnail and an + * image-only vision read of them came back nearly empty. + * + * The runtime image has no system fonts. pdfjs was opened without its own + * font data and told to lean on system fonts, so any text it could not draw + * from an embedded program (a non-embedded standard font, or an embedded + * program it cannot parse) came out as nothing, and the page that reached + * the provider was the background alone. A workstation hides this because + * its own fonts stand in, so the rendering assertions below are regression + * guards here and the container is where they were proven red; the option + * and registration assertions are red on any host. + * + * Fixtures (`tests/fixtures/pdf/`, synthetic text only): + * - `ocrmypdf11-two-images-type1c.pdf` — the reporter's shape: PDF 1.7, + * pikepdf producer, one 630.5 x 861.6 pt page with a 150 ppi RGB JPEG and + * a 75 ppi RGB Flate image, Type 1C Helvetica and Palatino-Roman, and an + * invisible OCR text layer. + * - `fallback-fonts.pdf` — a letter in non-embedded Helvetica and in a + * Type 1C Palatino-Roman whose program pdfjs cannot parse. + * - `invisible-text-only.pdf` — text in render mode 3 and nothing else, so + * every page renders blank. + */ + +const { annotate, registerFromPath } = vi.hoisted(() => ({ + annotate: vi.fn(), + registerFromPath: vi.fn(), +})); +vi.mock("@/lib/logging/context", () => ({ annotate })); + +const getDocumentOptions: Record[] = []; +vi.mock("pdfjs-dist/legacy/build/pdf.mjs", async (importActual) => { + const actual = + await importActual(); + return { + ...actual, + getDocument: (opts: Record) => { + getDocumentOptions.push(opts); + return actual.getDocument(opts as never); + }, + }; +}); +vi.mock("@napi-rs/canvas", async (importActual) => { + const actual = await importActual(); + return { + ...actual, + GlobalFonts: { + ...actual.GlobalFonts, + registerFromPath: (path: string, alias?: string) => { + registerFromPath(path, alias); + return actual.GlobalFonts.registerFromPath(path, alias); + }, + }, + }; +}); + +import * as raster from "../rasterize-pdf"; +import { generateThumbnail } from "../thumbnail"; + +const FIXTURES = join(process.cwd(), "tests", "fixtures", "pdf"); +const fixture = (name: string) => readFileSync(join(FIXTURES, name)); + +/** Share of dark pixels in a JPEG, measured the way the rasterizer does. */ +async function inkOf(base64: string): Promise { + const image = await loadImage(Buffer.from(base64, "base64")); + const canvas = createCanvas(image.width, image.height); + const ctx = canvas.getContext("2d"); + ctx.drawImage(image, 0, 0); + const { data } = ctx.getImageData(0, 0, image.width, image.height); + let dark = 0; + for (let i = 0; i < data.length; i += 4) { + if (data[i]! + data[i + 1]! + data[i + 2]! < 384) dark++; + } + return dark / (data.length / 4); +} + +beforeEach(() => { + annotate.mockClear(); + registerFromPath.mockClear(); + getDocumentOptions.length = 0; + raster.__resetPdfjsAssetsForTests(); +}); + +describe("rasterizePdf font and decoder data (#1124)", () => { + it("gives pdfjs its font, CMap and wasm directories and no system fonts", async () => { + const result = await raster.rasterizePdf(fixture("fallback-fonts.pdf"), 1); + expect(result.ok).toBe(true); + const opts = getDocumentOptions.at(-1)!; + expect(opts.useSystemFonts).toBe(false); + for (const [key, probe] of [ + ["standardFontDataUrl", "FoxitSerif.pfb"], + ["standardFontDataUrl", "LiberationSans-Regular.ttf"], + ["cMapUrl", "UniJIS-UTF16-H.bcmap"], + ["wasmUrl", "openjpeg.wasm"], + ] as const) { + expect(typeof opts[key], key).toBe("string"); + expect(existsSync(`${opts[key] as string}${probe}`), probe).toBe(true); + } + expect(opts.cMapPacked).toBe(true); + }); + + it("registers a fallback face for every generic family pdfjs falls back to", async () => { + await raster.rasterizePdf(fixture("fallback-fonts.pdf"), 1); + const families = new Set(registerFromPath.mock.calls.map((c) => c[1])); + expect(families).toEqual(new Set(["sans-serif", "serif", "monospace"])); + for (const [path] of registerFromPath.mock.calls) { + expect(existsSync(path as string)).toBe(true); + } + }); + + it("renders the OCRmyPDF page with its scan and its text", async () => { + const result = await raster.rasterizePdf( + fixture("ocrmypdf11-two-images-type1c.pdf"), + ); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.images).toHaveLength(1); + expect(await inkOf(result.images[0]!.dataBase64)).toBeGreaterThan(0.005); + }); + + it("renders a letter whose fonts pdfjs has to substitute", async () => { + const result = await raster.rasterizePdf(fixture("fallback-fonts.pdf")); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(await inkOf(result.images[0]!.dataBase64)).toBeGreaterThan(0.001); + }); + + it("reports a document whose every page renders blank as render-failed", async () => { + const result = await raster.rasterizePdf( + fixture("invisible-text-only.pdf"), + ); + expect(result).toEqual({ ok: false, reason: "render-failed" }); + expect(annotate).toHaveBeenCalledWith({ + action: { name: "documents.rasterize.failed" }, + meta: { reason: "blank_pages", pages: 1 }, + }); + }); +}); + +describe("generateThumbnail on the OCRmyPDF shape (#1124)", () => { + it("produces a preview of the first page", async () => { + const result = await generateThumbnail( + fixture("ocrmypdf11-two-images-type1c.pdf"), + "application/pdf", + ); + expect(result.ok).toBe(true); + }); +}); diff --git a/src/lib/documents/rasterize-pdf.ts b/src/lib/documents/rasterize-pdf.ts index f61e74fc3..452ddfae5 100644 --- a/src/lib/documents/rasterize-pdf.ts +++ b/src/lib/documents/rasterize-pdf.ts @@ -31,6 +31,9 @@ * preserves the "a bad document never aborts an upload/batch" contract. */ import { Buffer } from "node:buffer"; +import { existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; import { annotate } from "@/lib/logging/context"; @@ -60,6 +63,14 @@ export const RASTER_TARGET_LONG_EDGE = 1600; /** JPEG quality (0-100). ~80 keeps text legible at a fraction of PNG's bytes. */ export const RASTER_JPEG_QUALITY = 80; +/** + * A rendered page counts as blank when fewer than this share of its pixels + * are dark. One short line of 16 pt text on an A4 page is ~0.1 %; a page + * number alone sits near this floor. A page set that is blank throughout is + * not a reading of the document, whatever the provider would make of it. + */ +export const RASTER_MIN_INK_RATIO = 0.0001; + /** One rasterized page, shaped as a vision `input_image` part. */ export interface RasterImage { mediaType: "image/jpeg"; @@ -98,9 +109,17 @@ interface PdfCanvas { height: number; toBuffer(mime: "image/jpeg", quality?: number): Buffer; } +interface PdfContext2D { + getImageData( + x: number, + y: number, + width: number, + height: number, + ): { data: Uint8ClampedArray }; +} interface PdfCanvasAndContext { canvas: PdfCanvas; - context: unknown; + context: PdfContext2D; } interface PdfCanvasFactory { create(width: number, height: number): PdfCanvasAndContext; @@ -125,12 +144,127 @@ interface PdfLoadingTask { destroy(): Promise; } interface PdfjsModule { - getDocument(opts: { - data: Uint8Array; - verbosity?: number; - isEvalSupported?: boolean; - useSystemFonts?: boolean; - }): PdfLoadingTask; + getDocument(opts: PdfDocumentOptions): PdfLoadingTask; +} +interface PdfDocumentOptions { + data: Uint8Array; + verbosity?: number; + isEvalSupported?: boolean; + useSystemFonts?: boolean; + standardFontDataUrl?: string; + cMapUrl?: string; + cMapPacked?: boolean; + wasmUrl?: string; +} +interface GlobalFontsLike { + registerFromPath(path: string, nameAlias?: string): unknown; +} + +/** + * Where pdfjs finds the data it does not carry in its JS: the standard 14 + * fonts (a PDF that names Helvetica or Times without embedding them), the + * CMaps (CID fonts with a predefined encoding), and the wasm image decoders + * (JPEG 2000, JBIG2). In a browser pdfjs fetches these from a URL; under Node + * they are read from disk, and without a directory pdfjs simply draws the text + * without glyphs. + * + * The runtime image carries no system fonts at all, so nothing else catches + * that: a letter set in a non-embedded font rendered as an empty page in the + * container while it rendered correctly on a workstation, where the operating + * system's fonts stood in. The image-only provider then read the empty page + * and returned a line of nothing. + * + * Resolved from the working directory with Node's own resolution, which is + * what finds the hoisted `node_modules/pdfjs-dist` in the standalone image + * and the pnpm link in a checkout. The specifier is assembled at runtime so + * the bundler leaves the lookup alone. The ICC profile option is not set: its + * loader needs a synchronous XMLHttpRequest that Node lacks, and pdfjs falls + * back to each colour space's alternate. + */ +export interface PdfjsAssets { + standardFontDataUrl: string; + cMapUrl: string; + wasmUrl: string; + /** The Liberation Sans faces registered as the generic fallback families. */ + fallbackFonts: string[]; +} + +let cachedAssets: PdfjsAssets | null | undefined; + +export function resolvePdfjsAssets(): PdfjsAssets | null { + if (cachedAssets !== undefined) return cachedAssets; + cachedAssets = null; + try { + const anchor = createRequire(join(process.cwd(), "package.json")); + const root = dirname( + anchor.resolve(["pdfjs-dist", "package.json"].join("/")), + ); + const fonts = join(root, "standard_fonts"); + const assets: PdfjsAssets = { + standardFontDataUrl: `${fonts}/`, + cMapUrl: `${join(root, "cmaps")}/`, + wasmUrl: `${join(root, "wasm")}/`, + fallbackFonts: [ + "LiberationSans-Regular.ttf", + "LiberationSans-Bold.ttf", + "LiberationSans-Italic.ttf", + "LiberationSans-BoldItalic.ttf", + ].map((file) => join(fonts, file)), + }; + if ( + existsSync(join(fonts, "FoxitSerif.pfb")) && + existsSync(join(root, "cmaps", "UniJIS-UTF16-H.bcmap")) && + existsSync(join(root, "wasm", "openjpeg.wasm")) && + assets.fallbackFonts.every((file) => existsSync(file)) + ) { + cachedAssets = assets; + } + } catch { + // Unresolvable: render without the data, as before, and say so below. + } + return cachedAssets; +} + +let fallbackFamiliesRegistered = false; + +/** + * Give the canvas a face for the generic families pdfjs falls back to. + * + * When pdfjs cannot use a font program (a damaged or unsupported embedded + * font, or a non-embedded face outside the standard 14) it draws the text + * with the canvas' own `fillText` under the PDF's font name and a generic + * family. With no system fonts in the image that text vanished. Registering + * Liberation Sans, which pdfjs ships, under `sans-serif`, `serif` and + * `monospace` makes it legible instead. Once per process. + */ +async function registerFallbackFamilies(assets: PdfjsAssets): Promise { + if (fallbackFamiliesRegistered) return; + fallbackFamiliesRegistered = true; + const { GlobalFonts } = (await import("@napi-rs/canvas")) as unknown as { + GlobalFonts: GlobalFontsLike; + }; + for (const family of ["sans-serif", "serif", "monospace"]) { + for (const file of assets.fallbackFonts) { + GlobalFonts.registerFromPath(file, family); + } + } +} + +/** Share of dark pixels on a rendered page (luminance below half). */ +function inkRatio(context: PdfContext2D, width: number, height: number) { + const { data } = context.getImageData(0, 0, width, height); + let dark = 0; + for (let i = 0; i < data.length; i += 4) { + if (data[i]! + data[i + 1]! + data[i + 2]! < 384) dark++; + } + const pixels = data.length / 4; + return pixels > 0 ? dark / pixels : 0; +} + +/** Test hook: the asset lookup and the font registration are per process. */ +export function __resetPdfjsAssetsForTests(): void { + cachedAssets = undefined; + fallbackFamiliesRegistered = false; } /** @@ -163,11 +297,31 @@ export async function rasterizePdf( const pdfjs = (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule; + const assets = resolvePdfjsAssets(); + if (assets) { + await registerFallbackFamilies(assets); + } else { + annotate({ + action: { name: "documents.rasterize.assetsMissing" }, + meta: { reason: "pdfjs_assets_unresolved" }, + }); + } + task = pdfjs.getDocument({ data: new Uint8Array(buffer), verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. isEvalSupported: false, // never eval font programs (defence in depth). - useSystemFonts: true, + // There are no system fonts to use: substitutes come from pdfjs' own + // standard font data and the fallback families registered above. + useSystemFonts: false, + ...(assets + ? { + standardFontDataUrl: assets.standardFontDataUrl, + cMapUrl: assets.cMapUrl, + cMapPacked: true, + wasmUrl: assets.wasmUrl, + } + : {}), }); doc = await task.promise; @@ -175,6 +329,7 @@ export async function rasterizePdf( const cap = Math.min(Math.max(1, maxPages), RASTER_MAX_PAGES); const pageCount = Math.min(doc.numPages, cap); const images: RasterImage[] = []; + let blankPages = 0; for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) { const page = await doc.getPage(pageNumber); @@ -196,6 +351,12 @@ export async function rasterizePdf( viewport, canvas: cc.canvas, }).promise; + if ( + inkRatio(cc.context, cc.canvas.width, cc.canvas.height) < + RASTER_MIN_INK_RATIO + ) { + blankPages++; + } const jpeg = cc.canvas.toBuffer("image/jpeg", RASTER_JPEG_QUALITY); images.push({ mediaType: "image/jpeg", @@ -217,9 +378,19 @@ export async function rasterizePdf( }); return { ok: false, reason: "render-failed" }; } + if (blankPages === images.length) { + // Every rendered page came out empty. Sending those to a vision model + // buys a transcript of nothing that would be indexed as a success; say + // the render failed instead, so the read reports `raster-failed`. + annotate({ + action: { name: "documents.rasterize.failed" }, + meta: { reason: "blank_pages", pages: images.length }, + }); + return { ok: false, reason: "render-failed" }; + } annotate({ action: { name: "documents.rasterize.ok" }, - meta: { pages: images.length, cappedFrom: doc.numPages }, + meta: { pages: images.length, cappedFrom: doc.numPages, blankPages }, }); return { ok: true, images, pageCount: doc.numPages }; } catch (err) { diff --git a/tests/fixtures/pdf/fallback-fonts.pdf b/tests/fixtures/pdf/fallback-fonts.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ee01b4fa016de054a70d677131593bbaf01ce44e GIT binary patch literal 1801 zcmeHIU1$?o6kdxU%r0tW#iAfbq*B?9Gj}F4$wWp!bpEc@iH>tCTLsL)qYS^Qc5ia#j&q$r9%_)tL~MDgB9V`3D2^u-BbhI7w3 z_niBE=ev#RL^6ODgoMTuFZaC>5J=#xSt~>$Af|Dicpx)N7Q~JXy#&NKrnunbAsQ8k zZE{DxTn>mkeKrLI;!?}>pbHc~w7VQ!%2x^H%au@@5}SGuEN%BQF)iHU46rN``>DDL z;h+pGR~oy_v0PS`cut>dusV6gNr$cYA1P2viIQp1{I6(~Rb5?5RTF&EawToV&eI-1 zVVUK#3Y56)UBHwih~b!oM|spGxF`(l-hAoO-Q*^Z;v!QPaYYLq^BDPgL94_8r!&c#oT?P z_a5P0htthxM#hf4`lIE_zs{_Mo%Q;}*>$s%+mG}=80z}AaQ*witJiOz79Z}Yzxw*c zq<_L=E1NlR;|+{M$WN-=eGB%8e~My?6#5CZ3=HBDn{{s4c@R4-J7#-U#n9fjhWgCl z{9wqJXyRw60L=mTaDy~vr)t?wAnO#d{mEUDmg~`Y4|YM}s4$ch#4}v0mpU$le0aq+ zJnn*I4a85>#hNf_59g;UF=l=Je~~ljt{Oj%V3{8=&*0-7;vmLxZ;Dv?9txVyaOarw zwSXLEOk*>d_dp4j_|Q_)iU86yzODe0WCXe#;*!A$!$z4X5H4|r8Pl&6~5Rn)6coF^`~@ILqAAdX8)P+_HfMo|oNHO`*9AZAh>smr5* zRNI0~>cZ#`nWm=m%f+8A{<8YOvB)=Pe{EXy;TR9NBqvCq3(~s>G*wkp$iX`sV#CfK zuq!r=$zP2P$xNJUY_i5KhZ>vuk$f6rcf$LA2&qg1YHW&Hm5;iZ0p$APopl_3i}=@0 zyS0&k4q|Hpx+12mg!B+HOjR?nDycaGVO=L`SdwKuhjP)+d-$~TD0ZoTTqqcpg~rAu H%ag)e)?od) literal 0 HcmV?d00001 diff --git a/tests/fixtures/pdf/invisible-text-only.pdf b/tests/fixtures/pdf/invisible-text-only.pdf new file mode 100644 index 0000000000000000000000000000000000000000..929c6870ccfc33702efebc7fdec1a66ebe2d402b GIT binary patch literal 747 zcmah{!EVz)5JieCsy$VR4;U%R0SWNh&N{J_S|Uk9t5V@66&$J_w)Q$Ka(9uvp@9Rp z9xLSodg^yn;=&z-khn+u0S9h~U8k|;X$G{#?MKd{tFhoowCeef|!a&Z10tf6po)mBdUP(Q!$Sb{? zYQi;DC?-?^?NzybmT=mabFG5)M*;%fwhLM-qPb42Y>inVXL+oB8pk84BjsfWNGTMC z+Us2dcdVOIRs9!COw}9cU@gKk#d7`6sA5AF%Y+$Hp>jq~(fqPN2l&;-qw%-1-w|#f zZ0t_37vcNM#Y5|2@niYG`g*cDfSM_-ZRBx3ih_pPPmwY{hEtsqTn?!nVqO|JZ4C`{i|7(3U#;V zIZF`+AK}{rJ|P}~6mBs`+b{=2jrqDyYnbEe*{xwN(Yv#Td4XP!3R5}dnQp9^&sa*c zDCK5N_(ujl*tLsq%2|7-k5I>;=lQec{5HQlkt{bFI8vZ}ycr`7WtMbY1Zh%^? I?Uy6;4-YiYi2wiq literal 0 HcmV?d00001 diff --git a/tests/fixtures/pdf/ocrmypdf11-two-images-type1c.pdf b/tests/fixtures/pdf/ocrmypdf11-two-images-type1c.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b9b9bd6a669397953861a28c48111a936fff1abd GIT binary patch literal 43948 zcmeFZb#NWck}oK>EM{irBW9K?X117_;fTRv28(5}EM{hAW@ct)UVr!9x$nC(vk`B1 zWB;3q=;-RItY2kiXJ%#hIoae2qGI$+3>+}zosYR6Fib>@M0SRjFuc4(02Oz86C!}T zp{2634G}=b#MaE&oQR2$iG>ItW?}7Y;`rxmZQyJoYGPz({D)GGh>7{HE=MP4A`T8B zetsAyXGaqQ8yL6rdbL&CU3MhzOZ71=8?*+A6J0tmPGAPsFV^CW3&?)MZi&e9fh349 z-LlUPmISI%IX}o6DhPixvjckuaZ(ulEn&1DUwUA>z96%7rLsU1+k=M^p@8-w4Ki&} zJO&}A2k7%pfCbt^+G9pFKaRG#ZmkV@!Gk9vb~meP7AQ$?5oFdt9NJcnw(IU`?zaT{ z4n;q7`Z)m>OM=zTw-spY3(TA87wjDj>{s z_EdwQ9r)0IZ9J=zb+nRP+%sVbS*xMaZ9D`r>y zD^GVW>FaE5v&RSRt7fOXfWT}0%bNx8=?Wfp)Cl&Q`QVndb#jf`0P>-wClv=r%FTgG z!HMX8@fcgX@>x#Qa||!_3|x1{xA19`7seJSvwGh-s7wR{PITWxY#N&1x>*!ZAL51b zn6>jc3zm&TSum(?I*BFW2>44PAqi71YNsb40g{@$CB*KENm})JzgbY3N%+vzBbLkQ zgsukSmc_5It+aDEOUuY$8kWz%gp_B98vCs_Em_S*s+#96J+H(P}rI@Lf7vg>geTZ<9*`T-)Yv9JynxF=Mk{m!1n8!7YyQ=nW zW_VhPrt-nb!4wJfB#De18?qpDDd!(2Yh)L|!3beI%YI+#XDi@@7G_y>@nZ@KmG6z( zIq=esr+a7(9_cyX!{^H^(^o)Fo$0C>(cA-a>5M(FJJHC&Uv1`F zPQHuXc{QG1B{oIH9H|RX9!6d@3Yc8+$jJdk^(SezgtYudkI{1>Xcx5=X2GaK9fz-QS!zngev* zt|eGnMA$mp3L`IA$_HS4VG&C!-DuOCH&s7d#J+rdp?=GUf-$i*{-^!_sr}_sm{|Yj zT$Ei5o&VuqG-PG|!_~-|I2#xnI2-)S*>Ezk{mqT=3b@%D7+IM(6B(MAS=jQCT()+S z5Lp=Wk*KrDGRoSEn3!8gcsQCUd&sF6c~}{78IuU`!|=LsyV=;={DCHNv$3{y;&$UB zG5Eue3HP7+U(Em#ei$MmUPoh7ZY5E%e-iz<;v+G4cDCmR09;*N8C+Qy>>SMiOk7-C z07hm2Gc)}k3VJ7ZTW13|dRr&be_;F*MbyN}$kD>y*}~5DpJ?`fL$h^a_{a1NMs_xU zKOF!j1{MIxfAD5w|F3EPjnK`&{tqGJzX<;fH27=fe`C?W!oubsO#ameuyq1h82=&v zYgq<{b}r6*B>&{`50`%^xD_1jj9rXO9Qo}ntW5qi5HT~bF);J~#nS)o`y-gY3hy5Y za*O=&<$s+0pM#&7k(r&IiIJX>RfUn0o0*ZDnS+**k(-f`7x14b{|kVfql%rKHNTya zqmBDtvobL;urvJGHi%SAoSaM?4UC+LJ|GzOnw!azRf4hbMbCc)b{F~XirPJ=sJh~#xjN4Blj70(-E)s#?Tg`P zN2S&6hYxpP_GtE)S=gsoIG7m~K!jjfMEF?1k;ErK!lWklXCBAcK#spcfV?E2OZ_)o zvHu@(#l*_^Z*0+z3&Hd-B0)}-zUBM>j@QW5>Q`%3PJ<2P#vqeqi`u%D)k^Jrib&j>)~Q@3rWH7Z1qos0yGnAh-o1ewbqIU#n*3rC>F96UF5o*lrWL zum9M`vI!si>Ii=;YJzcQ$wEO%2flAuH%#$>hZ6iI0;41xm;1SnNFqXq`rj;y<8O-+ zws3Y*FmV*Iv$40cHL-Oj;v@oy*jd{-D*y3ke*z~_6QG5WiITYRzXB_*{}ft@im3cE z%F-bMNSIidnL86Pad5E_0siGjC2b7M{$TyDpiSdX)MoO>lmO}$#(xEEEPs6OzaltR zX2yRPcq4^az|Bl4D`_HVd+WFD}& zu3(UC59&pIdfT z_6r>^VM8THDWIIKu8)v`qJydnd~QQUM^1HldtPm6Yhk0Ev9q<8p`)e8!RyS;$jQn} zOw{;$KyTh|{H&hYVdriSn0?j9`?cP{w;$24xR{t6PrHnH%oF{lwEBP3j}c4O?*po?ph4Ysp- zXp>)61)_NMF)E^V>cUog{4(iH*hT1R`7-m<3U=QX!|6v$xON1-F7UV@SrAlIFrTCz ziU}rmMvCy{GP+;W`X##>3zT;PVIS;c@MYenZ=jNr0&4PDFilFLkQ>yKzQ6Dn+&Jq> zU_IOX*d{ARl;8H5)Mtxx&S?Plj6b$C%8*REY0jGW#z?E$!F2f2SUAym>*p=s`(x~8 z1fA7a_eug+lQU2JDj$Qeghv5vG0+lZ&x=0H;iSC!5h9b;lne1gqhn_(qLs4}C|)I< zTT)uI8s!V^i9rNn(t}IZO>Qwztr)5sI&VI^Guh_d)UOmRe3%L&myb8j02Nnz$D9hc zsNRTsm513B-O73Q;+3fD=W1a?FCUK;+0TZm>NrF~PLfVy%ZFKWdUHOlTaa0ty-*)8 zc__U_?ZzpNMZyuPuA-A7B%uZ)y!s{lOP`#vvri$B2J#Oe?=P=wRXT@y9vPMe9;~*B zH(kEYtYJ?^7ROrWMoC>SA7(`R^w_W#xJA7+@>Rxy=o(Q_>$yTqp~ICWC4<{==qyI+ zduF#xoCR8L5Bt=;o=mmzly5ers$=Ju6eAy`*ubO3P0n`cRyb4hb~vCFU&c4kh36$p z-!PlDz>-++&==gm^xt&?M7{nzZnJW}%nh%p(_;}rJ5oCZ{c&@=i3YD~rCsOlyRK)) z-=}IXT)0-!e&*h+62gyX25F*s=jLI?VOw!TrGs7aC&0WPbbMS-57|=WxW!#txtKNY?2`E}g{FQLJIZ)?5cSsrDw0Oh}D zGN5}d275Pm7chDBpm{i^cCiL~8qKSRh3h;L9_&v2Yyb6$@UzR*%Tt?z^mRT+=LvLM zn}S+H7g4chdoSfXp~7;}(yfJRP&{`BS){!>&Wh?z!9Z;0S_CyB;9tb0xr#Vq>0vYT-)Zx9?1hnzrM70==)R@WnG1(Hz>dDWhnEj z@#p*ats`E#&6hWuR``^agWsna`#!zE61^ld)ZI(GqS+>}72WB$jE+n{Ws**8><4M0 z>!e2080mZQ@?12cczvFHSa)nRMz_T*XvKzxR<7cl@^seLnetSp)E%#33QHhobtEsx zYpT1>2qphJY^Tq}xsBcB)Q>FMNB9NJ+-qgHVJNRq8Ht~qIO64G2tPOAn*-F})~=P; zaqdmlV|YH5ry)*gr@>CD>r>Tk#q2lMC` zFXhbYU}4E8Cwz~=*<-h0_rju)|8zic!~2O%hF0!ZK9*z_*dY`wfvv&b=Tn$_*8EC- z5z#_oFOWlGM-)dzErcMBMw5B2f5o~To%n+EjZj%;&cIxXN?}Kr(MZYQQFgO!G-3%QGKTF(tofj(BUd@<=j{>m(6FnBbGO zxOS#NOF0~q`$dW4 zeP0SsI&t`Za*rSDRRiC~Fr2Tttxgq7VDqO|UDI-e-{ENJX?7Nx2)+kaYfP}^;03FC1XNs^=5w2DW{BLL7-G`tvEwSAx(_x#e^EB2A!tu=N_ zU&}1j|G0=h&GAxrl^7=P&`ymB1Pcx4$bT-kGgMfy@})bxQmWu0G-mR6?lf zSP_A9OU1$@ML8eEJ-I05TT7&`5!? zQrg$hx^~51RK6#G%4SKd-Kpl$)Ec6=Ui15ghpSn$WmaF?`sis?-NT+B_e07ipf-R- zZQndPtnp;s#nP&p!LtNrzolW8SgV)YtK%DPd|=r3^kC3HYiBdr-$L|j@@!h1ma=+t zzyM#zUz|kU;v$DU>fwHErDnruMIF=ZK?h&?hFq_&7N6P(l*Um+XM1(C?sPuB2&L}) zyu~KCJr)h?#icnNV!mB#cuD)Y{H$gqq_l)9vG*8o5Jb&zyAQTHxb;N%(Rw`<8=tmuz!^d+k z_NTBecKv*Dt6~NMYw>sdjMj7BnV-F_kb%mgtIMV+4m2?cze!$yPR_o`G>NZbuLhUq zKB1h~wb%_f`BlM5Kg(eL=Iwz$kD#@(3(mspv=n} zS?xs}TDVAmqrR6FKP+!#IpcgeNc#@uI4sbAXDz6GTX?87(Q*7N5FK1`@;*cc1R@_aRq|mTmG%>_s_u#_)%+9X{=$^jx(#$e-tMGIQ%_-bx6Yi$ooGNMzn%42f z?IB>(cuiAGR>^`S;6wC>io?gUO)iXr!a{Lad{(JJ6Enr7KOnX{}+D7 zz>LN5#T9@}aYmKT&td>(z`t^h{=>WILF&8uIRgBVP4{c;7a*=991&kkBuaF1JtX*S zkXcV6CMsy--tsoY8MPm>tEllt)Nb^9v>Gtr;l91FRzP|Hgnk{-69ZjH4~1*CiPUNE z>w0HpZ5mcFSCRm2))#Rz8W$yj89XC=9TKB$d^feH>7uH(Z&qV{3iucwkrwYLi1u$K z!b8Fz=`upI&K=|Sj?!T6YM>yiGUQcQ+tTjo-5eBJ zc{)iS7aCLyj!n_^+!pa}9Ztjd9a7I3ta|2*GN~$VGJ*s-#ew-^quFWK2`!<*l5TT| zA+I)vk)6K_OD_0NZxMdxQPasrxeZKjTx@ks?=8@CGIz`nt;^o3x5)McgeFf5gLg%1i{@6~E^_3g2r^%0W|@6aqlaE| zB{mH!4;p$p>=4rld@q)C0=40KO8?R?w)Q@EeVc5D_=N!KY2==yJh*xR?=HY;8!uni zCf^_gt<#^iBu#Tx2F|nYDcK$`iPZai)G`yB0r#x*{E+^3_TCaM7cj@v}-yjSFlN%HVwW(?f^BPzK39jY@PT^GUCp>g?G z!QgQ8*QuT85%+j^Rkxu2N32eNX#%TXjm-^T5d$GGSXYEdO5(fI-Axp47Ye7TQ46*~! zuYz0ZY_a7Q6s?&P`hClMB4BmmQS@OS+aPeqO@4B&qq&!}3}842@DLDXxK0$Cz5a9@6~&3uul$Oqqtz0MQ1 za3|g@I+rr(B-zk@xRU@SIk-F-lR{m4kSZsW;iWTanGbXldy8w5=>q4~8b&l4AZKZ& z_IMg=-(x2I^3csTjxQ3dV*%}2fKRzd?S52adca+9NK0+`y6ki}wk;L)zNHcU11ui99d?rX}^@?MV%*;m;at7JC2Pfy{C z>xMnmx1?x-?4X$Rea<8A-+x$T<_wFR-r=?aICX~A&eVl89!;=v>;j%j)ZX;Sh-~=X zoNsvEs0Ct`AF}FJ#or$mZ-!R7J2z|!xpztyfgYO91QV!JC@OZt4erU%x!2FX z${(u(*yns=+QfQokADG7b$~dgz2A}MUD#IYU)SVE$yr)9&ZjMeyb)Z$fBW=THMv;)jowUndvo_hK7dQ$L25Q#+kY=Wls<+y_}Wx;i=lMhreT=; z)NmcbN8Qr7%wl&NM(nNiD6at0tL~iEp`??{XgYUx57VW~wT#8=c;aFA$V>4^J4O_U zHxPWQ3|!+!`z-Ek);ZlqfC(*mW!t#C&83TID<8hO<|vJHIj?jiic0a{OFn-nonJ~KJ;DR{F$H3Jd}O(s|KfssqN*@9f^)` zY?{Fi2z?x|sBRLCv~>U)aNkd)jCIg#fuoJx<yiSXFU$7-x3$mlAXMM3jjoxg2t&)d`8|*(I6G5 zI~-?b(g`^8kN2Z0wlgkJXJ2v|7BNN=!`@xjc!%jl$_^s*4C+L z+K?jcivyBPo#<-BRyWdMs8pabBd!LOtH|uqH0!T;?MCS1GR=c;R;D>T6)=X!2`3RO zvx}i6UutzXGmXvtcDa(y7|iQvT<&UG z5Amcx89f%q7swym{dy3A6yEm@Xesi0WGf5KRON#YWvkrKIfGQ(_GW3@HiYF)$C(qo zICky$=9$VcFK?Rc5jS398-NB9Gb`@J*_$B(DyxoQxFSc*O09fLygeFUN}_Q_s>Sp@ z3jYxj@mSmN>x7)NIrhp(eDN2esMsNa21PRyN#7y7ARm zd#Bdy56yXsGDO@+KQB4_Q6Z2=!#Jnq1Ne&PowSexA<0V{6$0vP0{N4eu>gbq2 zwI3>rWBN&DH(IpnO#D(q2i<^o+V5PnNEZsGp$V{lzXhop84QMK#RomB-L|96oa2Q3 zRaYpy4jCP%9hYYv2RZny?Wj|HG--2NH2Ffp@q6D3=Wp8}Au9Bh7h3qX%D2hEs*md1 z{IQO5wJ4{x9{oly4*3betMBrsq+~AfFvX7-VQAQpqQBjEvhsCrS0+f&rV4DMzP>K%Ld0=gr|OwGp-Ces!D1kUqyx}rXa7yYgBV_KJ+)~@$FThM0)xF~y+ zbv5;gmF@7;H3oVYieWj`e(~`)RV4BbEw=`YT6fwyG%07>=H+f;3N(eIewtsSKw^Qi zz*D!9W>qiC5U8Vwa|qyTiJ7;HkF6AV>A^pRcT$M^Ir)hWslCEC@|53I0;QWb^0O>} zt!t5EXwTKX2`!+byY4}ktgaujr~-!f%tlhj5N7q_>if9y=xd&9v77JogpNto}#$Z6d+&ca0zGxYDKa{PwXj63l{O{LAo(nq$&8|`cM6NB3&m|t>hJn!~x z5e$fJ6h=INGj{U*5o;-v_NJ9Xv6W{c8+nzfU+*k?KSMzoAGqb#Yot;p!(vZ02a#Lc zRjsQUXIO8D3w0Ud+Db2f0a7LMtTS!$NCVrGOG-CZm`i+fpJsD2T8{ZKg2?I{COJ%v z*IV%D39WNmz1^zT17NP?aCbAg!kb5!a|eRrd~3r`1PD=QQ7m$PU^=c$L0aDtA)kF5 z0PMBQ`p+PZZj&Xu$kLvUhd01@nq&#?l_)wEqog~vcTda0*d8?7r+FCXX@806WZ%59 z>sLU}^u1h))s`g;+Wh%_h4FK!&RONhz!M)5M)daCV4Qm#CAg4=>POh9<)c*4tB~sR zshygnDt{ix!kaaxmAB1MnvAx3qTm9&tr%ra$JhI;+WvA3B%pw+1{KndzW!~a8VrI` z;zg~T=5+WoL&KwCkWykbO7hx+_7ckXIDpc{kwE8bYR9NgCMo0v=Yj3=9{posdZrFGIlQ`y7{y%u|i*p8~D{ zBt(5|J@eZ?pSPd{#mnyxrTSTMCuVKVAv8U!ozIgbYZ|m&JNELJDR0xNIzXmZE4Cey ze;p|ZhU)v2dSpb?_;rkw;zFWPe#TfO`=2R*=y1vYD*C8MPpBBL&&ol?FmyqS-S#?O zT~Q5?+YP#fS6)B8AmND2NW2zrqf1x=QYfZ1H0kx4^Q_iP@nx9bsulEy{sK2vrZD=AxSWy zU2EXRWMKq$M7UjWkci_g$Dvf@tq=2-)PZu+(X7`do0wRs)^k3w6e|TZ>dUK4HQ$kn z-mX8J*^ zwp<53mh&31KFyvwaf2S+rt2N%s^|m=o_B7PCGmTClA~gsqI<;!1=CvH3Uijq&rVl^|T=gAKIWEMAS(D5{4pOGFoe$?Ik>VF4lCj6hY7K6Ams zZC@HRx4ybT+1G`>Wh5X=xYJU-ajB3a9Jhxn#q%=Ot@r{d^c`6eVlix{hwkP;1KWX5 zx1$Ur@7T$BB0dt_OnXH(>PjjRYWHhCrLZH`WqQcOHY_Q&AxyX|`m~VCbMe?XU$6(K zXbiz3s^}5I%!~ZD@SegQvD5bDjASao?9a1t>v(94vrBnkiW`9X*@<6fif-QBY}?oH z&bntA-5d8o7}RuV)gW-zckJ6+F|mjpT(ET`mkHYH#f7}*$k(2lul4>Fe4#J9ctW2= zp-P))yw#(wG-HyWD8fA04FY(;1H_BhSF<-duE)_9AB&82BYIoNQH1Pg#W$x8WmEm9 zhBRSOj`;Z(a@RxONH%-a$%W*k@g>#8rmi5OZ3gU)o{Cbm|u#0sg*o@seGdKO)!f$$WEG>zfr<4^tG-@YUon$WUB#C+! zLZm#umBpi$maOVpAJ)QcE+M$4~ce4T&v#dWfK3#WjK>7!lb z_jwnKM&s|(EXQ<{_5~ii^~Gety%{bbT-*%Y1&tGK3aMLOc*JOpkM8gjc9jCdExlSL z_x7#STx>k7KR4C~IV}vu?amRXuLFX4JhKGbpDoAGNL(PID>6a{#UT_0D$fR6`SnZV zLnnI547Y6byFB?Iw^qj=gU%<+plO0m_)#yLe@u*toUP$YZOnMh|1!YroOmylOz<07DDD>Dh$;-)2fJY1ZEfSp ztzWusd)kvPi-l$>3Rni9?HcBN9*F}`F`6sCA(O6Ch~8T(JmB*-I8OTZm!LrM zikQ7B=Tk@__Hc{lSodqgd`+?+Hh6L_p5^ObiQz{WL%aSBagdk%kW78qdk%lFx)-bU z<<`Rf(7wQg@o43iGuk!2MIBnSc(4VLfi7Pxg>ZDbpoRG0fxWFpCE!Q0Il_2Sr+bluNzl zV#XnCXWC7UYN*LLLR{N!P-BImh{*mV6Ff0c8X|BRp9DcSOCmQi>+DvuYUkMFyObmO zvE5$FammAb9oO4<8aqbB{Hz&AMR?`2`WFterxNbIAUi@2v!cVjVLcJhoRW`2fi7{p z)V`INMyjYvOUS4GYS*v(Zm%5YFY6KKmmT!pS2AJxpIg@!970b|)G(o9>OTqJDp4GR z-_?iGW4aE}QUiU94Z|rB8v7_bp3@Sr!APL4%Z(Z8&M^?L1{Au4Nnm0D+&8N4C+z63 zVf&G83yb6F8c>3_V;+2|G=~~<8|E@Q=}|n}TJ!LeyzNwY3WqV;b+1b^bdQ3&K5=9G ze_qWu&h?xs39dY(K|)-;$=FwfX%(wAC@}UK{<=TNzCvw@ZQkP#d*tyREr}cP9W4%r zJd<;^ z)5>Hu7=;D&58tcF+gr5$7<-I5+7+NyRb%JJUMZPGo%Hs!Rgjsp3H&XOH>^&*uJ@9)7m}YM*3BJcJWYoO>Eh0Ay`i7>@%94~gqbxy68ju^k$Pmz3<{yuO zHe}76O2A{2g{D;U!xMY$3w~G03n`eJJ71a$Nd4$M!5z~3g|O*xw%=Xg*Y3j?uO7k| z;}Y@9;$%`8GMo8E?r(v8m#0lS+me}fk4cPDU8Z7Jd%TxpSlZ7|gvow{I!HoQW?v9r z7puOAn}hh;>Kh7YXSd`8d)=$r`{8wWMaU=X-6Mu5rOO=9_Vjhpl$v0o-PS7wqm_o37=%O}_tw1sb^6wOB3RF5%-2u$XO z*FCxjjzs)$own--7!|X3L)P?!;$82uxPbJmHW|XMQX={;_Zy@8-K}#@o@%*m*DxS{ zerN~Zc}5a^@)3rA^476D^&~vwJU4bBDAv&3Dr9$$W`X~p%0F&yHVz{H1lMznya9|7 z4v;C^Np~fQp-?V$^uoJlbQ&cMri!6KEZe zVdut&DrVZB>$geXi3t5Rxj5`@f@hU1#Z~J zrqN`%U^EdRjWyoJPMvpZtQ&3L?sv3oXl^d{`D$g6lIAhwXA=3m>3$3}ZMTVMOV55$ zeT*EnpX}H90gj|^*rNZ0U0I=iV4g-msc%PdTAAVCK;cMzIm%r!Cx};aSGKdJ>)EN` z_xoP2WtTb3O!%e~XoPDzz}?q|`!l{85ViXA4god-{2h*)4k6SmTFlqa3hK5zrh>m5 z@%vept?5ta&F~j{jlRvq%&Hls1L_2ZC<#=E5Wh|EEdf`?y^!aso9n+5fm!>d;YaWgH$8Fi z+ZyD7m`fz%SlPEYwh#Xh1&k4wvi97L9N*7N^8?>vd`HecI1;}pE3#Yqd%6q>Fw>L1 z>Pnf(ph}-s^WK9HN(!`~sVi}cla!-q~L3@OlT><+VATqoJj~wIEIGqTtJV zMPm)$DE&C_O4K^}c0bt)*o}Pd)eyQMYnBHKs*n!eu}ozYzQ=sJGf_93$W+pzlVv3#A;xNMfNs@AB|*N^f~<&#*Bw^;4T0)$k?l6JTiLY~Jetst9+| z%ps2A8h+|LaZrPKP}J<=#c_9QN;^N|9>Ls5T{bz^?K_Er>+L}KA}UTLrxxPw^@mE* zI`OH4A#OBhMs(VMwY&@$UA2CxIufS9db&2mog9b{!n`J(q4boN#80s z=;XTlsFK~kn5Sp)bKq^{Te{gUyfu<^ccx#8hpovJ7v=*GX}`KKgyn+MPc?^R+)*4h zR?-1G&Glvd1t;CDNSM3lcJv;f$H?BzS_BGS>0~WTDw9(2LOTEv;c0F;`Jy^1%z;jgI%pEFKQ6z z)g(9R&K}JFerYm9u^{0HfKa~hV_CC_jZ zkT=ac1HVS;l`GKX|Y zCrZnJTIn(Y{I;==oj(KEh0(NR2#$lnm-iHR?0JO(z$&tRaNZ+d?BTb;+KQg^lB>-XG+YTZ76mr+bOu)qny1gO*WIb6sMmi z-R^zTD@x4SZPc{xaZhOW_N7Zo{piK|%qvKW@Vw3EkbV4Zl*TKXoXQf3vv(!f!cTs0 zf_N#QJxe)8rW0`cQM)+tRywo)OhV1}?!sp(2+11_9)giLXTC$*2?eKb&uxw9M^_u! zT=KpL-gA9o+)+HTnk-)8?NEycY3*${4t^RHr0lr&vm?>RZbn4Ll};GTbYz#^`btp7 zDAx8FQ%Dip=A8COveZvY4+Ydrh~L#?VBWKIt#rkz@m=PA<9j0xIo)XyHsRvD$a6nI z^1(;o0qBArubeVTIOZh}1MPS|l)dOaPl^RcgqYFvxO68l(3(3mm3^KKQVLsuZS2wb zPG55a;0Rn*RN}EMl08iMI`wGwxPj;)5eP;Hu*E>XTH`dY3KIey-{r^WZKKY-CY!KU zx_Wi}y&m^NwRLmwzTOgUl>Q7#?_d{#5K+JHia6U|J{J>$k_vp{-k;fNd)s3|4! z!#n%tsA-m`@fjjLYYVv#A+o$0p@u9+M!wvkNZo?Yl%d$LNLT`eU=)7}tc z6zyE9fY6tQ@65DLdlvK4>zFUH)@Tk`)3#e}X=(Elmnw#(kQBr0a|q&^Y0Y29M_&}= zQTXvkjCw6^^*RTo&i5}#4QTITnt4y4Oz*@zu{C_I=gC3n&hKis$ADA!x(ocQs9ro* zUYga7@pqeaQ6>_LTxUb!=k?Rt=ii!CV?q{a#lIU|r98zTAe!_RQ#4XDje?{?$b&x6 zHgKb?qmuoIf4liq)wgMUy-z_V_x83&?dEXC);;xITlFyXwL4l|Ha}h9@D*yd3W}gO z?{^Ok?`I2odVMlf`!Z#KUu0ITgT87wqnRPj6S{m0U00u2cdHaf#(u6An{nIQ1Ze!_ zb=843iB+mndfio>nxyO}y6)1hc$Iq#bw}6&Z%jn+AhvS4gR$kd&IGeCT*Y$Tp);njadge}d14--P7(q$#$D9wOmEM?hy5yCcBDrFMRO)T|<*iKCT0!F} zz?fH7d3#DZj;E?vK7uZ*>&-D+v|YmDn&T$H=Eu$INV>RQ!nu}Nu$M`{4eFA7pJ%1^ zp7}aH4jtGv5@7zf2y9jya^1%4L#l5T7q<=CW$560O5nco?<-vczRF%eIa>PlKBhtl4Ni?Rm`3 z*P&)?<*D&hl~ujVmC=jBJ0WT0Y&Dmb^#kfI#d_Mjo^6i;p4&mtYd*){Ti%Xu7SgBJ zV&#QSqp&BDlE4y{buossF-2WBPJm(r6Wsf;(8G4#NtP7N9WgSF>=b(uIqw;4O=ZxI z_1jDM1Z^ctNkMQa$D0j&?3OLuD?d*ln`mV;xq>#1fhLb;OYhsy^wtY-9uxz@@X{`I znlglvmiNIqM@l`jsFW?>OFNcc&nBdTUQeWpm&M5=w@zoYM&hXKt_t|lO}9tWUdvOv zClTG*@p4U%W;jpf8d@*{EQ`2Qh8h*R!S~8bkv;&2x{#pTLMzJ8uGa8y1Ozt=DJ5Iw z%Be(YG0c}L4a?F~Bp_U%&E4;T>I#X)PAJm==<6B@H^#=vgn8)@29j`?Pe=+(#t+1V z%7GQh`C2)MN?uAZuj0+ax_))#-_W&amw! zx1G0Ny`{0^U)jz_d3-8k1OwxjbI5~CGY$f)V~kqE9Hp?XqH1W9DYmaYS_1|aD!K`9 zle?qS=$3;l`=zKYJhY2v=|kjQ6$Jyaxyu)@SFOrFj658-;5O>Zr5lu)xl7U=BEgcZ zsgc6vIVJ*l4xx77=JdZ$9wUN#lZP3KD|AI`^E zw)0$c3}b%9;e>6AX3W@$~r&?BgifQ}87GY4)Z2h95v2N9oyiSd7Ze4VVJ1(gDAW^Eo zeMW{%$NDy$=VzX}V1RZd@>R}GR|YWCr58OZSllXr83ZQdEF!B+hmMtnIdacD zNVHp-ENl`0`@K5q3%y@^sJ~Nt>|_6IaiB{8lPaBu#|{eTQG$6dl{tT-Bjft-fcOLu zP2g+&X}i0d|6x!6FCma(tFp7mD$HMAB2NwC$?N!#ew2Ke- z!HBoeHPN<2feDiO$r^_=x8!yjqd{*xZ@Ao62hov*!!0xp%}*{08d#^&Jii(Mt7sX_Mj4s5Nj$gIW?(intHJn`@k=I+JUS;Y@fKO z3R_`-x3ZGE&LfBLf2s{IiWniWqXjgIUS2m>LfAv zK$(4r0%yN7!=TV@1j8;S@MRspj-gC;|7ul|Xmm6=^omq2Q2DjR)T7ePblAG2eyfCX zdWuq!=wjvQkf9QFZOI*>*Qh^;JIpgkRK^{A4(%O=y%k{``9mR;ofuboPf-^7=o0?^ z(_3+u*wI3G)!r-xXb`(0?Pp70&Pe$OTBcAD-g=wq+|0&GH(u9-qNL zD6P;VZ$g<1%*F2~6vkOAIdWd=?vmsHbq(h~JGivbc9L#Sq9s#KB+3w@ev2rQL`Fg( zH0T`gD`ks>Jgi7%`YLc&!pz|Dfj0uhZm9r)W}BeIi>F#J+eo3bJ+^kFw*Q(1ggNLF zvNBXRDA;3P36xh+Tl1-GEli2)WQvtYNgyKJNr^@vL**$&KxU>OpnF+4mac^06xKnW zsvvX-TwdDm1eSZ=UEezp!xR;L=j9OsU9U|>kjiOV8J`dF^g0XFZ(OdrAIrOd`dtxK zZqmZ2Z18+4ys&oW=gK248wqS;8VF+O?Hr86MHA8>`-=WPQxU~ZGmlRD8Z$+y+P%L0 zl3qM9Aim97TBNFTmGuCM#a;nw*PYelT-MAlt#$T#__K#LKZgGfB(e8`RZwmjte&A)~BdZ z=a2lrNk)++s!2cr&*jv~=4}p`Akj&4Xa-9cf`VRf6E6Z0N*5P`55J!ZD%J8*k&e|S5htQFb zFyG&K$hGDi64r!<4$b|S0nA$93YrIQCJj*8bz|2f*6@NBsk@2GIDRwCgS545Ao(WI z>ZVp8Qlog4V+$Y&NnEWNOy?Oh#5# zSl=|nyjQeqbO5ut8qANQ8};tDw)4zZ3OBB=gBpFzbPfT}E00;07j=u5Z+$%+1nI3O zPK^~yHShMGw|&JD8Qb=2Nk zZbnJ#65`lagoKC}_`16vdL8a${a@_8Q;==Zwx*l5S8}Co+qP}nwr$(CZQFL{O51j3 zt$pHD)jn1G)~%|j8*v{h=6o0ry~Ui-TkB)azmLCvty8!CIHmaN?;dyNjqhRb@&32) z=5@ENqi3V~!p^t-N7~QaYocgOcK3HnvY_B#Cs>Rw^Mh=5;=zsV@p@#>FMy6IBVCaT zcEFR9+Z%drB zEG%DNhFP7*Y=Ajqk`D$A-VTEk@Y<`Nk?vWm&yVl<7g5@?iYHH>r`wj}eMMGkZeO1= zZ|Ct}wBL`cni&_M(7C$pHhP~%yxng{Ca0ShFVdP!PJ5E~$FuW|oD4P%q-<}YD4iKE z>^%Y}-$psvyD$LSh7Gi}Pj|mg-cxTbJMG71eZFF6rspL+{^rKJ5hs(!Q=RczmbhwZV~2g`Tl4v;wYuv})AIS6cl}S+yAk~7iO>V+eY~ORXxH2P z3+Y4EI9MA}+9m5|hkE_ACcH}xubs8;N%2fV4-eC4dM%Z0?W*<0OLwmIdDq6}eNSL% z#yF0*&lE^2$*apFaWDNJHfIyNjcLhT&%N8UUYCDV#RcX8fGSYpmx(4W_xDC9?}r&< zCn4JR`pcrO#)^A+W0CK+!|o>Ux1^`{@ul8}pw?AouiNb#=k>bXj-aL}4fA@T@99No z?`JRW`+1ga%TodA-2|rx)xG3e$oaP{$lS^Roc>Pq#;DHSYIA+RG-f=m>DYT- z{@L)Za_ns^dinfZec)|~b6QxFvqG2P?8 z3HU#wkor*0dyAZ4y#9qwW&dZLDq!pOS6kBkeX%go(XcY$v9bJRI^q2lmH!ekB=Q%2 z^ly+FdOTVMV@F$O2SZ~=ynpZpMQs0~asF{(Lj4ylh4&9EkiuW!&p+P(RiFMN{J*2z z{5!UYiRC{^(qg4f8!UR5%Nxp9;8D3EA_+c(zF5b2nm(5hO3dpTy9uxE^3@s_*iB(O z1eJyGtUZ`w)-ZuG7^7Zf>=6#no`KxOD+eUC?|@28xPXn`3dHe2hhW@VQ;y9ZY8`yf zLuCCo@Yvf#GPeTsFeR(CtTQyO{(HI z1!58xUKI1UrjLR9#Y8H8IUxqK%zsWFUSC#$TtwhF4pE_(k7Cp)uu^&LwK=D+2F{nK zYer$3tlFM5*5Vt!a6^jxGHVg(thz8aDQY%obU44k%cbkh+`o!w|0(A=ROY74G7hcD zwtu3=`n@>mK~D`Sov_T4ad@-~HAYhJ2&sMB>9?U5LLP#zAIk{kL{jZlDQoTBnNcyF zy{&w21~o0kb=>6(ZxPDV#A>WHnmt&o#ak1fR}`C!94o@CS!$AWM+CA#F4qNRe7ozR zE_0J_j4$%vmmh2$KxqmC5e?_JI_u63iJMy22l-6AN=R1Toa=wC5TCV>?8$@n~ya=sO22t1?z`*syi&Se*)I#|SlS z1j1|KC4hk}(H|;%qR|iydCHav*Ov;!=e-BEkPx8>H>#2wm{|b}V-aA;Pc*{7PpT0j zc$(f`Lux*Pz6{3|hl^4@(r@c(}2|FPWg^8^L}`1$!81p9a30Nze-lQ8=KI0OLT z;_ra}y_l>2#n1TxK=>E+!|*>>KlFIafA4Jn8KK3<$oLyE(ioTy(Zr`DBeydUT<5FKOzK1ni1bU9x8-wngSWZtnEW`a5k{j`O&U;3TiV z87f#^8vyVxU=V;OJG##w_yqs4KxqLmDLq7zJY+&C@ZWl1p+J-&Y{+#2SP`IYA0M5J zYsQTwOt(}O_~Pk_A%*%&S|h0n$5r0`#SY!+(|(}2xEL={z?pq9E>Lv zqV6}mt1>1GwqLd@o)@}543^?2*WcOQFY6~z@!5k!obh5r>s4V=azo^SKtan=Iwqr& z!d;t0J{jCUL4*F|&z@qtvl5nIgRau-PgM^%rr+;MoGB3-_?_e~H7<)n+XadRly?+I z0l|@I(q2?0xR|I_+FiQ&D`bVrHqa;3T4>mbMJgd7!|l^>83!$9{Q<$W)Eaem`I>qO znp$ZC*0mp~t=f0~HWvkj20%&@!6r4Da!nG2OGD(!`8Mk1Q9;r zPMet6MM^`&(_gChs3<6rk+F&p0i6R0OcQw1zWVDpe5vw)_CF$%LAy8Om;}rbAjJaQ zsTvR&tgF;~D3`}F!r#MxLd1mSvOKip@?@zBl_&68VJqZ{CHV2v<6Ip#QL1AKZ_Uig zdcjTi?vEzP*q~9Ncdb;x$_rh~Q8Iq~mw@HthhKC3LA9NadM6pyOg`QSpJYr6pI^}q7fsB3m15CcGBtJU|*_r zp1JtDm_kY9Mn{i}B>WF^GLre%qn*nSK$6~3ErpMH>)wn)7OF3~}uI;&P z+{GNE#;Dz=9jcFOjXR8^jsuG49yTbEz$TVRevr_nbG@%Akj5*nCq)mP6Mb6F?WOqiihg+|5C9v+DV?uNo8TcG zZ3LSpN6TyY3Uub`7Vs^mxIM<5#3f>;-R5Y%rfU-LByP|y=o~>489aujo~!d4@@)WX?)`M-VOt`)BZe~iUf^Q;Tb>?c zlIr5gR;_a@g83B36Jm}(aB{5-65z$SwiWx~%r`!Fu zHUAQJxUyB)GZwyub`MVb#_St2kA)Fq2?-IV`nG^o{&x&lYmv-y^37U_%=nwz824JQT@ zmo)VxbRA-_Mwwj=`xTh$#V!#zG&^9LA?s!Pv2}-?;hcVa)TXQK#lo6sQ0+!GfXmy| zTGJ0@lM<97)IvUf7PQO#)1)U9D9{|Vb*NJWf4$hP zA;g^1XtR$BME~h<>zH>|ZrRMUO|fryd$OeksV;3_P|5W;^V2oy_?6U?XR9MhQ)u0p`jt0E zU*p%~xQ9)B&{Tu5BdzSIdlpZ0v6m!BPvHvbhL9K0pYd)2eAxrs_TmJqO)i^sK1O}E zke@=~rTl{wNju(~nQn@7GekGkXpb&Ya54GDj<7J%o~N%}$foW}@|mvZv_RTrs)Vt|KnG24~=(N%X0) z)tuLMwIP>_wm?EF@#C-dU8pzxv3tDxs&_!2$o2$`CDxhU>V{=kae{Yy34_9?)LP9r zF#J-Vb{6hrD4rg`?lEGAG>t7!SZ!pSkb%|Pg=0q|;WzTj;(WYbZ>n6dwFEY=;tjDZ z4a-PKV;5Cd2?+ujW08X$2ncoX$VxV z15{TrnD1}VB=OFH7aZc$EOr;fGO2rHKaQPYy2Fueut@h%D-M{%@^G1EOI6Qp%k_#z z+5RUEnZe9X54_Id6=yv*@m4zH)+j}sZd=3+4s2^Ire5ro4%dLzE4uMULf>|)9-^tXLzG;FGRGOR<@VEb;2jQeL zTYU+nIN~tKb3=^|z-AQnh%QE}s*@CjvD$%=8X{u%;9>$<;KG-8mL&xqKUcF_hsF~r zyh{Wv9bD`Wnvp4R!zVFx@rT0Qa8C-^d+k4EA@CnuMJ`9Z?{n=|II`~{E&#%R{fsIl zD%&8h$2b~v zsATjkFWB$8v>=`TauEESwQ6hiY!=GM*Mq5c_8f0~?v zh^vM`;~dt=Ek#6LtRTd6pNGQ3v@Vzha9j&uH^`3;mf|Mge;8AuG3q+O-lx_on@gAa zEGZ=5Isc7Df{$a$PAp7)MaFwT^+!Y-viL4Y@|jIYF1^^?_CeM)>=jgPNEk}%1X${` z=^<%$ojBx%&R!k~El^^AfiLGPCK3nD4gThGm{u`eeJwpQ9s#a)-vIQ1k65%$HMA@P zes^gJ7RfEi_e4r0)bT=)%WZY|SL)U-6BF}35!)=jj)FGY1{sst zyI%VlOAc*E)XC-n#GeUs;y@P4RHm2?U2t3MQUgwZtD-|Aah&wYVyRONn*ki9RX@w~ zAR2vm`*h0;JGt1*h`jpj`W(mQ`MO6YcbMKcI+?+CcKS`0&u zUATct^;fmU&xA9}#C8lG9NiuB>o(O$ign@8fCd7jBuH}n-Iy^F)Rj?79r0J9e0kuD z(N&$~z3ICvdcw0E908dYQc{MW>d0SlqB`r_lY+FvKc6^452uO7};hY>~-_v?H935BrQup>Hbp>)hHjq{)4gOsp$ zCT-zaIwP1oGH+|^!+oB6vhA;Nr*xo1f(rGqknLUH9;#O6Y}wxf21%WAvtT1@+e@5K z+jH^ssF7MQhUX?Rl4!KKm&ZfplG^)+(GzY%((p|V zrsvH3p#CswR6+Vg`t$6?%2M5Pt5pjcV)HkE#mgK zMfiJRv9}#0yZObNpTwmqAx#d|o7TPDhKib!l9HN|yb_c9y_`KizqvlYzWv_eD4eWI zg9%sv&rr5Qy(F>@$y9@;&53EM!Qb;R!b;TVNZDxHh~8B1`U^?ojwJ3f7EP^*92HobSfg0HPE8Ui5-s8r#J~!i<&jPyP5kfuhi{3Aq(6^od-b6*vYg}D zI*t~0znwgGjj|bhBSw*zlLE+1OSas--VZKtW!*o=4y0p$7@Q}TgcVn1r?cL>8zUbo zXTE2mXGpVpUD6Nlt0oE1f+e#g(m{fpsN9tk*Kzc)=OBx33JHE5*ywn!C@Eklc0O-;p|X9Wj6It(!*U8+OhDaAsEdac zZmWn(oW!ZBTt=WRS(!rRM+q$qD}Bq*h2WW)tS=~(6W z?Ma}Noxl+1jq9)YjhV1C%U_!rl5vlCy z`e^!lH?c{Hs3>J4qu{c*NBr{9r$}+`a$Tv>k@wKHoMeN?{#$no=SBU*KdfL?ru zPa+rZpd&UV5@N>$R~)vy?x>S7JFdu+6>Sdtfu`uG1o7$1077yR4(3~1()yyfFuWC| zi6)h_$O-^;iv6x57E)B56_QyVAKJXb)Gw^=rglId&{@OlKi58(*8nCfU6Pl2 zs8xTCrMKiN+YTn|F6`T&fjjkny_#@C3(i_)oappt!Lif`-F$LH?!60vkrH$*Q z(>;tjhP&ud%62NnxxRhafwo7G$5_Ex!x` z*zZ~YT=2HXsk_bb^=o(BNg*>*>`>$3{W+J++U%0BfpjJVvKm&6v#(8E~E*=ZPTEv(2ef(N%Y4#BqePh5TY zbX+zXVfraJ$)rPFX+L80kYQQ+`qSo(-mg+wog_}?EI zYudqgD048!@A(sOO3T3q{%0Q53^4onI;j{rp#=A*)mjom4pca`@oMrbvjfDs%>-JZ_aOZcSVDM#~a-!<|m4(mq&cot=TYE zT8pO}9bl>ahc|-bdn9TDGpPN%oJ;;Ts%c`guvAE5*x+=NJ(u9qOsohwVhbH6K!>9y zRalDZ4MVwrypK$CftY_@RyF3L9+ccua{J{T60&RPsY~rc+R6jU6)!y)AgVF(8w%Su z*F{BllbKo4+`JBRoq)W`+#CpYlj`zwpy+p-UtP)mRMN@?b1He07&Qz(4M=k z_Y?WLAo{%PkdvtLkaBIw@`q{j>mrsRgIk_(RjEe1P7BoM<^WAdu4FefFA!XZKSq=u zObjV3WeAG1{&wY$E#8NUn_OJ?xK%@tTf7}t?EVp9>9UDw`=8Ldpx{ar>sCob>2_yh zauhqTJEpqEk_=4YF@YYX#+&1%3%%Mp+4Z&D;D;yW8>pn&rOVBAXAY0nUw&)37He_* zfnQStHPq7OMH+ReroX76bI1?YhZVjmBb!j{u{=+lJvnT61& g}%n=-4nRykclJY zh@0%=y|ZG@4)s6M!EfkvayI-c;Vw@K7j&Yd{NZ zfLlhS2#;^>=HcK&;3mkI-5{x%tCP4CiH7YPH0-BJh$Wcqodc2pz!1uTDLj1aE08lp z`^NODY>~M?bMwWvkAKB(c(h90j8Y4FP6n+FrLT+R~2Eu81j*1#_sb)&dp=J=Jv@ zlkPnTIf~TDq@7V>#<6X~b=nFwXLHLy8B^)8*pX;UfWp2-LSsWi%`8B!bjWOyEp|Qq z8;}dL#}i-;z#lRV`JTjo*Kbn)AB%%mK7Nq7$ak3>0!=ud2zLzL;*R-x^#FQYCI!V zdbcI4z1QBZGF9qc=HG3W>UYIEAxt;c$d&oZFt5G=Xz34{*O*`}NuS)A{hN%CPT9v+ zFMu8VPZsFZis)6d1;kI}(CDrVbyo73tyY;RNr!Ww09}Ah^1aOC#2BEx$Fxu{q z7U0Jj4e>xq&S@1LG-bc1Y^SIc3$M|_gSx~?9$chcuJVz{0$U~;7K52G=$Sc_v~%P= z5%%adTdVzCZ|mKfsZ)MGANLwpfMxvVfDRfd;L1`s^M;PEZacwgF~!*(&A6~hm4gMf zXu;1to2;HF05kIX~*O(E$SlLv?2VGEk-abK~(>4nH6CNjo> zG$|Bp<^#8w;hoqM47RGIO(*B40g?z0nyCGDW&b1*#F_|iF2Zo@;W5B~ z58qQLtXWVQz&rn^1@UHhHSL^4Q5em-sNP_ivcs>N-@k%7q}?wCkau{(Z{8%p4QUB# zivDve4|0}L1Q%s3%U2&SxL2@rG_j=%xJmyDk=97zyy-|IEI3F=PS8MDLlbFW@$5D! zXXfze2zvrBDMeYHa&-B!)H3E5=|I(^I-*_-HkO(7bppS$OYuyae2y;A@Yt5hvSYb? zvvf;1G1(TUc>Z&AgsLBI6-IO$7@B_5_trSn>`2kCN0xQ{OH84KS25`jumZduNRi$B zx!rD&4E7J^{rRW{7zlxuT#EJ1lk|DNoTbD^1oz`f+&gHso^!Qv`FUIq~RUERNzwFReBzAZCFbxCA- zSp!)rPk1szODAsn-<};?A7AwmZrHfxOE24VnOk z6Sed{h$Z&L@PRlWyubC5SQ7$M5d*ezRnk#PlP!$>ejv(tXLv8WD>Zr~iX3S= zr?di=JiaV%tRa?17`k*b@ER#HI21Vo$Jn_S_D4+C*6Ydlw!j|qm3TNP6x{eKXf@7B zSioJ<)6={66PnC#=A4xpoA8l?CGRGxCC}3of;8nFqcAeiPfY*J#iLfk+vH|?2*ti% z->C|wJhB({MMR|SH_-eHCgK`K?d$vf>WwWHu?xeQqL%bI)aYg!LVGaQ`~VfS#VvUceUH`iFyUZ+9rvezrF<<&r~!(Jr^Wn%jMG)F;xhD6^Eh}m47>!hbk64n z+(rRB)SMolIyNSQ@T8df$alF3-M=}+mMic<*6-DBC1vD~rRzX&>6H{=ZuOrVzf4mm zpZy4C3^kXQ=^ilaV1W!!IOTHGJTlu9{Te7?ccaGugqQOkCbR4aq}+dx?sc~*Sb0PxN`heSF_z86ux{KGI*R< z!$cb>7W&}VodV>uQxRH*r>@Dw*ePJ7XD;n+Bd^h~L=$&DICEre%Nkj{_(Y4f^ThIU zB=g&fItnI?T9lXpIgHnZOAyqTcwF7t$}|){f`V?zC&>Z zCxxVHR~fHqzvX?fz$iLm@$ZqLH5n9JOd#UP29T0J0%#X~Sh|1C-~3Y`T+gp?qTOCs zrGF#2^wIT*J1=btbd0Ye{OQxN;8~%j3~tDZDB3vPcr@Ybo~{OVI4f&d)N}zruK$yH zs>DQ?Gc7YcGtbXrge6WMQ5HrS6$n~q%t;kY9muou(fez;AOr##3*~}x6M$zF2`A62 zVT+)~jEsgnkbrEOcGh|^m!W9@&F%@x3yu|KurY+CS%g&(?cA#Qaa<#BeJ*V-PA$Ls zPIN&?Si%A*&n(cKFGiO8cY361#rn3S;Bzf!n88fk=H>BTo~x1VAm8Y&cRu3>_B@6@ zh<=WnfWdOZ|eFahD7z8PmLzP7H3G_HP%u#wEO_D=U{j;<2$tCwaBckB}QB zp_Ws@tHJY!!cZDM#dIL24+{*_=Cr zaGaq}M!rMErJpic`Cjh{K551nVUP19M6<*%_oxjFZ7sR-p)94OHbMQAidE}pIz}u~ zZzAScC#Lz6TrT5|EtB*;ykSTws|pJ{#>WYf3#$jKqlCfz*|@1^RFoa{k0GG=$p~RS zVw8R>W{-m^4{1uHLp~ooL_I-1K@Enl3@t04INzTcOltPaC;(AJ9*~U#maQ zyD@(5$w@;>ko{#|e9I{Ldl)1t#s9m0KcOI@V80^hJSYRiQB9p8+vTz2GRzB~dt3`} zIl^A)0T!d4sZ0Cf7H(uNV;&}0PEp>1Js>KSyh79`)?!Ct5iqFOh|Qcg(Tfi|Ki|-u zWLgD)j7`g~?bHqtxu9i@q*$O(!3|zWE6$l;Q&`YYP*KqLWfhswhr*wEdmrhn1A{LHG6f4URG_Ikv(7=z7{zT_uW z*SzQlTu+bO^k4XE{<$sdzjTNFKly9eS^pj={AW{LvIn@D(rO0_;r03?#*hF@xha?= zRlN|?2$E-iL;wL4h1iW?X!+D!a4ei426~7NNIZewO*dxev6@WBxVVf%jj8oD!OPF; z$l=!E=Z|-eBd)y3jdG7|_s&N%b3|1Ld?Ql0P@$IBfcL%tay(;;kjN`pkZPh^Itne? z9o*on{mQ52Ah%VO>aStn8C(d%*FW;2JD#S0)V{CLDN|}%1yl;FE~`hvIfP1uSNBoy z8?jENe7-EFe?1p}_w-HYf=TmFV&L`|aTrKT9~Ka+oLWM=sT4V&Hlx zLfdu$LaS{qu(bBQu~kGwgx<&mfDB4GUxg_Q1P#hP_~GLH1rt+5L56}}-^j33z`%v( z;P>D(3BLq@_LUZY*t%r+Q-_B~K_Q^E1C;CWed>Q0hSx)RMCJy6W;?rdUO)Dh_v!&5 z@d4=<%Rze67qPvNzu|10#&Hj;;oxs0BQZ3{qI!Lzl!9K_t5iUO$$-d+YF9bg+Jr`i zIbf{tjMfvRtHu;t(=5v!oHw?XSpYm z8-%$9v}HG&Vyo-21T8nFTQN?EG!vFYrLh_ehHVqdM7J^T4~OB3e&UXcje91fCqyIZ zhAS|TVN++5IwyEYbQYTywHD15^KNt*Gfs{yiWoMomMk4ljlql7ptoBZ4UBOT`qpDx zb1tq7yTn!-EDgy&h_uIWuz9?mvWm{4SL14Py)UDiMA=Inr2VPiS~E9n9M!@6sNPoU z%(T7qI-ps#uJv*RayO>@Tq^F`pVbgr`%Y%o30Jj;OUMPEY_mtwZ1C12@b*p69Z&z( zcmmLn&z``-EL0dC+LLaaLsUbc) z{ktAbD#V?;N38`t9?k`hEmz_d(LB8SRZ6e-n%0kJ;|C43F*&vqfbdkxa&cJ*JIWl7 zur@zV!sbu=kjH80uPg3cmAtP*&+AHS=e;^emEaIsB|=Ru`SqY!CQ}xD=L5vaHOX9q z)%Ql%+Z2vY(-40YwxCY5tyc4mDM+|TS?eBh&pQasgZTS4Et{^P&{fA@L@p07zlhgz zd`G%z>xU^5W!ilwc&#xvKc1#X*Y1+CUGPvf9rb-6#lMwGUV5}sXSOWaTwZMiWu2ew z`Iw2g{JC@V7X(2rV>a4-u2(4gO}%(W31eKbLpV?CFY<+PYqu}SfNKl#j+x2ua@==@ zt?8+RT$B@QJw64WF|@4Oe6iQfYpZxsy>V>%(sNQe!?X@u?RSx`8CQ896*M@r_(nH3 zVzF!eNX1^fF^{?2{_tP$@4N3a?TlmZGLwwYHO*;n6oVU`_dyH@esL1_z<0dsW& zofkt>D7;Quzaj@U^r1%<)aNMTH#5r9%*%s!=_MG@Ud!$jXkcw{q6BV1-5yz4anr_S z4Zi*oGsTFLU*`sXXm93I>Sfyokq^bDKaOR@_T}2ClHS++p=zf; zg~GJs1b90{1o#e8_ev%-_glPm4S3G}Y#$|<+G#mng{L}Jh(MPXkPyKN9)=GKBVvr} zW~Jn&trYsk-h$&qvLh>70cPpjdU{Y5Bp6B-S_x*RfIj^cb5@Y*8(;gl&$JtHGWwZ5 zs7~GEWJ_K^@z$F(jMZSshkW81z3x*Iu-Df?Tvl71g6E~$_<^1IS;*L7sEZ+^UcjyMzCDswc)yY3w)rt9dr?;7hSg7cD>}&(s zfZO+h38ShO^7YNo=p}ftfz8)WfDwmNUnImDN9RY~=4CFv^{9`1!!&v)E73o6kD+60 z)m_l#Vb_MVeH-mT4Aj-pwGx-D@JUDKBK4Z>tr@0f8LP`Ki6AiX<8^PkV5PR2cF1w< zk5~2IK4he>o@V=fR9unW2d-N`r!B{@OJWP_iDR8Z&ocs6#JvEb(%#)ZMm!iiIa09# zuRu7vTdG5zw43g@lE!u$PW=l?%A0B{P^Yw^#|gp?=haKz_HfR7vmj_L2c(XQ#6d8* z6l@I$M7N4o+|Q6LEKZxw`_)dbz~S1HUHySuOySypTRm%;nMF4!+O)cK47unu`qxbJUNw$xVN>2T#{61NK! zWf5wBmDgZOZVbNLS@M1__SB+o<%Kd(=pDS8TCIkWAIewXv^LbRPaR^aL3Lju!9NE#{Cq4_e9y`%O-(J#_S( z0UnMUOM2MX9!nPaPYLe&G@_w+ zjf=F&zo`b3fFkF1IZ#t7m?InFkbKAYUwXU!tB>YtuVF3anMBf8lNEND9ogcmDJjYm zu#pYV-mo8zJ>O*#d{0O9#Z6Sdg?pc0Bly-JJre{B_Pf6V8BYK=s>!!$Te~YRPJFKs z?|#B}4tWwFCg94I*>3?^Rqw|Mt_{Pj>7Dj_0^5othkPT^93O~UI#0uCdVCf~RUI(g z2J{I165P{lqykXP>@c8l&z>Y1%LYCw*NNXc9NhT0;bTp8`I=kY04oMN*z70vg_xHE zqec<;y+Mbm89r0cV?i26m@-wc6|~`hVsYmEz`wy+b?<)<7k<402-{bF0ZmkKWMGBN z-?}dP;Cm}vbiDo)$o}L-_}+VT&~YFb^@_3av;GiLFX`-~Y#DH@eO)NX_dfOcuF)cv zU{a9}bk;v&^Oo+&u-9eVG$|Rlnx)f87=HU=BfwS-%(fjN5zri}h*`NR23vUC-mU6Y zCsnESS0f$6b@7{aBrIp86Ym$WgR6dinYh=M_ugDz#gOtT@;(#Z?7fVsG^De|wTjij zU}d7=ZqGFra$d_LbF^i*v!lgA^)&)e?@zL&VR0_oy6)n@2QymvKUL?80?QEhR>aKWBJJX*jDWo@?mf+3*FTd2S0atH$}UaCrtA$~)i2uYW4BG~SY_ajWM z$?C32?+Lo`+C&dsS2r*9u8Lo-)RgwJLWrrq-gpXsrgr&nvN_%Rt=ttI7KnRJuD)+2a^z4fHY>ggxf$cC$C5*Z@3NQWI_l zQDYZ8y_<1}jfsehQSt=dv^}{%f`;5V$Vgof61wu~#ufR1aF~0zV~)z)-4{Mv|clTxM2l zR=bpUDQlKN6yf+P0StFVV}{DSGm;T1J-y;Z_JevtjPC(IqZ#l5FSGkB7PKGM%1hnx z0b^TGVR`8t^eD2E{bPPvn9Jj1@`6>1UAN~&?9zJuZ7}!-StwMGYm1d}*GzX{yOuq3 zo4V$mD6QfESlgOqj`)>+oob2u>cKm&o2zbE=p8q=GPl<%6Y+QZ6f+q2g7p|)X(N!F zBQ<p&%Bdr24GZ(g=Ds8RM0MV<{4x-xdz&>hUKzDKex#0Ah5li+XgS`taHD|Jo6ok0FJU)avUy2H_sh7+> z)r&4I_zEmgl0)GDhOv{1v>dr+&T1M5qFVC0*j@7-g?jJLXrZd_oxc~Q+o(rpzV_fC z2x98%Jaxf4nwsF#d;&yGgfp!aO|Z^TFH_y-K-#u6;;;{saa4{-)wmGD^ZTXk(aD~a z;g7tKmfBjyLd`@TgpYsk$$+o%fmh@yAoHTA(jLS+1yg&9t`;SSJqB%?b+qZ;VwM0M z$x(-N<|#|I+E-j;>`KuPFw_!@PV&KZZ?kj(6|>2a@z_G5qT+%g(3SvjVRJ~+qmx)@ zH@mvK`YAFyblHI`AW}kO0%PJlvcfYDjsAkP=VU*Gva#^TLtOafT!Xgy--eOl5Swv0B)r6=)U<_S z%5fSH!E#DA=R&WrUa{sK@>*LNtVv9vLNHaO1O?yX8aye_GJ6QBDC?nS+YJZgE&3Ii z8kCz3Wnt?YEzFZ?a2#`%%6I+a8%EL&3MTR*`XccKvfFS+dK9!)C?@abl4~kuQX8$% z-je-jiJ($NwLH{Lw8UE^;g5D3NMz6;@g*-a880 z5VD~;H>)jw7z!0!v|XG!1d8S6#ae=0%&DqWhO4E!;`L&RkJ#0v$>q;Bt1MKQOLoRe zvOf-2`+r89Ek9dgFzqy6B7c2kE}2n0k?oB4t55ffITo>Z*jd9d)U_%#Bi6wnQaI^2 zPkA3T2pKa?;d*aDuVwtGmT_9&~2d#yqLhoJhy3OX_3TIqq9$X;D+uO%S zY`h+?Y;2&WZ%b~mAuo>6c>aEO@I|Yyhj#XxUSj$Ao$}YOngaa8Oht=8(Dj;&sAFr8 z4ZR?#BkGo_YxpB4HSs&*OGN*W#AFlV<7W7~DSz!oxEX6T<_lAZmuE1uTvDB^*cG`< zTEJIrM1XqfBu4<_Z7Qc}Y>JynV2XTum! z+%t;Rvq&my{p#k*?lwc<)|^>!}mX06e$$6JII7_0&$E z_xlfEs#U+(=M5^-&%he*5)sWI#Ybeh@vFG7tzw`}S*SdGDC z4<@$c;Iggdj5Vb|9%)fPiS|=qGM)19O-K2in4QaSPSnXe--fvy!5=(LZO|K=Asv)` z0nBmdac6UFejObjet949oM>rRAATPn9eH{Ep6N0FBimbE2hppl7D(M+W7Yquu(Jw^ z1Bli%Hn_V78-fKLGz52dcXx;24nYD0cMF3%4DRkWK!RIv5AIp+-P&7qcW>1`bXV7T z>8gJEPuG9GpS(J!TCD23);GzvpL(hZhn&h>;2{Yz`t9FG;!6HD<9(0S-idNv}u^QjMw%?Hv7{gX&|b zvPpD`YRr$73wi=lPt6|j3shayuhUjPryR0bvrl-~q@74e*c)st{)}EYiaRn(M@w~m zwtFmdEYc{dNigDA;oI-xcr<7I-X^>si^?~cVCp94uAmYb&$ucE2PyQkFd3aYpr^65 zcL<}QVdbIu$ z{i0*;BCk>1%?>;J+(?fn#cJr^zZY3WK`Nh3nm}pSfz?vT+2P@=ncdgff}_;e9WyNi zVtBTM;+TNL*Fm_nvy8Pv!O2U$-xdH zaSfcOBK}>4I`Gjyz##}x9Axpp3uSwmlbGwcIq-#&s8Kl+yo9(|M00$z-Ee=!aoXLQ zLD8I!n#QbZteVbJGD#(tyt)<)<{eSo9dq2~MUY0Vt*j@4KNIeK<*@(izne zxmKYfl9XL@4tb%B!&L^T*j^iw}3Woogybw=d+_`FGY2YdqwP{_bRrnI!5x@?8~3oeF~7oWFr zpWgn+e^AhG^y#hzmb4W@*Qmnog8dIA9KY!&7zrF7(S=QMkQMPF1f?7&o_zyP)ng=X zySm~ere(ZGBHi8~ES}zp87l5_OEf&jMvM3(HGM~Om63H(l$Psk%vvXLUi)>@ zg?%{g5%8?2{iiVSi4W~#s-b9g%=lrXB7U(3Ili*0T7?J75A+p5%fxg>us4}M3|c>L zNA>#dJ@pS$_9ojgT_TV5qtf8~E$fGxZFI*SuX>m2N$+kLRe3h%h0HDW!CLd}!tzri z>I(WZ{GppGix?VLpmjYnS^|^&#hbttd)K7MBDq0-zd=|a@+J>YN0x40z6Rhc&RxIt zn3kRey?r=Gnw-v$QjR&uptH%gsN`~^fBTFp@L8~kMY9espo33V*!com+c#xr20G4Z zLQT3h=da?R>`3F)9p)EsnK^7)ICh|)0F`;P& ziiQcA?P=u>hGfdhqAWwzGRBZ^$aU`KQweyPBWTmkp$xOYVul7)4Ib`NkX{vvQjD_N zx6|bkJ!G3E?}+v?`RQl%#qsffEDSCpV?uIlbA7m=)})AH*{nSe_ik|!E6I>%x!%Ap zX9xH*b6KJ(C+SIrvb|CLAKj&Va9Je7{^(K-c(y-=!qcOqCz5h`QX#x}8?bw~ z!ntq+p&_<#ux}RVf)E5{wPK#RCUrICk_Cj}yr`%)(O0gcEu!l#IM|O?1Ln4*2+1KU z0LC6C{fhF?{G%FY4z&%R%Q})^UXIHp9eqPhBfIgFacQ+v*^R;EG0cYCPRZTD?1w)s z(ePd)P7j7GBi)>g*x92bPUn4v8+uqPB;MNZre+VVzr|a=Y-BwbpMM1$dY{dD8Yeup zoCwu#wckVIeESe57@MFNu@Obh+Kp(}$&u%Vzv~orQKwUDiG5gFQ)05gii|wsHr;JHX~M zNj4gP-KX{rzvPzt1#S2!SzG?f*R=p+kJHxQ-#WZ#?T;0izqk=myKN{}PHEsC znW*s+s`KxAzo=l!4&L&j>_Rjy)Es>~>O0Ul8KsjTSuI}=saR!RBhbJFzA@=bx?$Qc zAtY|LY!S>K(VZB{TDiI?mMZE-Zeke?+GBa6&4bO#JtBnb)Y+_wO`{0j^#4%YlR5+9 z75GJmX_FuLYuKAjaD>zX8XylTs#IZqg$<}2z<$!0Ejq1nS0 zv7o87`L zsVS;c*=&@Ke2SQ9eo_T9wIr#TT!RhS5igu_szUK4t26d8dk|w(4bkdE-3m>kOJ!<$ zLYuTaBv@bvzsh2|yR5QJMBG0|XD5U)*(VjYl#SvlHyQHq7 zu)e#y<3?SC4gOFm4agKoM{nzcE$2$y@bn2+wU6H z@+Fo8YCv7g2@Z^xbWVFSeKUPHrWVxu&?gy9mri>0pQ;+w4JC7G`k%tGZUotjN`9^6 ziJv7-H#oAG`opE-K(cNVFz>&e)zz?Vao+URU#SzR#_Q;98D%e_JQvO_tw^5GNVY49 z9#mkUkVHd1N0kv(sGA1_IEt_LWlc26w7Yw%oDzzBaPkjb?#XX^&3u7LA^RRu#-7Kh zm9f|aku5K;`P}g|D)`8{RDOlFzE7}?a2?{3U~Oj4)LLm4M&a1 zBiwXTNB&Sr(l)8&>2ig*-CoBhUACfPE!$9=Gie6D@>4Ft{)-pA>8SVBa92WI(+i7- z?%fB`I_e-c+6ah;xLgBMh}wtQPDkZShDq{{`>mTf!*jRtgR9`nCXf+g7VYP-cqQQ&p? zB?&zYI>N#SGx~92O>D)Jl0D{kBEJKAD^A7yQ4&P9Fa|7XuY0TU19uW1g*qWt$5>Ly;?T&k z3C3W5k$|CczuhoXmMSG>SH)sS%hqVmPYSGf%`(=0SS?z&Fap1@A$m%Jx-N3RI#7QP zK{q$)#=MP#5U^lQ3^*IT9M&_M@tb9PerAwo5*SH{28 zUk!HPk_&UQOQZMb=Ec;q^1z})XctH1;Ejc0MSR-GnB+03W^RoH?Mo?&r@zkE zHCu;(ZCFaSvh7?VtGh4Jqv)p_&_NZiDE;voIL^c`0?nU0KzK=1K%i`T_-KCAr^{EQN8qkuEhNNY_9zQ-ph|sGIO0B3r>dx?})RnAJBI zQydGkue&EqHWv~4O9S7ICsxXvj%l^VY%vf{ZlWnO8T8)=R~&}52h%!F_>u|9uF};n zWnRwHg-zQOA?=?^%bB(kB@_wy_j9mUnL~AL7=l%Fg8uFNxYl3n1V}_E?|6)mv zGZg*WpxNzr8a(7fSgV4eou0p0ln^>TwqcSDrfqZ&{iT%K@$~h?bN{a~Fp0P^+Mz)T zerQZ*QV!>M1a4eQ&TjoyVp_9kFZa}7&6s+2_zBdzx%%F_u?ky`Q!bggshDIh_*hYM z>I@|fyA4ZX*sn?gI?CjTcE*?hS-hF6S|`;zkWtq{44i>cu#>a$qCa~}x<&_}3_s-P z=)mZAnc|!tu04A23Q+VxDp}^RF}qC*Kid!yEahK*fntxixfa&5w|p;|hB{=4sp~h` zZ^0T6yu~iHEtO!b%Ve=`I2IJ51xZ^~F2`mPlyAjHMMi8v@Z(tNg=i#2vvcaa1pdHg zg3S}ot7k)yXdX|K#6MGx3v3(Q){`-O4QVxDb6;~SxRC%T{wad&yxIs^^pv_ zGG_bRd{RAZMd%d;C6GfI9AI@z+Sw!vuZFXDT6N>-G-GS%7|4pG+<75|@ayh8iG7oU zp_Z)P6i@gjkS(*}atAEN5WI8MASJPy52=j`;Q;Jf;fvznr)b63Qm>H0%v`qho1aAjN1%`RDELb10Re*sstf7z5f6t;!;=;G=7RmsiI^oTwNAXZ z;)*-N5)t$~9I8l1Ka$KDG)TC8m_+Kw!!u?}GK3!C`P_EsC=)J`UeHT^HdaA|uw*qK z6&gm2;*_e@ztMF-gI{>I^a7bOMzMa#(pj0voo6X%+fW7&$=q&fWJx>PP;%E(0#<1F zAb#1D+2GwAI%(CAbNYF)QM%!pTyVz zz~uZWRl)bLqNtd0xVMbzmFo4Cit5+KfWXfC#kSH7{`zUvGEPwDt3u3}x z2W0-?Tf@KOyfD}&R-&;63U~FBl*^XG^lnTj9I#=vPUaBAzYJ$GlfTBrYT3U8oTWc9 z!0@TvFySdxyX-QlcW7l$3@9Urv^%#MMdS_|2X^1cu?;zMRYe|-2WHF}P^N!K9N%OA zH&Ffh{Lo=^n6j7t&-nch=e*I)t9wp=Vv!4$gXwMDkr6)lmJ|lOTbna(3d7I=dwBTB zOyolzJdF!OmtQ~saJ1=Gwzv!ns{%tI1M5?O4hqI(M=0rqTY{0^LhK9{MuyFXAsY;q z=tUwiVO|1UiBdDbB8d`|0WbHW}<3uem*^%@)HJkSbSYPu4rGyc+LXCqDST!?;6UF%&e8W4^ljy6vGC}yHvB4(m#;%|IG$y77>Wr?wp z6!mGfa}_Jezd!AM-u=Y+xhh#HITIZpJsmqYh(6dkh&vQBm^I`SrKn!7!CgU{Dn3<2 zS!7hGG>e_ZIh8kDa#q4UTLYy(B5oI0Po;`#4i;_iv>NrN~zxVN(*S(6zR8Ijq) zGUDqGS)8?SPTEOUCZ`>~E&DVk?9=a;?^{p)tDCKlYO!g~Fe>ei@2D|d4=+x#47bcJ z?v1e}aPnM`GpV+rFg0|3U6WdlG@rNOvaGZvv-TUmYs&4W*sNK1>f2Zg%zgzT4fZV; z_L|Nn9usfr>^)4%jxny357|yHFY%8&CZVkity}g?WK`xbuQrW&PERd3X6xsF9$TO3 z8h>s3mWH%hw2F7C$ ztIhk3S+5(ggO+77|9bk z?d-DV;xb^7;i?n3bSQNjmr3UAkhGCfchqk+z8^JZJutT|x;?d>PuodzOuJ8ONNZ1X zuyl7SeyUAQrbJ>LX+QR9Pf1c&Dpg8RT3_lzR9!;>X=dm@;6mV`zKAjAYvQUQL$kzkj5)no+^>($Ri4R91 zpg70{ba=iRS~)Llfnb$0g+H-tiD_YLc{Y_b?lmry7dCEsaO0%8)^30Dz_aS;xmtLZ zxa#+n;bL@^qix>jV9YSw@Nb9H!SO-be)lQ`q~o(PtF!gS;MeKRrrp@x>+6v_ozwDH z%G-d``!K)X=hbV^O~L9ejVe8LotN%79f$i^h{75#br3eYO_O@rzlpdCD76+IJ-2Vk z#b(9J#dt`9SRN@H6R6`G6l?yMQf;a}XVm?L5b;pdgqRw+)bXV88BGpN+C{HLNA3o0 zKYKziq_(!(CY zMfeu@Vyv5fzNExzd4Gqn6fYXJmVqi5vjm|S53?6&yH83bz`L*Z9(Gz(HAjStiq4A z)<$OHb`yOXf z+|)c2DF+j~d+pNgN)uW+wEVI;6jeo#P=QdHWc09OJ2+%REqyW;{6A_`Tt;f z|C9F2rlV$Q^rlMxy|@^OxbzN_(33ZQxK=Q1s}U5 z$czKTWx>T~&SPmRWcc62sJR&o9K94`Aokzr5R_4Ky0C|tq}$Z!kTx6}m_HzFR{|NG z3^ Date: Sun, 4 Oct 2026 10:37:24 +0200 Subject: [PATCH 15/21] fix(documents): log why a document got no thumbnail Refs #1124. The thumbnail job opened no wide event, so every annotation it made went nowhere: one instance ran it hundreds of times, left sixteen PDFs without a preview, and the log never said so. Each run is now a `job.document_thumbnail` background event. A run that should have produced a preview and did not is raised to warn and names the reason; a type that never has a preview stays at info. --- .../__tests__/native-canvas-support.test.ts | 2 +- src/lib/documents/__tests__/thumbnail.test.ts | 9 +- src/lib/documents/thumbnail.ts | 34 +++++-- .../jobs/__tests__/document-thumbnail.test.ts | 98 +++++++++++++++++++ src/lib/jobs/document-thumbnail.ts | 26 ++++- 5 files changed, 154 insertions(+), 15 deletions(-) create mode 100644 src/lib/jobs/__tests__/document-thumbnail.test.ts diff --git a/src/lib/documents/__tests__/native-canvas-support.test.ts b/src/lib/documents/__tests__/native-canvas-support.test.ts index df978c965..df282df1d 100644 --- a/src/lib/documents/__tests__/native-canvas-support.test.ts +++ b/src/lib/documents/__tests__/native-canvas-support.test.ts @@ -115,6 +115,6 @@ describe("generateThumbnail behind a closed gate", () => { Buffer.from([0xff, 0xd8, 0xff]), "image/jpeg", ); - expect(result).toEqual({ ok: false }); + expect(result).toEqual({ ok: false, reason: "native-canvas-unsupported" }); }); }); diff --git a/src/lib/documents/__tests__/thumbnail.test.ts b/src/lib/documents/__tests__/thumbnail.test.ts index 59a03ab64..05a2630d6 100644 --- a/src/lib/documents/__tests__/thumbnail.test.ts +++ b/src/lib/documents/__tests__/thumbnail.test.ts @@ -126,19 +126,19 @@ describe("generateThumbnail", () => { it("returns { ok: false } for an unsupported MIME type", async () => { await expect( generateThumbnail(Buffer.from("plain text"), "text/plain"), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: false, reason: "unsupported-type" }); }); it("returns { ok: false } and never throws on malformed image bytes", async () => { await expect( generateThumbnail(Buffer.from("not a real image"), "image/png"), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: false, reason: "error" }); }); it("returns { ok: false } on a malformed PDF", async () => { await expect( generateThumbnail(Buffer.from("not a pdf"), "application/pdf"), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: false, reason: "raster-failed" }); }); it("refuses a decompression bomb before decoding (pixel cap)", async () => { @@ -153,6 +153,7 @@ describe("generateThumbnail", () => { bomb.writeUInt32BE(30_000, 20); // height await expect(generateThumbnail(bomb, "image/png")).resolves.toEqual({ ok: false, + reason: "pixel-cap", }); }); @@ -164,6 +165,6 @@ describe("generateThumbnail", () => { loadImageMock.mockResolvedValueOnce({ width: 30_000, height: 30_000 }); await expect( generateThumbnail(Buffer.from("desynced jpeg bytes"), "image/jpeg"), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: false, reason: "pixel-cap" }); }); }); diff --git a/src/lib/documents/thumbnail.ts b/src/lib/documents/thumbnail.ts index 8d975954f..26ed69c1b 100644 --- a/src/lib/documents/thumbnail.ts +++ b/src/lib/documents/thumbnail.ts @@ -157,9 +157,23 @@ export interface Thumbnail { height: number; } +/** + * Why no preview was made. `unsupported-type` is the expected answer for an + * Office file or a text document; every other reason means a preview should + * have existed, so the thumbnail job logs it as a warning. + */ +export type ThumbnailSkipReason = + | "unsupported-type" + | "native-canvas-unsupported" + | "pixel-cap" + | "raster-failed" + | "empty-render" + | "error"; + /** Best-effort generation outcome — never an exception. */ export type ThumbnailResult = - { ok: true; thumbnail: Thumbnail } | { ok: false }; + | { ok: true; thumbnail: Thumbnail } + | { ok: false; reason: ThumbnailSkipReason }; // Minimal structural types for the slice of `@napi-rs/canvas` this module uses. // A type-only shape (erased at build) so we never eagerly evaluate the module @@ -229,7 +243,7 @@ export async function generateThumbnail( action: { name: "documents.thumbnail.rejected" }, meta: { reason: "native_canvas_unsupported" }, }); - return { ok: false }; + return { ok: false, reason: "native-canvas-unsupported" }; } // Lazy import: keeps @napi-rs/canvas (+ transitively pdfjs on the PDF path) // out of the eager server chunk. Resolved on first generation, then cached. @@ -244,7 +258,7 @@ export async function generateThumbnail( action: { name: "documents.thumbnail.rejected" }, meta: { source: "image", reason: "pixel_cap", pixels }, }); - return { ok: false }; + return { ok: false, reason: "pixel-cap" }; } const image = await napi.loadImage(bytes); // Post-decode backstop: the header sniff returns null on any marker @@ -260,10 +274,10 @@ export async function generateThumbnail( pixels: image.width * image.height, }, }); - return { ok: false }; + return { ok: false, reason: "pixel-cap" }; } const thumb = drawScaledJpeg(napi, image); - if (!thumb) return { ok: false }; + if (!thumb) return { ok: false, reason: "empty-render" }; annotate({ action: { name: "documents.thumbnail.ok" }, meta: { source: "image", width: thumb.width, height: thumb.height }, @@ -274,11 +288,13 @@ export async function generateThumbnail( if (mimeType === "application/pdf") { // Page 1 only — a preview never renders the whole document. const raster = await rasterizePdf(bytes, 1); - if (!raster.ok || raster.images.length === 0) return { ok: false }; + if (!raster.ok || raster.images.length === 0) { + return { ok: false, reason: "raster-failed" }; + } const pageJpeg = Buffer.from(raster.images[0]!.dataBase64, "base64"); const image = await napi.loadImage(pageJpeg); const thumb = drawScaledJpeg(napi, image); - if (!thumb) return { ok: false }; + if (!thumb) return { ok: false, reason: "empty-render" }; annotate({ action: { name: "documents.thumbnail.ok" }, meta: { source: "pdf", width: thumb.width, height: thumb.height }, @@ -287,7 +303,7 @@ export async function generateThumbnail( } // Every other MIME (Office/text/TIFF/HEIC/XML/JSON) — no preview. - return { ok: false }; + return { ok: false, reason: "unsupported-type" }; } catch (err) { annotate({ action: { name: "documents.thumbnail.failed" }, @@ -297,6 +313,6 @@ export async function generateThumbnail( message: err instanceof Error ? err.message.slice(0, 300) : String(err), }, }); - return { ok: false }; + return { ok: false, reason: "error" }; } } diff --git a/src/lib/jobs/__tests__/document-thumbnail.test.ts b/src/lib/jobs/__tests__/document-thumbnail.test.ts new file mode 100644 index 000000000..4270507d1 --- /dev/null +++ b/src/lib/jobs/__tests__/document-thumbnail.test.ts @@ -0,0 +1,98 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * GitHub #1124 — the thumbnail job ran 355 times on one instance, left + * sixteen PDFs without a preview, and the log held not one line about it: + * the job opened no wide event, so every annotation went nowhere. Pins: each + * run is a `job.document_thumbnail` background event; a run that should have + * made a preview and did not is raised to `warn` with the reason; a type that + * never has a preview stays at info. + */ + +const { evt, findFirst, upsert, generateThumbnail, withBackgroundEvent } = + vi.hoisted(() => { + const evt = { addMeta: vi.fn(), elevateLevel: vi.fn() }; + return { + evt, + findFirst: vi.fn(), + upsert: vi.fn(), + generateThumbnail: vi.fn(), + withBackgroundEvent: vi.fn( + async (_name: string, fn: (e: typeof evt) => Promise) => + fn(evt), + ), + }; + }); + +vi.mock("@/lib/db", () => ({ + prisma: { + inboundDocument: { findFirst }, + documentThumbnail: { upsert }, + }, +})); +vi.mock("@/lib/documents/store", () => ({ + decryptDocumentContent: vi.fn(() => Buffer.from("%PDF-1.7")), + encryptThumbnail: vi.fn(() => new Uint8Array([1])), +})); +vi.mock("@/lib/documents/thumbnail", () => ({ generateThumbnail })); +vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() })); +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/background", () => ({ withBackgroundEvent })); + +import { runDocumentThumbnail } from "../document-thumbnail"; + +const DOC = { + id: "doc-1", + mimeType: "application/pdf", + contentEncrypted: new Uint8Array([1]), + contentCodec: "binary2", + thumbnail: null, +}; + +beforeEach(() => { + vi.clearAllMocks(); + findFirst.mockResolvedValue(DOC); +}); + +describe("runDocumentThumbnail wide event", () => { + it("runs inside a background event named for the job", async () => { + generateThumbnail.mockResolvedValue({ ok: false, reason: "raster-failed" }); + await runDocumentThumbnail({ userId: "user-1", documentId: "doc-1" }); + expect(withBackgroundEvent).toHaveBeenCalledWith( + "job.document_thumbnail", + expect.any(Function), + ); + expect(evt.addMeta).toHaveBeenCalledWith("document_id", "doc-1"); + }); + + it("warns with the reason when a PDF gets no preview", async () => { + generateThumbnail.mockResolvedValue({ ok: false, reason: "error" }); + await runDocumentThumbnail({ userId: "user-1", documentId: "doc-1" }); + expect(evt.elevateLevel).toHaveBeenCalledWith("warn"); + expect(evt.addMeta).toHaveBeenCalledWith( + "thumbnail_missing_reason", + "error", + ); + expect(upsert).not.toHaveBeenCalled(); + }); + + it("stays at info for a type that never has a preview", async () => { + findFirst.mockResolvedValue({ ...DOC, mimeType: "text/plain" }); + generateThumbnail.mockResolvedValue({ + ok: false, + reason: "unsupported-type", + }); + await runDocumentThumbnail({ userId: "user-1", documentId: "doc-1" }); + expect(evt.elevateLevel).not.toHaveBeenCalled(); + }); + + it("stores the preview and stays at info when one is made", async () => { + generateThumbnail.mockResolvedValue({ + ok: true, + thumbnail: { jpeg: Buffer.from([0xff, 0xd8]), width: 2, height: 3 }, + }); + await runDocumentThumbnail({ userId: "user-1", documentId: "doc-1" }); + expect(upsert).toHaveBeenCalledOnce(); + expect(evt.elevateLevel).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/jobs/document-thumbnail.ts b/src/lib/jobs/document-thumbnail.ts index 91a8d44ed..1b0a1a91d 100644 --- a/src/lib/jobs/document-thumbnail.ts +++ b/src/lib/jobs/document-thumbnail.ts @@ -15,6 +15,12 @@ * kind icon), and `runDocumentThumbnail` returns cleanly so pg-boss does not * retry it. Only an unexpected throw (a DB hiccup) propagates for retry. * + * Each run is one background wide event (`job.document_thumbnail`). A run + * that should have produced a preview and did not is raised to `warn` with + * the reason, so a document left with its kind icon is explained in the log. + * Without the event every annotation below went nowhere: a whole class of + * PDFs went without previews for weeks and the log never said so. + * * The queue MUST be registered in the maintenance registrar * (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it. */ @@ -25,6 +31,7 @@ import { } from "@/lib/documents/store"; import { generateThumbnail } from "@/lib/documents/thumbnail"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { withBackgroundEvent } from "@/lib/logging/background"; import { annotate } from "@/lib/logging/context"; export const DOCUMENT_THUMBNAIL_QUEUE = "document-thumbnail"; @@ -49,7 +56,20 @@ export async function runDocumentThumbnail( ): Promise { const { userId, documentId } = payload; if (!userId || !documentId) return; + await withBackgroundEvent("job.document_thumbnail", async (evt) => { + evt.addMeta("document_id", documentId); + await renderThumbnail(userId, documentId, (reason) => { + evt.addMeta("thumbnail_missing_reason", reason); + evt.elevateLevel("warn"); + }); + }); +} +async function renderThumbnail( + userId: string, + documentId: string, + warn: (reason: string) => void, +): Promise { // Owner-scoped load; the blob column is selected ONLY here (the job that // renders the preview), never in the list/detail queries. Skip a document // that already has a thumbnail so a re-enqueue is a cheap no-op. @@ -84,6 +104,7 @@ export async function runDocumentThumbnail( action: { name: "documents.thumbnail.decryptFailed" }, meta: { documentId }, }); + warn("decrypt-failed"); return; } @@ -91,8 +112,11 @@ export async function runDocumentThumbnail( if (!result.ok) { annotate({ action: { name: "documents.thumbnail.run" }, - meta: { documentId, generated: false }, + meta: { documentId, generated: false, mimeType: document.mimeType }, }); + // An Office or text document has no preview by design; anything else + // was meant to get one. + if (result.reason !== "unsupported-type") warn(result.reason); return; } From befac59576457653181d72d0d1ef81ab2f435188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:42:59 +0200 Subject: [PATCH 16/21] feat(ai): send a per-entry reasoning effort to Local and gateway providers Reasoning models on Ollama and similar servers think by default and can spend the whole max_tokens budget before the visible answer starts, so the call comes back empty or times out. The Local and OpenAI-compatible chain entries now carry an optional reasoning setting: Default sends nothing, Off sends reasoning_effort "none", Low / Medium / High send their own value. It is stored on the entry in aiProviderChain, so a chain can mix a thinking model with one that does not know the field. - Every exported resolver binds the entry's value onto the client, the connection test included, keyed on the instance type so the operator provider, OpenAI, Anthropic and Codex never carry it. - PATCH /api/user/ai-provider accepts localReasoningEffort and compatReasoningEffort and GET reports them. A provider absent from the chain holds its value on a disabled entry; restating Default on a chain that was never customised writes nothing. - PUT /api/insights/provider-chain accepts reasoningEffort on local and openai-compatible entries, refuses it on any other, and keeps the stored value when the body leaves it out. Its schema is now shared with the OpenAPI registry. - A Default request body is byte-identical to the one sent before. Refs #1126 --- docs/api/openapi.yaml | 63 ++- .../ai-reasoning-effort-guard.test.ts | 124 ++++++ .../provider-chain/__tests__/put.test.ts | 91 +++++ src/app/api/insights/provider-chain/route.ts | 68 ++-- .../user/ai-provider/__tests__/route.test.ts | 93 +++++ src/app/api/user/ai-provider/route.ts | 59 ++- .../provider-reasoning-effort.test.ts | 192 +++++++++ src/lib/ai/__tests__/reasoning-effort.test.ts | 366 ++++++++++++++++++ src/lib/ai/local-client.ts | 9 + src/lib/ai/openai-client.ts | 11 + src/lib/ai/provider-chain.ts | 51 +++ src/lib/ai/provider.ts | 48 ++- src/lib/ai/reasoning-effort.ts | 83 ++++ src/lib/ai/types.ts | 8 + .../__tests__/account-settings-backup.test.ts | 33 ++ src/lib/openapi/routes/insights/schemas.ts | 37 +- src/lib/openapi/routes/profile.ts | 7 + src/lib/validations/ai-provider.ts | 91 +++++ 18 files changed, 1350 insertions(+), 84 deletions(-) create mode 100644 src/__tests__/ai-reasoning-effort-guard.test.ts create mode 100644 src/lib/ai/__tests__/provider-reasoning-effort.test.ts create mode 100644 src/lib/ai/__tests__/reasoning-effort.test.ts create mode 100644 src/lib/ai/reasoning-effort.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 25431a0bd..e92efe758 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -25855,6 +25855,28 @@ components: minimum: 10 maximum: 600 - type: "null" + localReasoningEffort: + description: Reasoning setting of the Local provider entry, stored on that entry of the provider chain. `none` = Off, + `null` = Default (nothing sent). Omitted leaves it untouched. + anyOf: + - type: string + enum: + - none + - low + - medium + - high + - type: "null" + compatReasoningEffort: + description: Reasoning setting of the OpenAI-compatible gateway entry, stored on that entry of the provider chain. + `none` = Off, `null` = Default (nothing sent). Omitted leaves it untouched. + anyOf: + - type: string + enum: + - none + - low + - medium + - high + - type: "null" description: "Partial update of the calling user's AI-provider configuration. An omitted key is left untouched; `null` (or an empty string, on the string fields) clears the column. At least one recognised key must be present, or the write is refused. Key material is write-only: it is encrypted on the way in and never returned. Unknown keys @@ -27071,6 +27093,7 @@ components: properties: chain: minItems: 1 + maxItems: 7 type: array items: type: object @@ -27096,15 +27119,30 @@ components: maximum: 9007199254740991 enabled: type: boolean + reasoningEffort: + description: Only on a `local` or `openai-compatible` entry; refused on any other. Omitted keeps the value already + stored for that provider, `null` restores Default, `none` / `low` / `medium` / `high` are sent as + `reasoning_effort`. + anyOf: + - type: string + enum: + - none + - low + - medium + - high + - type: "null" required: - providerType - enabled + additionalProperties: false description: The whole chain, in the order it should be walked. At least one entry and at most one per known provider type; a repeated type is refused. required: - chain + additionalProperties: false description: Replaces the account's provider chain wholesale. No credential is written or read here — only which - provider types exist, in what order, and which are enabled. + provider types exist, in what order, which are enabled, and the reasoning setting of the Local and + OpenAI-compatible entries. InsightsFeedbackRequest: type: object properties: @@ -48543,6 +48581,27 @@ components: - type: "null" description: v1.22 (#89) — per-user response timeout for AI generation, in seconds (10–600). Null = the built-in comprehensive-briefing default (~120 s). Mainly for slow local/self-hosted backends. + localReasoningEffort: + anyOf: + - type: string + enum: + - none + - low + - medium + - high + - type: "null" + description: "#1126 — reasoning setting of the Local provider entry, sent as `reasoning_effort` (`none` = Off). Null = + Default: the key is not sent." + compatReasoningEffort: + anyOf: + - type: string + enum: + - none + - low + - medium + - high + - type: "null" + description: "#1126 — the same for the OpenAI-compatible gateway entry." serverProviderHealth: type: string enum: @@ -48587,6 +48646,8 @@ components: - compatModel - hasCompatKey - responseTimeoutSeconds + - localReasoningEffort + - compatReasoningEffort - serverProviderHealth - serverProviderOffer - serverProviderConsent diff --git a/src/__tests__/ai-reasoning-effort-guard.test.ts b/src/__tests__/ai-reasoning-effort-guard.test.ts new file mode 100644 index 000000000..e555ab141 --- /dev/null +++ b/src/__tests__/ai-reasoning-effort-guard.test.ts @@ -0,0 +1,124 @@ +/** + * Structural guard (#1126): `reasoning_effort` reaches a model only through + * the setting the resolver bound to the provider instance. + * + * Default must send nothing, so the key may not appear on any wire except as + * the bound value, and only the two clients whose endpoint a person chose may + * carry it. Pinned here: + * + * 1. the key appears in exactly the Local client and the OpenAI client, + * once each, and only as `reasoning_effort: this.reasoningEffort`; + * 2. nothing but the binder assigns `.reasoningEffort`; + * 3. every exported resolver in `provider.ts` calls `bindReasoningEffort`. + * + * That clients are constructed only in `provider.ts` (so a resolver always + * binds them) is frozen by `ai-call-timeout-guard.test.ts`. + * + * Its limit: it reads source text. A body built under a computed key, or a + * resolver that binds through an alias, would slip it. + * `reasoning-effort.test.ts` and `provider-reasoning-effort.test.ts` prove + * the behaviour on the wire. + * + * Mutation check: hard-code `reasoning_effort: "none"` into either client, + * set `client.reasoningEffort = ...` in a route, or drop `bindReasoningEffort` + * from `resolveProviderForTest`: each one goes red here. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +function stripComments(source: string): string { + return source + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/(^|[^:])\/\/.*$/gm, "$1"); +} + +function code(rel: string): string { + return stripComments(readFileSync(join(SRC, rel), "utf8")); +} + +const files = walkSourceFiles(SRC, { floor: 3000 }) + .filter((p) => !p.startsWith("generated/")) + .filter((p) => !p.includes("__tests__")) + .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx")); + +describe("reasoning_effort on the wire", () => { + // As an object key: the published descriptions name the field in prose. + const KEY = /\breasoning_effort["']?\s*:/g; + const sites = files.filter((rel) => new RegExp(KEY).test(code(rel))); + + it("appears in the Local and the OpenAI client only", () => { + expect(sites.sort()).toEqual([ + "lib/ai/local-client.ts", + "lib/ai/openai-client.ts", + ]); + }); + + it.each(["lib/ai/local-client.ts", "lib/ai/openai-client.ts"])( + "%s sends it once, as the bound value", + (rel) => { + const source = code(rel); + expect(source.match(new RegExp(KEY))?.length).toBe(1); + expect(source).toMatch( + /\breasoning_effort\s*:\s*this\s*\.\s*reasoningEffort\b/, + ); + }, + ); +}); + +/** Where `.reasoningEffort` may be assigned, and why. */ +const ASSIGNMENT_SITES: Record = { + "lib/ai/reasoning-effort.ts": + "The binder, the one place a provider instance gets its value.", + "lib/ai/provider-chain.ts": + "Writes the stored chain entry, not a provider instance.", +}; + +describe("only the binder assigns it to a provider", () => { + it("names every assignment site, with a reason", () => { + const sites = files.filter((rel) => + /\.\s*reasoningEffort\s*=(?!=)/.test(code(rel)), + ); + expect(sites.sort()).toEqual(Object.keys(ASSIGNMENT_SITES).sort()); + }); +}); + +describe("every exported resolver binds it", () => { + const source = code("lib/ai/provider.ts"); + const resolvers = source + .split(/\nexport\s+/) + .map((chunk) => { + const m = /^async\s+function\s+(\w+)\s*\(/.exec(chunk); + if (!m) return null; + const header = chunk.slice(0, chunk.indexOf("{\n")); + if ( + !/Promise<\s*(?:AIProvider|ProviderChainResolved\[\])\s*>/.test(header) + ) { + return null; + } + const end = chunk.indexOf("\n}\n"); + return { name: m[1], body: chunk.slice(header.length, end) }; + }) + .filter((r): r is { name: string; body: string } => r !== null); + + it("finds the resolvers (an empty set is a failure, not a pass)", () => { + expect(resolvers.map((r) => r.name).sort()).toEqual([ + "resolveProvider", + "resolveProviderChain", + "resolveProviderForTest", + ]); + }); + + it.each(resolvers.map((r) => r.name))( + "%s calls bindReasoningEffort", + (name) => { + const resolver = resolvers.find((r) => r.name === name); + expect(resolver?.body ?? "").toMatch(/\bbindReasoningEffort\s*\(/); + }, + ); +}); diff --git a/src/app/api/insights/provider-chain/__tests__/put.test.ts b/src/app/api/insights/provider-chain/__tests__/put.test.ts index 510fa5fe7..3413fc0a2 100644 --- a/src/app/api/insights/provider-chain/__tests__/put.test.ts +++ b/src/app/api/insights/provider-chain/__tests__/put.test.ts @@ -53,6 +53,8 @@ vi.mock("@/lib/db", () => ({ prisma: { user: { update: vi.fn(), + // #1126 — the stored chain, read for the reasoning settings it carries. + findUnique: vi.fn(async () => null), }, }, })); @@ -172,4 +174,93 @@ describe("PUT /api/insights/provider-chain", () => { { providerType: "codex", priority: 2, enabled: true }, ]); }); + + // #1126 — the reasoning setting rides on the local / openai-compatible + // entries. The chain editor only reorders and toggles, so saving the order + // must not wipe a value it never sent. + describe("reasoning setting", () => { + function storedChain(chain: unknown) { + vi.mocked(prisma.user.findUnique).mockResolvedValueOnce({ + aiProviderChain: chain, + } as never); + } + async function put(chain: unknown[]) { + vi.mocked(prisma.user.update).mockResolvedValueOnce({} as never); + return (PUT as (req: Request) => Promise)( + jsonRequest({ chain }), + ); + } + function persisted() { + return vi.mocked(prisma.user.update).mock.calls[0][0].data + .aiProviderChain; + } + + it("persists a value sent on a local entry", async () => { + storedChain(null); + const res = await put([ + { providerType: "local", enabled: true, reasoningEffort: "none" }, + { providerType: "openai", enabled: true }, + ]); + expect(res.status).toBe(200); + expect(persisted()).toEqual([ + { + providerType: "local", + priority: 1, + enabled: true, + reasoningEffort: "none", + }, + { providerType: "openai", priority: 2, enabled: true }, + ]); + }); + + it("keeps the stored value when the body leaves it out", async () => { + storedChain([ + { providerType: "openai", priority: 1, enabled: true }, + { + providerType: "openai-compatible", + priority: 2, + enabled: false, + reasoningEffort: "high", + }, + ]); + await put([ + { providerType: "openai-compatible", enabled: true }, + { providerType: "openai", enabled: true }, + ]); + expect(persisted()).toEqual([ + { + providerType: "openai-compatible", + priority: 1, + enabled: true, + reasoningEffort: "high", + }, + { providerType: "openai", priority: 2, enabled: true }, + ]); + }); + + it("clears it on null", async () => { + storedChain([ + { + providerType: "local", + priority: 1, + enabled: true, + reasoningEffort: "low", + }, + ]); + await put([ + { providerType: "local", enabled: true, reasoningEffort: null }, + ]); + expect(persisted()).toEqual([ + { providerType: "local", priority: 1, enabled: true }, + ]); + }); + + it("refuses it on an entry that cannot carry it", async () => { + const res = await put([ + { providerType: "anthropic", enabled: true, reasoningEffort: "none" }, + ]); + expect(res.status).toBe(422); + expect(prisma.user.update).not.toHaveBeenCalled(); + }); + }); }); diff --git a/src/app/api/insights/provider-chain/route.ts b/src/app/api/insights/provider-chain/route.ts index 8ea535990..477cb8159 100644 --- a/src/app/api/insights/provider-chain/route.ts +++ b/src/app/api/insights/provider-chain/route.ts @@ -1,6 +1,4 @@ import type { NextRequest } from "next/server"; -import { z } from "zod/v4"; - import { apiHandler, HttpError, requireAuth } from "@/lib/api-handler"; import { apiSuccess, @@ -15,10 +13,9 @@ import { getLastWorkingProvider, } from "@/lib/ai/provider-runner"; import { prisma } from "@/lib/db"; -import { - parseProviderChain, - PROVIDER_CHAIN_TYPES, -} from "@/lib/ai/provider-chain"; +import { parseProviderChain } from "@/lib/ai/provider-chain"; +import { isReasoningProviderType } from "@/lib/ai/reasoning-effort"; +import { providerChainPutSchema } from "@/lib/validations/ai-provider"; /** * v1.4.16 phase B5b — read-only chain summary for the Settings → AI @@ -85,29 +82,6 @@ export const GET = apiHandler(async () => { }); }); -const chainEntrySchema = z - .object({ - providerType: z.enum( - PROVIDER_CHAIN_TYPES as unknown as [string, ...string[]], - ), - // Priority is recomputed server-side from insertion order so a stale - // client cannot persist a chain whose displayed order disagrees with - // its priority field. The number is accepted (and may be present) but - // ignored on the wire. - priority: z.number().int().optional(), - enabled: z.boolean(), - }) - .strict(); - -const chainBodySchema = z - .object({ - chain: z - .array(chainEntrySchema) - .min(1, "Chain must contain at least one provider") - .max(PROVIDER_CHAIN_TYPES.length, "Too many providers"), - }) - .strict(); - export const PUT = apiHandler(async (request: NextRequest) => { const { user } = await requireAuth(); annotate({ action: { name: "insights.provider_chain.update" } }); @@ -117,7 +91,7 @@ export const PUT = apiHandler(async (request: NextRequest) => { }); if (error) return error; - const parsed = chainBodySchema.safeParse(body); + const parsed = providerChainPutSchema.safeParse(body); if (!parsed.success) { return apiValidationError( "Invalid provider chain payload", @@ -142,11 +116,35 @@ export const PUT = apiHandler(async (request: NextRequest) => { // Normalise priority to insertion order (1-based). UI hands us the // visual order it wants persisted; we make priority match so a later // GET reflects the same order without depending on the client's math. - const normalised = parsed.data.chain.map((entry, idx) => ({ - providerType: entry.providerType, - priority: idx + 1, - enabled: entry.enabled, - })); + // + // #1126 — a reasoning setting the body leaves out is carried over from the + // stored entry of the same type, so the chain editor (which only reorders + // and toggles) and an older client cannot wipe it by saving the order. + // `null` clears it; the schema has already refused it on any type other + // than `local` / `openai-compatible`. + const stored = await prisma.user.findUnique({ + where: { id: user.id }, + select: { aiProviderChain: true }, + }); + const storedEffort = new Map( + parseProviderChain(stored?.aiProviderChain ?? null).map((e) => [ + e.providerType, + e.reasoningEffort ?? null, + ]), + ); + const normalised = parsed.data.chain.map((entry, idx) => { + const reasoningEffort = isReasoningProviderType(entry.providerType) + ? entry.reasoningEffort === undefined + ? (storedEffort.get(entry.providerType) ?? null) + : entry.reasoningEffort + : null; + return { + providerType: entry.providerType, + priority: idx + 1, + enabled: entry.enabled, + ...(reasoningEffort ? { reasoningEffort } : {}), + }; + }); await prisma.user.update({ where: { id: user.id }, diff --git a/src/app/api/user/ai-provider/__tests__/route.test.ts b/src/app/api/user/ai-provider/__tests__/route.test.ts index 9c5e5809b..0eb415494 100644 --- a/src/app/api/user/ai-provider/__tests__/route.test.ts +++ b/src/app/api/user/ai-provider/__tests__/route.test.ts @@ -623,3 +623,96 @@ describe("GET /api/user/ai-provider — the shared-provider offer", () => { } }); }); + +// #1126 — the Local and gateway entries' reasoning setting, written from the +// provider forms onto the chain entry and read back from it. +describe("PATCH /api/user/ai-provider — reasoning setting (#1126)", () => { + const patch = PATCH as (req: Request) => Promise; + function storedChain(chain: unknown) { + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + aiProviderChain: chain, + } as never); + } + function written() { + return vi.mocked(prisma.user.update).mock.calls[0]?.[0].data as + Record | undefined; + } + + it("stores Off on the local entry of a chain that was never customised", async () => { + storedChain(null); + const res = await patch( + patchRequest({ provider: "LOCAL", localReasoningEffort: "none" }), + ); + expect(res.status).toBe(200); + const chain = written()?.aiProviderChain as Array<{ + providerType: string; + reasoningEffort?: string; + }>; + expect(chain.find((e) => e.providerType === "local")).toMatchObject({ + enabled: true, + reasoningEffort: "none", + }); + expect(chain.filter((e) => e.reasoningEffort)).toHaveLength(1); + }); + + it("holds the gateway's value on a disabled entry when the chain lacks it", async () => { + storedChain([{ providerType: "local", priority: 1, enabled: true }]); + await patch(patchRequest({ compatReasoningEffort: "high" })); + expect(written()?.aiProviderChain).toEqual([ + { providerType: "local", priority: 1, enabled: true }, + { + providerType: "openai-compatible", + priority: 2, + enabled: false, + reasoningEffort: "high", + }, + ]); + }); + + it("does not touch the chain when Default is restated on a default chain", async () => { + storedChain(null); + await patch( + patchRequest({ provider: "LOCAL", localReasoningEffort: null }), + ); + expect(written()).toEqual({ aiProvider: "LOCAL" }); + }); + + it("answers 200 for a body that only restates Default", async () => { + storedChain(null); + const res = await patch(patchRequest({ localReasoningEffort: null })); + expect(res.status).toBe(200); + expect(prisma.user.update).not.toHaveBeenCalled(); + }); + + it("refuses a value outside the five", async () => { + const res = await patch(patchRequest({ localReasoningEffort: "max" })); + expect(res.status).toBe(422); + expect(prisma.user.update).not.toHaveBeenCalled(); + }); +}); + +describe("GET /api/user/ai-provider — reasoning setting (#1126)", () => { + it("reads both values from the chain, null for Default", async () => { + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + aiProvider: "LOCAL", + aiProviderChain: [ + { + providerType: "local", + priority: 1, + enabled: true, + reasoningEffort: "none", + }, + ], + } as never); + vi.mocked(resolveProviderAvailability).mockResolvedValue({ + aiAvailable: true, + managedBy: "local", + }); + const res = await (GET as () => Promise)(); + const body = (await res.json()) as { + data: { localReasoningEffort: unknown; compatReasoningEffort: unknown }; + }; + expect(body.data.localReasoningEffort).toBe("none"); + expect(body.data.compatReasoningEffort).toBeNull(); + }); +}); diff --git a/src/app/api/user/ai-provider/route.ts b/src/app/api/user/ai-provider/route.ts index 7f19b619b..1b976d5ec 100644 --- a/src/app/api/user/ai-provider/route.ts +++ b/src/app/api/user/ai-provider/route.ts @@ -20,6 +20,12 @@ import { } from "@/lib/sharing/provider-work-authority"; import { annotate } from "@/lib/logging/context"; import { aiProviderPatchSchema } from "@/lib/validations/ai-provider"; +import { + chainWithReasoningEffort, + parseProviderChain, + serializeProviderChain, +} from "@/lib/ai/provider-chain"; +import { reasoningEffortFor } from "@/lib/ai/reasoning-effort"; export const dynamic = "force-dynamic"; @@ -42,6 +48,8 @@ export const GET = apiHandler(async () => { aiCompatModel: true, // v1.22 (#89) aiResponseTimeoutSeconds: true, + // #1126 — the reasoning settings live on the chain entries. + aiProviderChain: true, // v1.38.19 — a managed profile is never the party that gives // consent for itself; the offer below has to know. managedProfileAt: true, @@ -141,6 +149,13 @@ export const GET = apiHandler(async () => { hasCompatKey: Boolean(u?.aiCompatKeyEncrypted), // v1.22 (#89) — per-user response timeout, in seconds (null = default). responseTimeoutSeconds: u?.aiResponseTimeoutSeconds ?? null, + // #1126 — the Local and gateway entries' reasoning settings (null = + // Default), read from the chain where they are stored. + localReasoningEffort: reasoningEffortFor(u?.aiProviderChain, "local"), + compatReasoningEffort: reasoningEffortFor( + u?.aiProviderChain, + "openai-compatible", + ), // v1.38.19 — see the block above. A tri-state and two booleans; // no count, no timestamp and no other account is inferable from them. serverProviderHealth, @@ -297,7 +312,45 @@ export const PATCH = apiHandler(async (request: NextRequest) => { updates.aiResponseTimeoutSeconds = body.responseTimeoutSeconds; } - if (Object.keys(updates).length === 0) { + // ── #1126 — reasoning settings of the Local and gateway entries ── + // Stored on the entry in `aiProviderChain` rather than in a column of their + // own, so the setting is per entry and a chain can mix a thinking model with + // one that does not know the field. See `chainWithReasoningEffort` for how a + // default chain, or a provider absent from the chain, holds the value. + const reasoningTouched = + body.localReasoningEffort !== undefined || + body.compatReasoningEffort !== undefined; + if (reasoningTouched) { + const current = await prisma.user.findUnique({ + where: { id: user.id }, + select: { aiProviderChain: true }, + }); + let chain: unknown = current?.aiProviderChain ?? null; + if (body.localReasoningEffort !== undefined) { + chain = chainWithReasoningEffort( + chain, + "local", + body.localReasoningEffort, + ); + } + if (body.compatReasoningEffort !== undefined) { + chain = chainWithReasoningEffort( + chain, + "openai-compatible", + body.compatReasoningEffort, + ); + } + // Written only when it changed: restating Default on a chain that was + // never customised must not freeze today's default chain into the row. + const before = serializeProviderChain( + parseProviderChain(current?.aiProviderChain ?? null), + ); + if (serializeProviderChain(parseProviderChain(chain)) !== before) { + updates.aiProviderChain = chain; + } + } + + if (Object.keys(updates).length === 0 && !reasoningTouched) { return apiError("No valid fields", 422, { errorCode: "ai_provider.no_fields", }); @@ -317,7 +370,9 @@ export const PATCH = apiHandler(async (request: NextRequest) => { updates.aiBaseUrl = null; } - await prisma.user.update({ where: { id: user.id }, data: updates }); + if (Object.keys(updates).length > 0) { + await prisma.user.update({ where: { id: user.id }, data: updates }); + } return apiSuccess({ updated: true }); }); diff --git a/src/lib/ai/__tests__/provider-reasoning-effort.test.ts b/src/lib/ai/__tests__/provider-reasoning-effort.test.ts new file mode 100644 index 000000000..0da8b93c1 --- /dev/null +++ b/src/lib/ai/__tests__/provider-reasoning-effort.test.ts @@ -0,0 +1,192 @@ +/** + * #1126 — every exported resolver hands out providers carrying their own + * chain entry's reasoning setting, and the request each one then sends + * carries exactly that value. A chain mixing a thinking model with one that + * does not know the field is the case the setting exists for. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/db", () => ({ + prisma: { + user: { findUnique: vi.fn(), update: vi.fn() }, + appSettings: { findUnique: vi.fn() }, + }, +})); +vi.mock("@/lib/crypto", () => ({ + decrypt: vi.fn((v: string) => `decrypted:${v}`), + encrypt: vi.fn((v: string) => `encrypted:${v}`), +})); +vi.mock("@/lib/logging/context", async (importOriginal) => ({ + ...(await importOriginal()), + annotate: vi.fn(), +})); +vi.mock("@/lib/safe-fetch", async (importOriginal) => ({ + ...(await importOriginal()), + safeFetch: (url: string, init: RequestInit) => fetch(url, init), +})); + +import { prisma } from "@/lib/db"; +import { + resolveProvider, + resolveProviderChain, + resolveProviderForTest, +} from "../provider"; +import { resetLocalJsonDialectCache } from "../local-client"; +import { singleUserTurn, type AIProvider } from "../types"; + +const ROW = { + aiProvider: "LOCAL", + aiModel: "gemma4:12b", + aiBaseUrl: "https://ollama.example.org/v1", + aiAnthropicKeyEncrypted: null, + aiLocalKeyEncrypted: null, + aiOpenaiKeyEncrypted: "openai-key", + aiCompatBaseUrl: "https://gateway.example.org/v1", + aiCompatKeyEncrypted: null, + aiCompatModel: "llama3", + role: "USER", + useCentralCodex: false, + managedProfileAt: null, + aiResponseTimeoutSeconds: null, +}; + +function withChain(aiProviderChain: unknown) { + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + ...ROW, + aiProviderChain, + } as never); + vi.mocked(prisma.appSettings.findUnique).mockResolvedValue(null as never); +} + +const MIXED = [ + { + providerType: "local", + priority: 1, + enabled: true, + reasoningEffort: "none", + }, + { providerType: "openai-compatible", priority: 2, enabled: true }, + { providerType: "openai", priority: 3, enabled: true }, +]; + +/** The `reasoning_effort` the provider actually puts on the wire. */ +async function sentEffort(provider: AIProvider): Promise { + const mock = vi.fn().mockResolvedValue({ + ok: true, + json: () => + Promise.resolve({ + choices: [{ message: { content: "ok" }, finish_reason: "stop" }], + usage: { total_tokens: 1 }, + }), + }); + vi.stubGlobal("fetch", mock); + await provider.generateCompletion(singleUserTurn({ system: "s", user: "u" })); + vi.unstubAllGlobals(); + const body = JSON.parse(mock.mock.calls[0][1].body as string); + return "reasoning_effort" in body ? body.reasoning_effort : "(absent)"; +} + +beforeEach(() => { + vi.resetAllMocks(); + resetLocalJsonDialectCache(); +}); + +describe("resolveProviderChain binds each entry's own setting", () => { + it("a mixed chain sends none, nothing, nothing", async () => { + withChain(MIXED); + const chain = await resolveProviderChain("user-1"); + expect(chain.map((e) => e.providerType)).toEqual([ + "local", + "openai-compatible", + "openai", + ]); + expect(chain.map((e) => e.instance.reasoningEffort)).toEqual([ + "none", + null, + null, + ]); + const sent = []; + for (const entry of chain) sent.push(await sentEffort(entry.instance)); + expect(sent).toEqual(["none", "(absent)", "(absent)"]); + }); + + it("the gateway's own value reaches the gateway", async () => { + withChain([ + { providerType: "local", priority: 1, enabled: true }, + { + providerType: "openai-compatible", + priority: 2, + enabled: true, + reasoningEffort: "high", + }, + ]); + const chain = await resolveProviderChain("user-1"); + expect(await sentEffort(chain[0].instance)).toBe("(absent)"); + expect(await sentEffort(chain[1].instance)).toBe("high"); + }); +}); + +describe("resolveProvider", () => { + it("binds the selected Local provider's setting", async () => { + withChain(MIXED); + const provider = await resolveProvider("user-1"); + expect(provider.type).toBe("local"); + expect(await sentEffort(provider)).toBe("none"); + }); + + it("reaches a selected gateway whose value sits on a disabled entry", async () => { + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + ...ROW, + aiProvider: "OPENAI_COMPATIBLE", + aiProviderChain: [ + { providerType: "codex", priority: 1, enabled: true }, + { + providerType: "openai-compatible", + priority: 2, + enabled: false, + reasoningEffort: "low", + }, + ], + } as never); + const provider = await resolveProvider("user-1"); + expect(provider.type).toBe("openai-compatible"); + expect(await sentEffort(provider)).toBe("low"); + }); +}); + +describe("resolveProviderForTest — the test button runs with the setting", () => { + it("testing the saved config (empty body) uses the saved provider's entry", async () => { + withChain(MIXED); + const provider = await resolveProviderForTest("user-1", {}); + expect(provider.type).toBe("local"); + expect(await sentEffort(provider)).toBe("none"); + }); + + it("with no provider selected, the test runs the chain head with its value", async () => { + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + ...ROW, + aiProvider: null, + aiProviderChain: MIXED, + } as never); + vi.mocked(prisma.appSettings.findUnique).mockResolvedValue(null as never); + const provider = await resolveProviderForTest("user-1", {}); + expect(provider.type).toBe("local"); + expect(await sentEffort(provider)).toBe("none"); + }); + + it("testing a named Local provider uses its entry's value", async () => { + withChain(MIXED); + const provider = await resolveProviderForTest("user-1", { + provider: "LOCAL", + }); + expect(await sentEffort(provider)).toBe("none"); + }); + + it("testing the personal OpenAI key never sends it", async () => { + withChain(MIXED); + const provider = await resolveProviderForTest("user-1", { + provider: "OPENAI", + }); + expect(await sentEffort(provider)).toBe("(absent)"); + }); +}); diff --git a/src/lib/ai/__tests__/reasoning-effort.test.ts b/src/lib/ai/__tests__/reasoning-effort.test.ts new file mode 100644 index 000000000..420c88bbb --- /dev/null +++ b/src/lib/ai/__tests__/reasoning-effort.test.ts @@ -0,0 +1,366 @@ +/** + * #1126 — the per-entry reasoning setting on the wire and in the schemas. + * + * Default must leave the request body byte-identical to what a setup that + * never touched the setting has always sent, so the Default cases compare the + * whole body string against a literal captured before the setting existed. + * Every other choice adds exactly one key, `reasoning_effort`, and only on the + * Local client and the OpenAI-compatible gateway. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("../local-host-allowlist", () => ({ + aiEgressPolicyFor: () => ({}), + isLocalAiHostAllowed: () => false, +})); +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +// The pinned tags dial through undici's own fetch; route every call to the +// stubbed global so the body can be read whatever the egress policy. +vi.mock("@/lib/safe-fetch", async (importOriginal) => ({ + ...(await importOriginal()), + safeFetch: (url: string, init: RequestInit) => fetch(url, init), +})); + +import { bindReasoningEffort, reasoningEffortFor } from "../reasoning-effort"; +import { + LocalOpenAICompatibleClient, + resetLocalJsonDialectCache, +} from "../local-client"; +import { OpenAIClient } from "../openai-client"; +import { AnthropicClient } from "../anthropic-client"; +import { + chainWithReasoningEffort, + parseProviderChain, + PROVIDER_CHAIN_DEFAULT, + serializeProviderChain, +} from "../provider-chain"; +import { singleUserTurn, type AIProvider } from "../types"; +import { + aiProviderPatchSchema, + providerChainEntrySchema, + providerChainPutSchema, +} from "@/lib/validations/ai-provider"; + +const PARAMS = { + ...singleUserTurn({ system: "s", user: "u" }), + maxTokens: 250, +}; + +function okFetch(content = "ok") { + const mock = vi.fn().mockResolvedValue({ + ok: true, + json: () => + Promise.resolve({ + choices: [{ message: { content }, finish_reason: "stop" }], + usage: { total_tokens: 5 }, + }), + }); + vi.stubGlobal("fetch", mock); + return mock; +} + +function sentBody(mock: ReturnType): string { + return mock.mock.calls[0][1].body as string; +} + +function localClient() { + return new LocalOpenAICompatibleClient({ + apiKey: null, + model: "gemma4:12b", + baseUrl: "http://ollama.example.org:11434/v1", + }); +} + +function gatewayClient() { + return new OpenAIClient({ + apiKey: "", + model: "qwen3", + baseUrl: "https://gateway.example.org/v1", + providerType: "openai-compatible", + }); +} + +// Captured from the clients before the setting existed. Default must keep +// sending exactly this. +const LOCAL_BODY_BEFORE = + '{"model":"gemma4:12b","messages":[{"role":"system","content":"s"},{"role":"user","content":"Return strict JSON only, no markdown, no commentary.\\n\\nu"}],"temperature":0.3,"max_tokens":250}'; +const GATEWAY_BODY_BEFORE = + '{"model":"qwen3","messages":[{"role":"system","content":"s"},{"role":"user","content":"u"}],"max_tokens":250,"temperature":0.3}'; + +beforeEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + resetLocalJsonDialectCache(); +}); + +describe("Local client", () => { + it("Default sends the body it always sent, byte for byte", async () => { + const mock = okFetch(); + await localClient().generateCompletion(PARAMS); + expect(sentBody(mock)).toBe(LOCAL_BODY_BEFORE); + expect(JSON.parse(sentBody(mock))).not.toHaveProperty("reasoning_effort"); + }); + + it("Off sends reasoning_effort none and nothing else new", async () => { + const mock = okFetch(); + const client = localClient(); + client.reasoningEffort = "none"; + await client.generateCompletion(PARAMS); + const body = JSON.parse(sentBody(mock)); + expect(body.reasoning_effort).toBe("none"); + const { reasoning_effort: _dropped, ...rest } = body; + expect(JSON.stringify(rest)).toBe(LOCAL_BODY_BEFORE); + }); + + it.each(["low", "medium", "high"] as const)("%s is sent as is", async (v) => { + const mock = okFetch(); + const client = localClient(); + client.reasoningEffort = v; + await client.generateCompletion(PARAMS); + expect(JSON.parse(sentBody(mock)).reasoning_effort).toBe(v); + }); + + it("the streaming request carries it too", async () => { + const mock = okFetch(); + const client = localClient(); + client.reasoningEffort = "none"; + // A buffered answer to a stream request falls back cleanly; only the + // first request body matters here. + await client.generateCompletionStream(PARAMS, () => {}).catch(() => {}); + const body = JSON.parse(sentBody(mock)); + expect(body.stream).toBe(true); + expect(body.reasoning_effort).toBe("none"); + }); +}); + +describe("OpenAI-compatible gateway", () => { + it("Default sends the body it always sent, byte for byte", async () => { + const mock = okFetch(); + await gatewayClient().generateCompletion(PARAMS); + expect(sentBody(mock)).toBe(GATEWAY_BODY_BEFORE); + }); + + it("Off sends reasoning_effort none", async () => { + const mock = okFetch(); + const client = gatewayClient(); + client.reasoningEffort = "none"; + await client.generateCompletion(PARAMS); + expect(JSON.parse(sentBody(mock)).reasoning_effort).toBe("none"); + }); + + it("the OpenAI and Codex tags never send it, whatever is stamped", async () => { + for (const providerType of ["admin-key", "codex"] as const) { + vi.unstubAllGlobals(); + const mock = okFetch(); + const client = new OpenAIClient({ + apiKey: "k", + model: "gpt-4o", + baseUrl: "https://gateway.example.org/v1", + providerType, + }); + client.reasoningEffort = "high"; + await client.generateCompletion(PARAMS); + expect(JSON.parse(sentBody(mock))).not.toHaveProperty("reasoning_effort"); + } + }); +}); + +describe("bindReasoningEffort", () => { + const chain = [ + { + providerType: "local", + priority: 1, + enabled: true, + reasoningEffort: "none", + }, + { + providerType: "openai-compatible", + priority: 2, + enabled: true, + reasoningEffort: "high", + }, + { providerType: "openai", priority: 3, enabled: true }, + ]; + + it("stamps each entry's own value, keyed on the instance type", () => { + expect(bindReasoningEffort(localClient(), chain).reasoningEffort).toBe( + "none", + ); + expect(bindReasoningEffort(gatewayClient(), chain).reasoningEffort).toBe( + "high", + ); + }); + + it("stamps null on every other provider", () => { + const openai = new OpenAIClient({ + apiKey: "k", + model: "gpt-4o", + baseUrl: "https://api.openai.com/v1", + }); + const anthropic: AIProvider = new AnthropicClient({ + apiKey: "k", + model: "m", + }); + expect(bindReasoningEffort(openai, chain).reasoningEffort).toBeNull(); + expect(bindReasoningEffort(anthropic, chain).reasoningEffort).toBeNull(); + }); + + it("stamps null for a chain that was never customised", () => { + expect(bindReasoningEffort(localClient(), null).reasoningEffort).toBeNull(); + }); +}); + +describe("the stored chain", () => { + it("keeps a valid value on local / openai-compatible and drops it elsewhere", () => { + const parsed = parseProviderChain([ + { + providerType: "local", + priority: 1, + enabled: true, + reasoningEffort: "low", + }, + { + providerType: "openai", + priority: 2, + enabled: true, + reasoningEffort: "low", + }, + { + providerType: "openai-compatible", + priority: 3, + enabled: true, + reasoningEffort: "max", + }, + ]); + expect(parsed.map((e) => e.reasoningEffort)).toEqual([ + "low", + undefined, + undefined, + ]); + expect(serializeProviderChain(parsed)).toBe( + '[{"providerType":"local","priority":1,"enabled":true,"reasoningEffort":"low"},{"providerType":"openai","priority":2,"enabled":true},{"providerType":"openai-compatible","priority":3,"enabled":true}]', + ); + }); + + it("serialises a chain without the setting exactly as before", () => { + expect(serializeProviderChain(PROVIDER_CHAIN_DEFAULT)).not.toContain( + "reasoningEffort", + ); + }); + + it("materialises the default chain to hold a value", () => { + const next = chainWithReasoningEffort(null, "local", "none"); + expect(next.map((e) => e.providerType)).toEqual( + PROVIDER_CHAIN_DEFAULT.map((e) => e.providerType), + ); + expect(reasoningEffortFor(next, "local")).toBe("none"); + }); + + it("holds a value for a provider absent from the chain on a disabled entry", () => { + const stored = [{ providerType: "codex", priority: 7, enabled: true }]; + const next = chainWithReasoningEffort(stored, "openai-compatible", "high"); + expect(next).toEqual([ + { providerType: "codex", priority: 7, enabled: true }, + { + providerType: "openai-compatible", + priority: 8, + enabled: false, + reasoningEffort: "high", + }, + ]); + }); + + it("clears without adding an entry", () => { + const stored = [ + { + providerType: "local", + priority: 1, + enabled: true, + reasoningEffort: "low", + }, + ]; + expect(chainWithReasoningEffort(stored, "local", null)).toEqual([ + { providerType: "local", priority: 1, enabled: true }, + ]); + expect(chainWithReasoningEffort(stored, "openai-compatible", null)).toEqual( + [ + { + providerType: "local", + priority: 1, + enabled: true, + reasoningEffort: "low", + }, + ], + ); + }); +}); + +describe("schemas", () => { + const VALUES = [null, "none", "low", "medium", "high"] as const; + + it.each(["local", "openai-compatible"])( + "a %s chain entry accepts Default, Off, Low, Medium and High", + (providerType) => { + for (const reasoningEffort of VALUES) { + expect( + providerChainEntrySchema.safeParse({ + providerType, + enabled: true, + reasoningEffort, + }).success, + ).toBe(true); + } + }, + ); + + it("refuses a value outside the five", () => { + expect( + providerChainEntrySchema.safeParse({ + providerType: "local", + enabled: true, + reasoningEffort: "max", + }).success, + ).toBe(false); + }); + + it.each(["openai", "anthropic", "codex", "admin-openai", "admin-codex"])( + "refuses the setting on a %s entry, null included", + (providerType) => { + for (const reasoningEffort of VALUES) { + const result = providerChainEntrySchema.safeParse({ + providerType, + enabled: true, + reasoningEffort, + }); + expect(result.success).toBe(false); + expect(result.error?.issues[0].path).toEqual(["reasoningEffort"]); + } + expect( + providerChainEntrySchema.safeParse({ providerType, enabled: true }) + .success, + ).toBe(true); + }, + ); + + it("the chain body still accepts a chain without the setting", () => { + expect( + providerChainPutSchema.safeParse({ + chain: [{ providerType: "local", priority: 1, enabled: true }], + }).success, + ).toBe(true); + }); + + it("the provider PATCH accepts the five values for both entries", () => { + for (const v of VALUES) { + expect( + aiProviderPatchSchema.safeParse({ + localReasoningEffort: v, + compatReasoningEffort: v, + }).success, + ).toBe(true); + } + expect( + aiProviderPatchSchema.safeParse({ localReasoningEffort: "max" }).success, + ).toBe(false); + }); +}); diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index 27c96df89..e753050ef 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -23,6 +23,7 @@ import { resetJsonModeDialectCache, type JsonModeDialect, } from "./json-dialect"; +import type { ReasoningEffort } from "./reasoning-effort"; interface LocalClientConfig { apiKey?: string | null; @@ -108,6 +109,8 @@ export class LocalOpenAICompatibleClient implements AIProvider { readonly type = "local" as const; /** See `AIProvider.responseTimeoutSeconds`; stamped by the resolver. */ responseTimeoutSeconds: number | null = null; + /** See `AIProvider.reasoningEffort`; stamped by the resolver. */ + reasoningEffort: ReasoningEffort | null = null; /** Local servers commonly reject an unknown `tools` field — never send it. */ readonly supportsTools = false; private config: LocalClientConfig; @@ -164,6 +167,12 @@ export class LocalOpenAICompatibleClient implements AIProvider { // most OpenAI-compatible local servers honour it; servers that // ignore it simply disregard the field. Omitted when unset. ...(params.seed !== undefined ? { seed: params.seed } : {}), + // #1126 — the entry's reasoning setting. Default sends nothing, so the + // body of a setup that never touched it stays byte-identical; "Off" is + // `none`, which stops a thinking model spending the answer budget. + ...(this.reasoningEffort + ? { reasoning_effort: this.reasoningEffort } + : {}), // v1.28.28 (#470) — the JSON surfaces opt in via `responseFormat: // "json"`. Default dialect sends the STANDARD OpenAI field (the // top-level Ollama-native `format: "json"` this client used to send diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index 63464563c..f54ea1bf1 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -10,6 +10,7 @@ import { rememberJsonModeDialect, } from "./json-dialect"; import { aiEgressPolicyFor } from "./local-host-allowlist"; +import type { ReasoningEffort } from "./reasoning-effort"; import { buildOpenAIMessages, buildOpenAITools, @@ -62,6 +63,11 @@ export class OpenAIClient implements AIProvider { readonly type: OpenAIProviderType; /** See `AIProvider.responseTimeoutSeconds`; stamped by the resolver. */ responseTimeoutSeconds: number | null = null; + /** + * See `AIProvider.reasoningEffort`; stamped by the resolver. Only the + * gateway tag ever sends it (`isGateway` below), whatever is stamped here. + */ + reasoningEffort: ReasoningEffort | null = null; private config: OpenAIClientConfig; constructor(config: OpenAIClientConfig) { @@ -150,6 +156,11 @@ export class OpenAIClient implements AIProvider { : {}), ...(tools ? { tools } : {}), ...(params.toolChoice ? { tool_choice: params.toolChoice } : {}), + // #1126 — the gateway entry's reasoning setting; Default sends + // nothing. `api.openai.com` and Codex never get it. + ...(this.isGateway && this.reasoningEffort + ? { reasoning_effort: this.reasoningEffort } + : {}), }), }, // 60 s ceiling so a tar-pit upstream cannot pin a worker diff --git a/src/lib/ai/provider-chain.ts b/src/lib/ai/provider-chain.ts index 6a92ad27a..ccfa8409d 100644 --- a/src/lib/ai/provider-chain.ts +++ b/src/lib/ai/provider-chain.ts @@ -18,6 +18,13 @@ * `PROVIDER_CHAIN_DEFAULT`. */ +import { + isReasoningEffort, + isReasoningProviderType, + type ReasoningEffort, + type ReasoningProviderType, +} from "./reasoning-effort"; + /** * Provider tags used in the chain. `admin-openai` is the legacy * `AppSettings.adminAiKeyEncrypted` fallback (so misconfigured users @@ -63,6 +70,8 @@ export interface ProviderChainEntry { providerType: ProviderChainType; priority: number; enabled: boolean; + /** Only ever set on a `REASONING_PROVIDER_TYPES` entry. Absent = Default. */ + reasoningEffort?: ReasoningEffort; } /** @@ -113,6 +122,7 @@ export function parseProviderChain( providerType?: unknown; priority?: unknown; enabled?: unknown; + reasoningEffort?: unknown; }; const providerType = candidate.providerType; if (typeof providerType !== "string") continue; @@ -128,10 +138,18 @@ export function parseProviderChain( : valid.length + 1; const enabled = typeof candidate.enabled === "boolean" ? candidate.enabled : true; + // A reasoning effort survives only on an entry that may carry one and + // only with a known value; anything else reads as Default. + const reasoningEffort = + isReasoningProviderType(providerType) && + isReasoningEffort(candidate.reasoningEffort) + ? candidate.reasoningEffort + : undefined; valid.push({ providerType: providerType as ProviderChainType, priority, enabled, + ...(reasoningEffort ? { reasoningEffort } : {}), }); } @@ -153,6 +171,39 @@ export function serializeProviderChain( providerType: e.providerType, priority: e.priority, enabled: e.enabled, + ...(e.reasoningEffort ? { reasoningEffort: e.reasoningEffort } : {}), })), ); } + +/** + * The chain to persist after setting (or, with `null`, clearing) one entry's + * reasoning effort from that provider's own settings form. + * + * The setting lives on the chain entry, so a chain that was never customised + * is materialised from the default first. A provider that is not in the chain + * at all gets a DISABLED entry to hold the value: the setting must reach the + * provider when it is resolved as the selected one, and a disabled entry is + * inert everywhere else (every reader filters on `enabled`). Clearing on an + * absent entry writes nothing new. + */ +export function chainWithReasoningEffort( + rawChain: unknown, + providerType: ReasoningProviderType, + effort: ReasoningEffort | null, +): ProviderChainEntry[] { + const chain = parseProviderChain(rawChain).map((e) => ({ ...e })); + const entry = chain.find((e) => e.providerType === providerType); + if (entry) { + if (effort) entry.reasoningEffort = effort; + else delete entry.reasoningEffort; + } else if (effort) { + chain.push({ + providerType, + priority: Math.max(0, ...chain.map((e) => e.priority)) + 1, + enabled: false, + reasoningEffort: effort, + }); + } + return chain; +} diff --git a/src/lib/ai/provider.ts b/src/lib/ai/provider.ts index 89bc856a0..a88e6e5ad 100644 --- a/src/lib/ai/provider.ts +++ b/src/lib/ai/provider.ts @@ -1,5 +1,6 @@ import type { AIProvider, CompletionResult } from "./types"; import { bindResponseTimeout } from "./effective-timeout"; +import { bindReasoningEffort } from "./reasoning-effort"; import { prisma } from "@/lib/db"; import { decrypt } from "@/lib/crypto"; import { CodexClient, resolveCodexVisionSlug } from "./codex-client"; @@ -501,13 +502,19 @@ export async function resolveProvider(userId: string): Promise { role: true, managedProfileAt: true, aiResponseTimeoutSeconds: true, + aiProviderChain: true, }, }); // The record's response-timeout setting rides on whatever this returns, on // every branch, the operator-key fallback for a managed profile included: - // the person waiting on the record is who the call serves. + // the person waiting on the record is who the call serves. The reasoning + // effort (#1126) rides the same way; it only lands on a Local or gateway + // instance, so the operator fallback never picks one up. const bind = (provider: AIProvider): AIProvider => - bindResponseTimeout(provider, userRow?.aiResponseTimeoutSeconds); + bindReasoningEffort( + bindResponseTimeout(provider, userRow?.aiResponseTimeoutSeconds), + userRow?.aiProviderChain, + ); const policy = providerCredentialPolicy( authority, userRow?.managedProfileAt ?? null, @@ -533,18 +540,23 @@ export async function resolveProvider(userId: string): Promise { } /** - * The record owner's `aiResponseTimeoutSeconds`, for `resolveProviderForTest`, - * whose many branches (saved config, unsaved override, the chain) are bound in - * one place on the way out. The other two resolvers read it with their row. + * The record owner's `aiResponseTimeoutSeconds` and `aiProviderChain`, for + * `resolveProviderForTest`, whose many branches (saved config, unsaved + * override, the chain) are bound in one place on the way out. The other two + * resolvers read them with their row. */ -async function readResponseTimeoutSeconds( - userId: string, -): Promise { +async function readBoundSettings(userId: string): Promise<{ + aiResponseTimeoutSeconds: number | null; + aiProviderChain: unknown; +}> { const row = await prisma.user.findUnique({ where: { id: userId }, - select: { aiResponseTimeoutSeconds: true }, + select: { aiResponseTimeoutSeconds: true, aiProviderChain: true }, }); - return row?.aiResponseTimeoutSeconds ?? null; + return { + aiResponseTimeoutSeconds: row?.aiResponseTimeoutSeconds ?? null, + aiProviderChain: row?.aiProviderChain ?? null, + }; } /** @@ -595,9 +607,12 @@ export async function resolveProviderChain( aiResponseTimeoutSeconds: true, }, }); - // See `resolveProvider`: every instance carries the record's setting. + // See `resolveProvider`: every instance carries the record's settings. const bind = (provider: AIProvider): AIProvider => - bindResponseTimeout(provider, userRow?.aiResponseTimeoutSeconds); + bindReasoningEffort( + bindResponseTimeout(provider, userRow?.aiResponseTimeoutSeconds), + userRow?.aiProviderChain, + ); const policy = providerCredentialPolicy( authority, userRow?.managedProfileAt ?? null, @@ -1184,10 +1199,13 @@ export async function resolveProviderForTest( ): Promise { // The test runs under the same timeout generation would, so a slow local // model that needs a raised setting is not reported as dead by the test. + // The same holds for the entry's reasoning effort (#1126): a thinking model + // that only answers with reasoning switched off must test green with it off. const provider = await resolveProviderForTestUnbound(userId, override); - return bindResponseTimeout( - provider, - await readResponseTimeoutSeconds(userId), + const settings = await readBoundSettings(userId); + return bindReasoningEffort( + bindResponseTimeout(provider, settings.aiResponseTimeoutSeconds), + settings.aiProviderChain, ); } diff --git a/src/lib/ai/reasoning-effort.ts b/src/lib/ai/reasoning-effort.ts new file mode 100644 index 000000000..ccf1f6906 --- /dev/null +++ b/src/lib/ai/reasoning-effort.ts @@ -0,0 +1,83 @@ +/** + * #1126 — the per-entry reasoning setting of the Local and OpenAI-compatible + * providers, and the binder that stamps it onto a resolved provider. + * + * A leaf on purpose: `types.ts` and the binder are reachable from workers that + * must not reach the provider machinery (`data-arrival-provider-isolation`), + * so nothing here imports `provider-chain.ts`. The stored entry is read from + * the raw chain directly, first match wins, the same rule + * `parseProviderChain` dedupes by. + */ +import type { AIProvider } from "./types"; + +/** + * The `reasoning_effort` values a chain entry may carry. "Off" in the settings + * is `none` on the wire. The absence of the key is "Default": nothing is sent + * and the model decides, which is what every chain stored before the setting + * existed keeps doing. + */ +export const REASONING_EFFORTS = ["none", "low", "medium", "high"] as const; + +export type ReasoningEffort = (typeof REASONING_EFFORTS)[number]; + +/** + * The entries that may carry a reasoning effort: the two whose wire is a + * person-chosen `/chat/completions` endpoint serving a model of their choice. + * OpenAI, Anthropic and Codex pin their own model families and parameters, + * so the setting is refused for them rather than silently dropped. + */ +export const REASONING_PROVIDER_TYPES = ["local", "openai-compatible"] as const; + +export type ReasoningProviderType = (typeof REASONING_PROVIDER_TYPES)[number]; + +export function isReasoningProviderType( + providerType: string, +): providerType is ReasoningProviderType { + return (REASONING_PROVIDER_TYPES as readonly string[]).includes(providerType); +} + +export function isReasoningEffort(value: unknown): value is ReasoningEffort { + return ( + typeof value === "string" && + (REASONING_EFFORTS as readonly string[]).includes(value) + ); +} + +/** + * The reasoning effort stored on a provider type's entry in a persisted chain, + * enabled or not. `null` is Default: no chain, no entry, no valid value, or a + * type that cannot carry one. + */ +export function reasoningEffortFor( + rawChain: unknown, + providerType: string, +): ReasoningEffort | null { + if (!isReasoningProviderType(providerType) || !Array.isArray(rawChain)) { + return null; + } + const entry = rawChain.find( + (item): item is { reasoningEffort?: unknown } => + item != null && + typeof item === "object" && + (item as { providerType?: unknown }).providerType === providerType, + ); + return isReasoningEffort(entry?.reasoningEffort) + ? entry.reasoningEffort + : null; +} + +/** + * Stamp the record owner's reasoning effort onto a resolved provider. Keyed on + * the instance's runtime type, so only a Local client or an OpenAI-compatible + * gateway can ever carry one: the operator's provider, the personal OpenAI + * key, Anthropic and Codex resolve to `null` whatever the stored chain says. + * Called by every exported resolver in `provider.ts` next to + * `bindResponseTimeout`, with the raw `User.aiProviderChain` value. + */ +export function bindReasoningEffort

( + provider: P, + aiProviderChain: unknown, +): P { + provider.reasoningEffort = reasoningEffortFor(aiProviderChain, provider.type); + return provider; +} diff --git a/src/lib/ai/types.ts b/src/lib/ai/types.ts index 541dcbb24..7ad8be174 100644 --- a/src/lib/ai/types.ts +++ b/src/lib/ai/types.ts @@ -1,3 +1,4 @@ +import type { ReasoningEffort } from "./reasoning-effort"; import { z } from "zod/v4"; // ─── Insight Result Schema ───────────────────────────────── @@ -123,6 +124,13 @@ export interface AIProvider { * `null` / absent → the surface's `timeoutMs`, else the 60 s default. */ responseTimeoutSeconds?: number | null; + /** + * #1126 — the `reasoning_effort` the record owner chose for this provider's + * chain entry, stamped by the resolver (`bindReasoningEffort`). Only the + * Local client and the OpenAI-compatible gateway read it, and only they are + * ever stamped with a value. `null` / absent → Default: the key is not sent. + */ + reasoningEffort?: ReasoningEffort | null; generateCompletion(params: CompletionParams): Promise; /** * v1.22 (#89) — optional true-streaming variant. A provider that can emit diff --git a/src/lib/export/__tests__/account-settings-backup.test.ts b/src/lib/export/__tests__/account-settings-backup.test.ts index b0768b7e9..3414b51d1 100644 --- a/src/lib/export/__tests__/account-settings-backup.test.ts +++ b/src/lib/export/__tests__/account-settings-backup.test.ts @@ -19,6 +19,7 @@ import { buildAccountSettingsBackupSection, } from "@/lib/export/account-settings-backup"; import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker"; +import { reasoningEffortFor } from "@/lib/ai/reasoning-effort"; /** A 1x1 PNG, the smallest image the avatar upload accepts. */ const PNG_1X1 = @@ -190,6 +191,38 @@ describe("admitAccountSettings", () => { ]); }); + // #1126 — the reasoning setting lives on the provider chain entries, so + // the chain column carries it through a backup unchanged. + it("carries the provider chain's reasoning settings through a round trip", async () => { + const chain = [ + { + providerType: "local", + priority: 1, + enabled: true, + reasoningEffort: "none", + }, + { + providerType: "openai-compatible", + priority: 2, + enabled: false, + reasoningEffort: "high", + }, + { providerType: "openai", priority: 3, enabled: true }, + ]; + const { accountSettings } = await buildAccountSettingsBackupSection( + prismaReturning(rowWith({ aiProviderChain: chain })), + "u1", + { purpose: "disaster-recovery" }, + ); + const restored = admitAccountSettings( + JSON.parse(JSON.stringify(accountSettings)), + ctx, + ).data.aiProviderChain; + expect(restored).toEqual(chain); + expect(reasoningEffortFor(restored, "local")).toBe("none"); + expect(reasoningEffortFor(restored, "openai-compatible")).toBe("high"); + }); + it("writes a database null for a JSON setting the file clears", () => { expect( admitAccountSettings({ dashboardWidgetsJson: null }, ctx).data diff --git a/src/lib/openapi/routes/insights/schemas.ts b/src/lib/openapi/routes/insights/schemas.ts index cc49e7eca..16b65fd5b 100644 --- a/src/lib/openapi/routes/insights/schemas.ts +++ b/src/lib/openapi/routes/insights/schemas.ts @@ -18,7 +18,7 @@ import { SPARKLINE_MAX_POINTS, } from "@/lib/insights/derived/types"; import { ANALYTICS_RANGES } from "@/lib/analytics/range-delta"; -import { PROVIDER_CHAIN_TYPES } from "@/lib/ai/provider-chain"; +import { providerChainPutSchema } from "@/lib/validations/ai-provider"; import { PERIOD_DAYS } from "@/lib/insights/narrative/period-narrative"; import { aiCapabilityState } from "../profile"; @@ -2531,36 +2531,11 @@ export const providerChainResponse = z "The account's AI-provider chain as the settings surface reads it. Provider TYPES and boolean state only — no key, token or account id is ever on this wire.", }); -export const providerChainPutRequest = z - .object({ - chain: z - .array( - z.object({ - providerType: z - .enum(PROVIDER_CHAIN_TYPES as unknown as [string, ...string[]]) - .describe( - "Closed allow-list. The mock provider is excluded from it structurally, which is what keeps production from reaching one.", - ), - priority: z - .number() - .int() - .optional() - .describe( - "ACCEPTED AND IGNORED. Priority is recomputed from the array's insertion order, so a stale client cannot persist a chain whose displayed order disagrees with its stored one. Send the order you want as the order of the array.", - ), - enabled: z.boolean(), - }), - ) - .min(1) - .describe( - "The whole chain, in the order it should be walked. At least one entry and at most one per known provider type; a repeated type is refused.", - ), - }) - .meta({ - id: "ProviderChainPutRequest", - description: - "Replaces the account's provider chain wholesale. No credential is written or read here — only which provider types exist, in what order, and which are enabled.", - }); +export const providerChainPutRequest = providerChainPutSchema.meta({ + id: "ProviderChainPutRequest", + description: + "Replaces the account's provider chain wholesale. No credential is written or read here — only which provider types exist, in what order, which are enabled, and the reasoning setting of the Local and OpenAI-compatible entries.", +}); // ── Read-only advisor + generation (`/api/insights/generate`) ───────── diff --git a/src/lib/openapi/routes/profile.ts b/src/lib/openapi/routes/profile.ts index 257f01428..261cd63a5 100644 --- a/src/lib/openapi/routes/profile.ts +++ b/src/lib/openapi/routes/profile.ts @@ -36,6 +36,7 @@ import { } from "@/lib/validations/user-prefs"; import { aiProviderPatchSchema, + reasoningEffortSchema, aiTestOverrideSchema, } from "@/lib/validations/ai-provider"; import { @@ -674,6 +675,12 @@ const aiProviderResponse = z .describe( "v1.22 (#89) — per-user response timeout for AI generation, in seconds (10–600). Null = the built-in comprehensive-briefing default (~120 s). Mainly for slow local/self-hosted backends.", ), + localReasoningEffort: reasoningEffortSchema.describe( + "#1126 — reasoning setting of the Local provider entry, sent as `reasoning_effort` (`none` = Off). Null = Default: the key is not sent.", + ), + compatReasoningEffort: reasoningEffortSchema.describe( + "#1126 — the same for the OpenAI-compatible gateway entry.", + ), // v1.38.19 — the setup flow's last screen used to promise // that insights "work for you right away" as soon as `managedBy` read // `server`. That is a presence read; it never knew whether the key diff --git a/src/lib/validations/ai-provider.ts b/src/lib/validations/ai-provider.ts index 7b51bcffe..dc8f92802 100644 --- a/src/lib/validations/ai-provider.ts +++ b/src/lib/validations/ai-provider.ts @@ -16,6 +16,12 @@ */ import { z } from "zod/v4"; +import { PROVIDER_CHAIN_TYPES } from "@/lib/ai/provider-chain"; +import { + isReasoningProviderType, + REASONING_EFFORTS, +} from "@/lib/ai/reasoning-effort"; + /** The five provider kinds a user may select. */ export const AI_PROVIDER_KINDS = [ "OPENAI", @@ -38,6 +44,17 @@ export const aiProviderKindSchema = z.enum(AI_PROVIDER_KINDS); */ const clearableString = z.string().nullable(); +/** + * #1126 — a provider entry's reasoning setting. `null` is "Default": nothing + * is sent and the model decides. `none` is "Off". + */ +export const reasoningEffortSchema = z + .enum(REASONING_EFFORTS) + .nullable() + .describe( + "Reasoning setting for a Local or OpenAI-compatible provider, sent as `reasoning_effort` in the `/chat/completions` body. `none` switches reasoning off, which stops a thinking model (Gemma, Qwen and the like) spending the answer budget on reasoning. `null` is Default: the key is not sent at all and the model decides.", + ); + /** * `PATCH /api/user/ai-provider` — a partial update of the provider config. * @@ -93,6 +110,22 @@ export const aiProviderPatchSchema = z.object({ .max(600, "Response timeout must be between 10 and 600 seconds") .nullable() .optional(), + /** + * #1126 — the reasoning setting of the Local entry. Stored on the entry in + * the provider chain; a chain never customised is materialised from the + * default to hold it. `null` restores Default. + */ + localReasoningEffort: reasoningEffortSchema + .optional() + .describe( + "Reasoning setting of the Local provider entry, stored on that entry of the provider chain. `none` = Off, `null` = Default (nothing sent). Omitted leaves it untouched.", + ), + /** #1126 — the same for the OpenAI-compatible gateway entry. */ + compatReasoningEffort: reasoningEffortSchema + .optional() + .describe( + "Reasoning setting of the OpenAI-compatible gateway entry, stored on that entry of the provider chain. `none` = Off, `null` = Default (nothing sent). Omitted leaves it untouched.", + ), }); export type AiProviderPatchInput = z.infer; @@ -115,3 +148,61 @@ export const aiTestOverrideSchema = z .strict(); export type AiTestOverrideInput = z.infer; + +/** + * One entry of `PUT /api/insights/provider-chain`. Shared by the route and the + * OpenAPI registry, so the published contract and the parser are one object. + */ +export const providerChainEntrySchema = z + .object({ + providerType: z + .enum(PROVIDER_CHAIN_TYPES) + .describe( + "Closed allow-list. The mock provider is excluded from it structurally, which is what keeps production from reaching one.", + ), + // Priority is recomputed server-side from insertion order so a stale + // client cannot persist a chain whose displayed order disagrees with + // its priority field. The number is accepted (and may be present) but + // ignored on the wire. + priority: z + .number() + .int() + .optional() + .describe( + "ACCEPTED AND IGNORED. Priority is recomputed from the array's insertion order, so a stale client cannot persist a chain whose displayed order disagrees with its stored one. Send the order you want as the order of the array.", + ), + enabled: z.boolean(), + reasoningEffort: reasoningEffortSchema + .optional() + .describe( + "Only on a `local` or `openai-compatible` entry; refused on any other. Omitted keeps the value already stored for that provider, `null` restores Default, `none` / `low` / `medium` / `high` are sent as `reasoning_effort`.", + ), + }) + .strict() + .superRefine((entry, ctx) => { + if ( + entry.reasoningEffort !== undefined && + !isReasoningProviderType(entry.providerType) + ) { + ctx.addIssue({ + code: "custom", + path: ["reasoningEffort"], + message: + "A reasoning setting applies to the local and openai-compatible providers only", + }); + } + }); + +export const providerChainPutSchema = z + .object({ + chain: z + .array(providerChainEntrySchema) + .min(1, "Chain must contain at least one provider") + .max(PROVIDER_CHAIN_TYPES.length, "Too many providers") + .describe( + "The whole chain, in the order it should be walked. At least one entry and at most one per known provider type; a repeated type is refused.", + ), + }) + .strict(); + +export type ProviderChainPutInput = z.infer; From 9cb06ec44d2ac322f31d234ef88997827f365c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:43:00 +0200 Subject: [PATCH 17/21] fix(idempotency): read a rebuilt multipart form as the same request A client that rebuilds a form for a retry picks a fresh random boundary, so the raw bytes differ and a repeated document upload ran a second time instead of replaying. The boundary named in Content-Type is replaced by a fixed token before hashing. --- src/lib/__tests__/idempotency.test.ts | 32 +++++++++++++++++++++++ src/lib/idempotency.ts | 37 +++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/lib/__tests__/idempotency.test.ts b/src/lib/__tests__/idempotency.test.ts index a93a23ee1..37efb2d1c 100644 --- a/src/lib/__tests__/idempotency.test.ts +++ b/src/lib/__tests__/idempotency.test.ts @@ -990,6 +990,38 @@ describe("request fingerprint", () => { expect(sys).not.toBe(dia); }); + it("reads a rebuilt multipart form with a fresh boundary as the same upload", () => { + const form = (boundary: string, name: string) => + enc( + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${name}"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4 body\r\n--${boundary}--\r\n`, + ); + const type = (boundary: string) => + `multipart/form-data; boundary=${boundary}`; + const first = requestFingerprint( + "POST", + "/api/documents", + form("----formdata-a1", "brief.pdf"), + null, + type("----formdata-a1"), + ); + const retry = requestFingerprint( + "POST", + "/api/documents", + form("----formdata-b2", "brief.pdf"), + null, + type("----formdata-b2"), + ); + expect(retry).toBe(first); + const other = requestFingerprint( + "POST", + "/api/documents", + form("----formdata-b2", "other.pdf"), + null, + type("----formdata-b2"), + ); + expect(other).not.toBe(first); + }); + it("hashes a non-JSON or empty body as its raw bytes", () => { const text = requestFingerprint("POST", "/p", enc("not json")); expect(text).toBe(requestFingerprint("POST", "/p", enc("not json"))); diff --git a/src/lib/idempotency.ts b/src/lib/idempotency.ts index 82d2cc0b6..7b374faf3 100644 --- a/src/lib/idempotency.ts +++ b/src/lib/idempotency.ts @@ -158,20 +158,52 @@ export function canonicalJson(value: unknown): string { return JSON.stringify(value); } +/** + * Replace every occurrence of the multipart boundary named in `contentType` + * with a fixed token. Any other body is returned as it is. + */ +function withoutMultipartBoundary( + body: Uint8Array, + contentType: string | null, +): Uint8Array { + if (!contentType || !/^multipart\//i.test(contentType)) return body; + const match = /;\s*boundary="?([^";]+)"?/i.exec(contentType); + if (!match) return body; + const boundary = Buffer.from(match[1], "utf8"); + const token = Buffer.from("boundary", "utf8"); + const source = Buffer.from(body.buffer, body.byteOffset, body.byteLength); + const parts: Buffer[] = []; + let from = 0; + for (;;) { + const at = source.indexOf(boundary, from); + if (at === -1) break; + parts.push(source.subarray(from, at), token); + from = at + boundary.length; + } + if (parts.length === 0) return body; + parts.push(source.subarray(from)); + return new Uint8Array(Buffer.concat(parts)); +} + /** * The fingerprint of a request: SHA-256 hex over method, path and the body in * canonical form. A body that parses as JSON is hashed as `canonicalJson`, so * the same object serialised with a different key order is the same request; * anything else (multipart, plain text, an empty body) is hashed as its raw * bytes. `truncatedAt` marks a body cut at `FINGERPRINT_MAX_BYTES`, hashed as - * that raw prefix together with the declared length. Exported for unit tests. + * that raw prefix together with the declared length. A multipart body has its + * boundary replaced by a fixed token first: a client that rebuilds the form + * for a retry picks a fresh random boundary, and the same upload must still + * replay. Exported for unit tests. */ export function requestFingerprint( method: string, path: string, - body: Uint8Array, + rawBody: Uint8Array, truncatedAt: string | null = null, + contentType: string | null = null, ): string { + const body = withoutMultipartBoundary(rawBody, contentType); const hash = createHash("sha256") .update(method) .update("\0") @@ -237,6 +269,7 @@ async function fingerprintRequest( path, body, truncated ? (request.headers.get("content-length") ?? "unknown") : null, + request.headers.get("content-type"), ); } From 39a5862d5447cbf13dee60411c2927aeb249dd20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:43:04 +0200 Subject: [PATCH 18/21] feat(settings): add a Reasoning select to the Local and gateway forms One select under the model field of the Local and OpenAI-compatible provider forms: Default, Off, Low, Medium, High, with a one-line hint that Off leaves the answer budget for the answer itself. It reuses the NativeSelect, label and hint layout of the field above it, seeds from the stored entry, and saves with the form. Strings in all seven locales. Refs #1126 --- e2e/ai-provider-dropdown.spec.ts | 112 ++++++++++++++++++ messages/de.json | 11 ++ messages/en.json | 11 ++ messages/es.json | 11 ++ messages/fr.json | 11 ++ messages/it.json | 11 ++ messages/ko.json | 11 ++ messages/pl.json | 11 ++ .../__tests__/reasoning-effort-field.test.tsx | 106 +++++++++++++++++ .../settings/ai/compat-provider-form.tsx | 17 ++- .../settings/ai/local-provider-form.tsx | 13 +- .../settings/ai/reasoning-effort-field.tsx | 52 ++++++++ src/components/settings/ai/shared.ts | 9 ++ 13 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 src/components/settings/ai/__tests__/reasoning-effort-field.test.tsx create mode 100644 src/components/settings/ai/reasoning-effort-field.tsx diff --git a/e2e/ai-provider-dropdown.spec.ts b/e2e/ai-provider-dropdown.spec.ts index ea2c00bb2..adacbdd2c 100644 --- a/e2e/ai-provider-dropdown.spec.ts +++ b/e2e/ai-provider-dropdown.spec.ts @@ -339,4 +339,116 @@ test.describe("Settings → AI provider dropdown UX (B2)", () => { .locator('button[type="submit"]'), ).toBeEnabled(); }); + + // #1126 — the Reasoning select on the Local and OpenAI-compatible forms + // saves with the form, and at phone width it sits in the same column as + // the model field without pushing the page sideways. + test("reasoning setting saves from the Local and gateway forms at 390 px", async ({ + page, + }) => { + await page.setViewportSize({ width: 390, height: 844 }); + const patchBodies: Array> = []; + let providerState: Record = { + provider: "LOCAL", + model: null, + baseUrl: "http://localhost:11434/v1", + responseTimeoutSeconds: null, + hasAnthropicKey: false, + anthropicKeyPreview: null, + hasLocalKey: false, + hasOpenaiKey: false, + openaiKeyPreview: null, + compatBaseUrl: "https://gateway.example.org/v1", + compatModel: "qwen3", + hasCompatKey: false, + localReasoningEffort: null, + compatReasoningEffort: null, + }; + await page.route("**/api/insights/settings", (route) => + route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + data: { + codexStatus: "disconnected", + codexConnectedAt: null, + hasAdminKey: false, + codexOauthConfigured: true, + privacyMode: "aggregated", + lastInsightAt: null, + }, + error: null, + }), + }), + ); + await page.route("**/api/user/ai-provider", async (route) => { + if (route.request().method() === "PATCH") { + const body = route.request().postDataJSON() as Record; + patchBodies.push(body); + providerState = { ...providerState, ...body }; + } + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ data: providerState, error: null }), + }); + }); + await page.route("**/api/insights/provider-chain", (route) => + route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + data: { + activeProvider: "local", + cachedActiveProvider: null, + configuredChain: [ + { providerType: "local", enabled: true, available: true }, + ], + }, + error: null, + }), + }), + ); + + const providerHydrated = page.waitForResponse("**/api/user/ai-provider"); + await page.goto("/settings/ai?provider=local", { + waitUntil: "domcontentloaded", + }); + await providerHydrated; + + const localForm = page.getByTestId("ai-provider-config-local"); + const localReasoning = page.locator("#ai-local-reasoning"); + await expect(localReasoning).toHaveValue(""); + // Same column, same width as the model select above it. + const [modelBox, reasoningBox] = await Promise.all([ + page.locator("#ai-local-model").boundingBox(), + localReasoning.boundingBox(), + ]); + expect(reasoningBox?.x).toBe(modelBox?.x); + expect(reasoningBox?.width).toBe(modelBox?.width); + expect( + await page.evaluate( + () => + document.documentElement.scrollWidth <= + document.documentElement.clientWidth, + ), + ).toBe(true); + + await localReasoning.selectOption("none"); + await localForm.locator('button[type="submit"]').click(); + await expect.poll(() => patchBodies.length).toBe(1); + expect(patchBodies[0].localReasoningEffort).toBe("none"); + + await page + .getByTestId("ai-active-provider-select") + .selectOption("openai-compatible"); + const compatForm = page.getByTestId("ai-provider-config-openai-compatible"); + const compatReasoning = page.locator("#ai-compat-reasoning"); + await expect(compatReasoning).toHaveValue(""); + await compatReasoning.selectOption("high"); + await compatForm.locator('button[type="submit"]').click(); + await expect.poll(() => patchBodies.length).toBe(2); + expect(patchBodies[1].compatReasoningEffort).toBe("high"); + expect(patchBodies[1]).not.toHaveProperty("localReasoningEffort"); + }); }); diff --git a/messages/de.json b/messages/de.json index 0c7173452..5186bb8b6 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6080,6 +6080,17 @@ "title": "Coach-Pläne", "description": "Die Pläne, die du mit dem Coach vereinbart hast, zum Lesen oder Entfernen.", "open": "Pläne öffnen" + }, + "reasoning": { + "label": "Nachdenken", + "hint": "Reasoning-Modelle wie Gemma oder Qwen denken nach, bevor sie antworten. Schaltest du es aus, bleibt das ganze Antwortbudget für die Antwort selbst.", + "options": { + "default": "Standard", + "none": "Aus", + "low": "Niedrig", + "medium": "Mittel", + "high": "Hoch" + } } }, "dashboard": { diff --git a/messages/en.json b/messages/en.json index 3cd9aa9a4..fc41483a9 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6080,6 +6080,17 @@ "title": "Coach plans", "description": "The plans you agreed with the Coach, to read or remove.", "open": "Open plans" + }, + "reasoning": { + "label": "Reasoning", + "hint": "Reasoning models such as Gemma or Qwen think before they answer. Off leaves the whole answer budget for the answer itself.", + "options": { + "default": "Default", + "none": "Off", + "low": "Low", + "medium": "Medium", + "high": "High" + } } }, "dashboard": { diff --git a/messages/es.json b/messages/es.json index d40131208..90919f624 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6080,6 +6080,17 @@ "title": "Planes del Coach", "description": "Los planes que acordaste con el Coach, para leerlos o eliminarlos.", "open": "Abrir planes" + }, + "reasoning": { + "label": "Razonamiento", + "hint": "Los modelos de razonamiento como Gemma o Qwen piensan antes de responder. Si lo desactivas, todo el presupuesto de respuesta se queda para la respuesta.", + "options": { + "default": "Predeterminado", + "none": "Desactivado", + "low": "Bajo", + "medium": "Medio", + "high": "Alto" + } } }, "dashboard": { diff --git a/messages/fr.json b/messages/fr.json index a5d0afa08..5d0c5570d 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6080,6 +6080,17 @@ "title": "Plans du Coach", "description": "Les plans convenus avec le Coach, à lire ou à supprimer.", "open": "Ouvrir les plans" + }, + "reasoning": { + "label": "Raisonnement", + "hint": "Les modèles de raisonnement comme Gemma ou Qwen réfléchissent avant de répondre. Désactivé, tout le budget de réponse sert à la réponse elle-même.", + "options": { + "default": "Par défaut", + "none": "Désactivé", + "low": "Faible", + "medium": "Moyen", + "high": "Élevé" + } } }, "dashboard": { diff --git a/messages/it.json b/messages/it.json index e0e489863..8654d8fea 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6080,6 +6080,17 @@ "title": "Piani del Coach", "description": "I piani concordati con il Coach, da leggere o rimuovere.", "open": "Apri i piani" + }, + "reasoning": { + "label": "Ragionamento", + "hint": "I modelli di ragionamento come Gemma o Qwen pensano prima di rispondere. Se lo disattivi, tutto il budget della risposta resta per la risposta stessa.", + "options": { + "default": "Predefinito", + "none": "Disattivato", + "low": "Basso", + "medium": "Medio", + "high": "Alto" + } } }, "dashboard": { diff --git a/messages/ko.json b/messages/ko.json index e51af7017..ac44f12e5 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6080,6 +6080,17 @@ "title": "코치 계획", "description": "코치와 합의한 계획을 읽거나 삭제할 수 있습니다.", "open": "계획 열기" + }, + "reasoning": { + "label": "추론", + "hint": "Gemma나 Qwen 같은 추론 모델은 답하기 전에 먼저 생각합니다. 끄면 답변 예산을 모두 답변 자체에 씁니다.", + "options": { + "default": "기본값", + "none": "끄기", + "low": "낮음", + "medium": "보통", + "high": "높음" + } } }, "dashboard": { diff --git a/messages/pl.json b/messages/pl.json index caa7f9245..17a2e68fe 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6080,6 +6080,17 @@ "title": "Plany z Coachem", "description": "Plany uzgodnione z Coachem, do przeczytania lub usunięcia.", "open": "Otwórz plany" + }, + "reasoning": { + "label": "Rozumowanie", + "hint": "Modele rozumujące, takie jak Gemma czy Qwen, myślą przed odpowiedzią. Po wyłączeniu cały budżet odpowiedzi idzie na samą odpowiedź.", + "options": { + "default": "Domyślnie", + "none": "Wyłączone", + "low": "Niskie", + "medium": "Średnie", + "high": "Wysokie" + } } }, "dashboard": { diff --git a/src/components/settings/ai/__tests__/reasoning-effort-field.test.tsx b/src/components/settings/ai/__tests__/reasoning-effort-field.test.tsx new file mode 100644 index 000000000..0f2f2efd2 --- /dev/null +++ b/src/components/settings/ai/__tests__/reasoning-effort-field.test.tsx @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +import { I18nProvider } from "@/lib/i18n/context"; +import en from "../../../../../messages/en.json"; +import { CompatProviderForm } from "../compat-provider-form"; +import { LocalProviderForm } from "../local-provider-form"; +import type { UserAIProvider } from "../shared"; + +/** + * #1126 — the Reasoning select on the Local and OpenAI-compatible forms. It + * sits right under the model field, offers exactly Default / Off / Low / + * Medium / High, and seeds from the value the server stored on the entry. + * Saving it is pinned end to end in `e2e/ai-provider-dropdown.spec.ts`. + */ + +function provider(overrides: Partial): UserAIProvider { + return { + provider: "LOCAL", + model: "llama3:8b", + baseUrl: "http://ollama.example.org:11434/v1", + hasAnthropicKey: false, + anthropicKeyPreview: null, + hasLocalKey: false, + hasOpenaiKey: false, + openaiKeyPreview: null, + compatBaseUrl: "https://gateway.example.org/v1", + compatModel: "qwen3", + hasCompatKey: false, + responseTimeoutSeconds: null, + ...overrides, + }; +} + +function render(node: React.ReactNode) { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return renderToStaticMarkup( + + {node} + , + ); +} + +/** The `", start)); +} + +const OPTION_VALUES = ["", "none", "low", "medium", "high"]; + +describe.each([ + { + name: "Local", + id: "ai-local-reasoning", + after: "ai-local-model", + form: (p: UserAIProvider) => , + stored: (v: UserAIProvider["localReasoningEffort"]) => ({ + localReasoningEffort: v, + }), + }, + { + name: "OpenAI-compatible", + id: "ai-compat-reasoning", + after: "ai-compat-model", + form: (p: UserAIProvider) => , + stored: (v: UserAIProvider["compatReasoningEffort"]) => ({ + compatReasoningEffort: v, + provider: "OPENAI_COMPATIBLE", + }), + }, +])("$name form", ({ id, after, form, stored }) => { + it("offers Default, Off, Low, Medium and High under the model field", () => { + const html = render(form(provider({}))); + const select = selectMarkup(html, id); + expect( + [...select.matchAll(/`); + } + expect(html).toContain(` { + const select = selectMarkup(render(form(provider({}))), id); + expect(select).toMatch(/

+
)} +