From 0919d8b962717b52bcdd80d91714a0f639a446a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 01:44:41 +0200 Subject: [PATCH 01/52] fix(settings): make Delete All Data delete what it says MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wipe deleted thirteen tables against a schema that carries more than eighty user-scoped ones. Mental-health assessments, lab results, the menstrual record, allergies, family history, the document vault, every AI conversation and live clinician share links all survived a confirmation that promised to permanently delete all health data. So did MeasurementRollup, the per-day reconstruction of the measurements the same transaction had just removed. The list now lives in src/lib/data-wipe/wipe-plan.ts next to its own exemption list, and a structural guard enumerates schema.prisma and fails when a model belongs to neither. Adding a model can no longer widen the set of things that survive a wipe by default. The same guard covers the User row's own columns, which had drifted the same way. The boundary is explicit: everything the account owns goes, except what is needed to sign in again. The confirmation and the toast say so before the person types the confirmation rather than after. Checkup reminders and allergies promised permanence over a soft delete with no restore path; both now hard-delete, so the dialog is true. The lab biomarker dialog was checked and is already accurate — it maps to the hard-deleting /api/biomarkers/[id] route, not to the soft-deleting lab result route. --- messages/de.json | 6 +- messages/en.json | 6 +- messages/es.json | 6 +- messages/fr.json | 6 +- messages/it.json | 6 +- messages/pl.json | 6 +- .../copy-matches-behaviour-guard.test.ts | 9 +- src/__tests__/data-wipe-completeness.test.ts | 286 +++++++++++++ src/app/api/allergies/[id]/route.ts | 26 +- .../api/measurement-reminders/[id]/route.ts | 13 +- src/app/api/settings/data/route.ts | 136 +++--- src/lib/data-wipe/wipe-plan.ts | 403 ++++++++++++++++++ 12 files changed, 796 insertions(+), 113 deletions(-) create mode 100644 src/__tests__/data-wipe-completeness.test.ts create mode 100644 src/lib/data-wipe/wipe-plan.ts diff --git a/messages/de.json b/messages/de.json index 4728f85d6..ee218f830 100644 --- a/messages/de.json +++ b/messages/de.json @@ -5217,10 +5217,10 @@ "apiEndpointAuth": "Authentifizierung", "apiEndpointExample": "Body-Beispiel", "dangerZone": "Alle Daten löschen", - "dangerZoneDescription": "Löscht alle deine Gesundheitsdaten und Integrationen. Dein Nutzerkonto bleibt erhalten.", + "dangerZoneDescription": "Löscht deine gesamte Gesundheitsakte und jede Integrationsverbindung. Dein Konto und deine Anmeldedaten bleiben erhalten.", "dangerZoneConfirm": "Alle Daten löschen?", - "dangerZoneConfirmDescription": "Dieser Vorgang löscht alle deine Gesundheitsdaten und Integrationen unwiderruflich. Dein Nutzerkonto bleibt erhalten.", - "dangerZoneSuccess": "Alle persönlichen Daten wurden gelöscht", + "dangerZoneConfirmDescription": "Dieser Vorgang löscht unwiderruflich alles in deiner Akte: Messwerte, Medikamente, Stimmungseinträge, Laborwerte, Zykluseinträge, Dokumente, KI-Gespräche und jede Integrationsverbindung, dazu alles daraus Abgeleitete und alles, was du früher gelöscht hast. Freigabe-Links, die du an eine Praxis gegeben hast, funktionieren sofort nicht mehr. Dein Konto bleibt bestehen, ebenso deine Anmeldedaten und Anzeigeeinstellungen wie Sprache, Zeitzone und Einheiten. Das lässt sich nicht rückgängig machen.", + "dangerZoneSuccess": "Deine Gesundheitsakte wurde gelöscht. Dein Konto und deine Anmeldung bleiben unverändert.", "dangerZoneDeleteFailed": "Löschen fehlgeschlagen", "finalDelete": "Endgültig löschen", "deleteAccountCardTitle": "Konto vollständig löschen", diff --git a/messages/en.json b/messages/en.json index 4191253be..4c498d987 100644 --- a/messages/en.json +++ b/messages/en.json @@ -5217,10 +5217,10 @@ "apiEndpointAuth": "Authentication", "apiEndpointExample": "Body example", "dangerZone": "Delete All Data", - "dangerZoneDescription": "Deletes all your health data and integrations. Your user account will be preserved.", + "dangerZoneDescription": "Erases your whole health record and every integration connection. Your account and the credentials you sign in with are kept.", "dangerZoneConfirm": "Delete all data?", - "dangerZoneConfirmDescription": "This will permanently delete all your health data and integrations. Your user account will be preserved.", - "dangerZoneSuccess": "All personal data has been deleted", + "dangerZoneConfirmDescription": "This permanently deletes everything in your record: measurements, medications, mood entries, lab results, cycle entries, documents, AI conversations and every integration connection, together with everything derived from them and anything you deleted earlier. Share links you gave to a doctor stop working immediately. Your account stays, along with the credentials you sign in with and display settings such as language, timezone and units. This cannot be undone.", + "dangerZoneSuccess": "Your health record has been deleted. Your account and sign-in are unchanged.", "dangerZoneDeleteFailed": "Deletion failed", "finalDelete": "Delete permanently", "deleteAccountCardTitle": "Delete account entirely", diff --git a/messages/es.json b/messages/es.json index d046fbd97..8c609dfcd 100644 --- a/messages/es.json +++ b/messages/es.json @@ -5217,10 +5217,10 @@ "apiEndpointAuth": "Autenticación", "apiEndpointExample": "Ejemplo de cuerpo", "dangerZone": "Eliminar todos los datos", - "dangerZoneDescription": "Elimina todos tus datos de salud e integraciones. Tu cuenta de usuario se conservará.", + "dangerZoneDescription": "Borra todo tu historial de salud y todas las conexiones de integración. Tu cuenta y las credenciales con las que inicias sesión se conservan.", "dangerZoneConfirm": "¿Eliminar todos los datos?", - "dangerZoneConfirmDescription": "Esto eliminará de forma permanente todos tus datos de salud e integraciones. Tu cuenta de usuario se conservará.", - "dangerZoneSuccess": "Se han eliminado todos los datos personales", + "dangerZoneConfirmDescription": "Esto elimina de forma permanente todo lo que hay en tu historial: mediciones, medicamentos, entradas de ánimo, resultados de laboratorio, entradas de ciclo, documentos, conversaciones con la IA y todas las conexiones de integración, junto con todo lo derivado de ello y todo lo que eliminaste antes. Los enlaces compartidos que diste a una consulta dejan de funcionar de inmediato. Tu cuenta se mantiene, junto con las credenciales con las que inicias sesión y los ajustes de presentación como el idioma, la zona horaria y las unidades. Esto no se puede deshacer.", + "dangerZoneSuccess": "Se ha eliminado tu historial de salud. Tu cuenta y tu inicio de sesión no han cambiado.", "dangerZoneDeleteFailed": "Falló la eliminación", "finalDelete": "Eliminar permanentemente", "deleteAccountCardTitle": "Eliminar la cuenta por completo", diff --git a/messages/fr.json b/messages/fr.json index 2eda57ea9..678b46765 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -5217,10 +5217,10 @@ "apiEndpointAuth": "Authentification", "apiEndpointExample": "Exemple de corps", "dangerZone": "Supprimer toutes les données", - "dangerZoneDescription": "Supprime toutes vos données de santé et intégrations. Votre compte utilisateur sera conservé.", + "dangerZoneDescription": "Efface l’intégralité de votre dossier de santé et toutes les connexions d’intégration. Votre compte et vos identifiants de connexion sont conservés.", "dangerZoneConfirm": "Supprimer toutes les données ?", - "dangerZoneConfirmDescription": "Cette action supprimera définitivement toutes vos données de santé et intégrations. Votre compte utilisateur sera conservé.", - "dangerZoneSuccess": "Toutes les données personnelles ont été supprimées", + "dangerZoneConfirmDescription": "Cette action supprime définitivement tout ce que contient votre dossier : mesures, médicaments, entrées d’humeur, résultats de laboratoire, entrées de cycle, documents, conversations avec l’IA et toutes les connexions d’intégration, ainsi que tout ce qui en découle et tout ce que vous aviez supprimé auparavant. Les liens de partage remis à un cabinet cessent aussitôt de fonctionner. Votre compte reste en place, de même que vos identifiants de connexion et vos réglages d’affichage comme la langue, le fuseau horaire et les unités. Cette action est irréversible.", + "dangerZoneSuccess": "Votre dossier de santé a été supprimé. Votre compte et votre connexion sont inchangés.", "dangerZoneDeleteFailed": "Échec de la suppression", "finalDelete": "Supprimer définitivement", "deleteAccountCardTitle": "Supprimer entièrement le compte", diff --git a/messages/it.json b/messages/it.json index f772e54ed..5a791cdc9 100644 --- a/messages/it.json +++ b/messages/it.json @@ -5217,10 +5217,10 @@ "apiEndpointAuth": "Autenticazione", "apiEndpointExample": "Esempio di body", "dangerZone": "Elimina tutti i dati", - "dangerZoneDescription": "Elimina tutti i tuoi dati sanitari e le integrazioni. Il tuo account utente verrà conservato.", + "dangerZoneDescription": "Cancella l’intera cartella sanitaria e ogni connessione di integrazione. Il tuo account e le credenziali con cui accedi restano.", "dangerZoneConfirm": "Eliminare tutti i dati?", - "dangerZoneConfirmDescription": "Questa azione eliminerà definitivamente tutti i tuoi dati sanitari e le integrazioni. Il tuo account utente verrà conservato.", - "dangerZoneSuccess": "Tutti i dati personali sono stati eliminati", + "dangerZoneConfirmDescription": "Questa azione elimina definitivamente tutto ciò che contiene la tua cartella: misurazioni, farmaci, voci dell’umore, risultati di laboratorio, voci del ciclo, documenti, conversazioni con l’IA e ogni connessione di integrazione, insieme a tutto ciò che ne deriva e a tutto quello che avevi eliminato prima. I link di condivisione consegnati a uno studio medico smettono subito di funzionare. Il tuo account resta, insieme alle credenziali con cui accedi e alle impostazioni di visualizzazione come lingua, fuso orario e unità. Non può essere annullato.", + "dangerZoneSuccess": "La tua cartella sanitaria è stata eliminata. Account e accesso restano invariati.", "dangerZoneDeleteFailed": "Eliminazione non riuscita", "finalDelete": "Elimina definitivamente", "deleteAccountCardTitle": "Elimina completamente l’account", diff --git a/messages/pl.json b/messages/pl.json index 45354f0dc..92b747092 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -5217,10 +5217,10 @@ "apiEndpointAuth": "Uwierzytelnianie", "apiEndpointExample": "Przykład treści", "dangerZone": "Usuń wszystkie dane", - "dangerZoneDescription": "Usuwa wszystkie Twoje dane zdrowotne i integracje. Twoje konto użytkownika zostanie zachowane.", + "dangerZoneDescription": "Kasuje całą Twoją dokumentację zdrowotną i wszystkie połączenia integracji. Konto i dane logowania zostają zachowane.", "dangerZoneConfirm": "Usunąć wszystkie dane?", - "dangerZoneConfirmDescription": "Spowoduje to trwałe usunięcie wszystkich Twoich danych zdrowotnych i integracji. Twoje konto użytkownika zostanie zachowane.", - "dangerZoneSuccess": "Wszystkie dane osobowe zostały usunięte", + "dangerZoneConfirmDescription": "Ta czynność trwale usuwa wszystko z Twojej dokumentacji: pomiary, leki, wpisy nastroju, wyniki badań, wpisy cyklu, dokumenty, rozmowy z SI oraz wszystkie połączenia integracji, a wraz z nimi wszystko, co z nich wyliczono, i wszystko, co usunięto wcześniej. Linki udostępnione poradni natychmiast przestają działać. Konto zostaje, razem z danymi logowania i ustawieniami wyświetlania, takimi jak język, strefa czasowa i jednostki. Nie można tego cofnąć.", + "dangerZoneSuccess": "Twoja dokumentacja zdrowotna została usunięta. Konto i logowanie pozostają bez zmian.", "dangerZoneDeleteFailed": "Usuwanie nie powiodło się", "finalDelete": "Usuń trwale", "deleteAccountCardTitle": "Usuń całe konto", diff --git a/src/__tests__/copy-matches-behaviour-guard.test.ts b/src/__tests__/copy-matches-behaviour-guard.test.ts index 5b26a9c1c..345f18467 100644 --- a/src/__tests__/copy-matches-behaviour-guard.test.ts +++ b/src/__tests__/copy-matches-behaviour-guard.test.ts @@ -69,11 +69,12 @@ const FINALITY_PHRASES: Record = { ], it: [ "non può essere annullata", + "non può essere annullato", "definitivamente", "definitiva", "permanente", ], - pl: ["nie można cofnąć", "trwale", "nieodwracaln"], + pl: ["nie można cofnąć", "nie można tego cofnąć", "trwale", "nieodwracaln"], }; /** Per-locale phrase that promises the short undo window. */ @@ -108,6 +109,12 @@ const PERMANENT_CONFIRM_KEYS = [ "mood.manage.purgeBody", // GDPR Art. 17 account purge "settings.deleteAccountConfirmDescription", + // hard delete of every user-scoped row except the sign-in credentials + "settings.dangerZoneConfirmDescription", + // hard delete since v1.32.40 — the tombstone served nothing and the + // dialog already promised permanence + "measurementReminders.deleteConfirmDescription", + "records.allergies.deleteConfirmDescription", ] as const; describe("delete confirmations describe what the surface actually does", () => { diff --git a/src/__tests__/data-wipe-completeness.test.ts b/src/__tests__/data-wipe-completeness.test.ts new file mode 100644 index 000000000..98315046d --- /dev/null +++ b/src/__tests__/data-wipe-completeness.test.ts @@ -0,0 +1,286 @@ +/** + * Structural guard binding `prisma/schema.prisma` to the "Delete All Data" + * wipe plan. + * + * The failure it exists to catch: a model is added to the schema, it is + * user-scoped from birth, and the wipe list is "the follow-up". Nothing fails + * — the route keeps deleting what it always deleted, the toast keeps saying + * "all personal data has been deleted", and the new table quietly joins the + * set of things that survive a confirmation the person typed by hand. That is + * how a 13-table wipe ended up facing an 80-table schema. + * + * Both ends are literals — Prisma model blocks on one side, exported arrays + * and objects on the other — so the matcher is exact. Nothing here greps for + * intent. + * + * What it proves: + * 1. every model carrying a `userId` column is either wiped or exempt with a + * reason; + * 2. every model without one either reaches a wiped model through a chain of + * required `onDelete: Cascade` relations (so the database removes it) or + * is declared instance-scoped; + * 3. every `User` column is either reset or kept with a reason; + * 4. the delete order lists children before parents, so the per-model counts + * in the audit row are truthful; + * 5. no exemption has gone stale. + * + * Its honest limit: it proves classification, not judgement. It cannot tell + * that a model was put on the exempt list for a bad reason — only that + * somebody had to write a reason down. And it reads relations, so a table + * linked to an account by a bare `userId` string with no foreign key (there + * are two: `TelegramScheduledDeletion`, `TelegramPromptContext`) is caught by + * rule 1 only because the column name matches the convention. + * + * Mutation check: remove any entry from `WIPE_MODELS` and this goes red naming + * that model; add a column to `model User` without classifying it and it goes + * red naming the column. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, it, expect } from "vitest"; + +import { + INSTANCE_SCOPED, + USER_KEPT_FIELDS, + USER_RESET, + WIPE_EXEMPT, + WIPE_MODELS, + wipeDelegateKey, +} from "@/lib/data-wipe/wipe-plan"; + +const SCHEMA = join(process.cwd(), "prisma", "schema.prisma"); + +interface Relation { + /** Target model name. */ + target: string; + cascade: boolean; + /** `true` when the relation field is optional (`Model?`). */ + optional: boolean; +} + +interface ModelShape { + scalars: string[]; + relations: Relation[]; +} + +function parseSchema(schema: string): Map { + const blocks = [...schema.matchAll(/^model\s+(\w+)\s*\{([\s\S]*?)^\}/gm)].map( + (m) => ({ name: m[1], body: m[2] }), + ); + const modelNames = new Set(blocks.map((b) => b.name)); + + const out = new Map(); + for (const block of blocks) { + const scalars: string[] = []; + const relations: Relation[] = []; + for (const rawLine of block.body.split("\n")) { + const line = rawLine.trim(); + if (!line || line.startsWith("//") || line.startsWith("@@")) continue; + const match = /^(\w+)\s+(\w+)(\[\])?(\?)?/.exec(line); + if (!match) continue; + const [, field, type, isList, isOptional] = match; + if (modelNames.has(type)) { + // A list side of a relation carries no column and cannot cascade. + if (isList) continue; + relations.push({ + target: type, + cascade: line.includes("onDelete: Cascade"), + optional: Boolean(isOptional), + }); + continue; + } + if (isList && type === type.toUpperCase()) { + // Scalar / enum list column — still a column. + scalars.push(field); + continue; + } + scalars.push(field); + } + out.set(block.name, { scalars, relations }); + } + return out; +} + +const schemaText = readFileSync(SCHEMA, "utf8"); +const models = parseSchema(schemaText); +const wipeSet = new Set(WIPE_MODELS); + +describe("data-wipe completeness", () => { + it("parses the schema into a plausible shape", () => { + expect(models.size).toBeGreaterThan(100); + expect(models.get("Measurement")?.scalars).toContain("userId"); + expect(models.get("Measurement")?.scalars).not.toContain("user"); + expect(models.get("User")?.scalars).toContain("heightCm"); + expect(models.get("User")?.scalars).not.toContain("measurements"); + expect( + models.get("CoachMessage")?.relations.map((r) => r.target), + ).toContain("CoachConversation"); + }); + + it("every model with a userId column is wiped or exempt with a reason", () => { + const userScoped = [...models] + .filter(([, shape]) => shape.scalars.includes("userId")) + .map(([name]) => name); + + expect(userScoped.length).toBeGreaterThan(80); + + const unclassified = userScoped.filter( + (name) => + !wipeSet.has(name) && + !(name in WIPE_EXEMPT) && + !(name in INSTANCE_SCOPED), + ); + + if (unclassified.length > 0) { + throw new Error( + `${unclassified.length} user-scoped model(s) are in neither the wipe list nor the exemption list:\n` + + unclassified.map((n) => ` ❌ ${n}`).join("\n") + + "\n\nA person who types the confirmation on Settings → Delete All Data " + + "expects these rows gone. Add each to WIPE_MODELS, or to WIPE_EXEMPT " + + "with the reason keeping it is not account deletion — and say so in " + + "the confirmation copy.", + ); + } + + const bothWays = userScoped.filter( + (name) => wipeSet.has(name) && name in WIPE_EXEMPT, + ); + expect(bothWays, "a model cannot be both wiped and exempt").toEqual([]); + }); + + it("every model without a userId column is reached by cascade or declared instance-scoped", () => { + // Fixpoint over required cascade relations: a model is covered when the + // database removes it as a consequence of a row the wipe deletes. + const covered = new Set(wipeSet); + for (;;) { + let grew = false; + for (const [name, shape] of models) { + if (covered.has(name)) continue; + const reached = shape.relations.some( + (r) => r.cascade && !r.optional && covered.has(r.target), + ); + if (reached) { + covered.add(name); + grew = true; + } + } + if (!grew) break; + } + + const orphans = [...models] + .filter(([name, shape]) => !shape.scalars.includes("userId")) + .map(([name]) => name) + .filter( + (name) => + !covered.has(name) && + !(name in INSTANCE_SCOPED) && + !(name in WIPE_EXEMPT), + ); + + if (orphans.length > 0) { + throw new Error( + `${orphans.length} model(s) carry no userId, are not removed by any cascade from a wiped row, and are not declared instance-scoped:\n` + + orphans.map((n) => ` ❌ ${n}`).join("\n") + + "\n\nEither give the parent relation onDelete: Cascade, add the model " + + "to WIPE_MODELS, or add it to INSTANCE_SCOPED with the reason it " + + "belongs to the instance rather than to an account.", + ); + } + }); + + it("every User column is either reset or kept with a reason", () => { + const columns = models.get("User")?.scalars ?? []; + expect(columns.length).toBeGreaterThan(80); + + const reset = new Set(Object.keys(USER_RESET)); + const kept = new Set(Object.keys(USER_KEPT_FIELDS)); + + const unclassified = columns.filter((c) => !reset.has(c) && !kept.has(c)); + if (unclassified.length > 0) { + throw new Error( + `${unclassified.length} User column(s) are in neither USER_RESET nor USER_KEPT_FIELDS:\n` + + unclassified.map((c) => ` ❌ ${c}`).join("\n") + + "\n\nA column on the account row survives the wipe by default. Reset " + + "it, or record why keeping it is not keeping the person's record.", + ); + } + + const bothWays = columns.filter((c) => reset.has(c) && kept.has(c)); + expect(bothWays, "a column cannot be both reset and kept").toEqual([]); + }); + + it("carries no stale classification", () => { + for (const name of WIPE_MODELS) { + expect( + models.has(name), + `WIPE_MODELS names "${name}", which is not a model`, + ).toBe(true); + } + for (const [name, reason] of Object.entries(WIPE_EXEMPT)) { + expect( + models.has(name), + `WIPE_EXEMPT names "${name}", which is not a model`, + ).toBe(true); + expect( + reason.length, + `WIPE_EXEMPT.${name} needs a real reason`, + ).toBeGreaterThan(30); + } + for (const [name, reason] of Object.entries(INSTANCE_SCOPED)) { + expect( + models.has(name), + `INSTANCE_SCOPED names "${name}", which is not a model`, + ).toBe(true); + expect( + reason.length, + `INSTANCE_SCOPED.${name} needs a real reason`, + ).toBeGreaterThan(20); + } + const columns = new Set(models.get("User")?.scalars ?? []); + for (const [column, reason] of Object.entries(USER_KEPT_FIELDS)) { + expect( + columns.has(column), + `USER_KEPT_FIELDS names "${column}", which is no longer a User column`, + ).toBe(true); + expect( + reason.length, + `USER_KEPT_FIELDS.${column} needs a real reason`, + ).toBeGreaterThan(20); + } + expect(new Set(WIPE_MODELS).size, "WIPE_MODELS repeats a model").toBe( + WIPE_MODELS.length, + ); + }); + + it("deletes children before their parents so the counts are truthful", () => { + const position = new Map(WIPE_MODELS.map((m, i) => [m as string, i])); + const violations: string[] = []; + for (const child of WIPE_MODELS) { + const shape = models.get(child); + if (!shape) continue; + for (const relation of shape.relations) { + if (!relation.cascade) continue; + if (relation.target === child) continue; + if (!position.has(relation.target)) continue; + if (position.get(child)! > position.get(relation.target)!) { + violations.push( + `${child} (#${position.get(child)}) cascades from ${relation.target} (#${position.get(relation.target)})`, + ); + } + } + } + expect( + violations, + `WIPE_MODELS deletes a parent before its child, so the child's count reports 0 for rows that existed:\n${violations.join("\n")}`, + ).toEqual([]); + }); + + it("maps every wiped model to a plausible delegate key", () => { + for (const model of WIPE_MODELS) { + const key = wipeDelegateKey(model); + expect(key[0]).toBe(model[0].toLowerCase()); + expect(key.slice(1)).toBe(model.slice(1)); + } + }); +}); diff --git a/src/app/api/allergies/[id]/route.ts b/src/app/api/allergies/[id]/route.ts index af618247e..10ee3ccb2 100644 --- a/src/app/api/allergies/[id]/route.ts +++ b/src/app/api/allergies/[id]/route.ts @@ -117,23 +117,25 @@ export const DELETE = apiHandler( const { id } = await params; const existing = await prisma.allergy.findUnique({ where: { id }, - select: { id: true, userId: true, deletedAt: true }, + select: { id: true, userId: true }, }); if (!existing || existing.userId !== user.id) { return apiError("Allergy not found", 404); } - if (existing.deletedAt === null) { - await prisma.allergy.update({ - where: { id }, - data: { deletedAt: new Date() }, - }); - await auditLog("allergy.delete", { - userId: user.id, - ipAddress: getClientIp(request), - details: { allergyId: id }, - }); - } + // Hard delete. The confirmation says it can't be undone, and nothing here + // needs a tombstone: no restore route reaches an allergy, no other table + // references one, it is absent from the `/api/sync/changes` delta feed, + // and every read already filters `deletedAt: null`. A row already carrying + // a tombstone from before this changed is removed too, so the record + // converges on what the dialog promised. + await prisma.allergy.delete({ where: { id } }); + + await auditLog("allergy.delete", { + userId: user.id, + ipAddress: getClientIp(request), + details: { allergyId: id }, + }); annotate({ action: { name: "allergy.delete", entity_type: "allergy", entity_id: id }, diff --git a/src/app/api/measurement-reminders/[id]/route.ts b/src/app/api/measurement-reminders/[id]/route.ts index aad5e46e0..6f37d2fd6 100644 --- a/src/app/api/measurement-reminders/[id]/route.ts +++ b/src/app/api/measurement-reminders/[id]/route.ts @@ -206,12 +206,13 @@ export const DELETE = apiHandler( return apiError("Measurement reminder not found", 404); } - // Soft-delete (tombstone) parity with the rest of the tree. A - // re-delete of an already-tombstoned row is idempotent. - await prisma.measurementReminder.update({ - where: { id }, - data: { deletedAt: new Date() }, - }); + // Hard delete. The confirmation says "permanently deleted", and this row + // has nothing that a tombstone would serve: it is not in the + // `/api/sync/changes` delta feed, no restore route reaches it, no other + // table references it, and every read already filters `deletedAt: null`. + // The tombstone bought parity with tables that need one and cost the + // dialog its accuracy, so the row goes. + await prisma.measurementReminder.delete({ where: { id } }); await auditLog("measurementReminder.delete", { userId: user.id, diff --git a/src/app/api/settings/data/route.ts b/src/app/api/settings/data/route.ts index beb67d1fc..38a744be4 100644 --- a/src/app/api/settings/data/route.ts +++ b/src/app/api/settings/data/route.ts @@ -9,11 +9,37 @@ import { } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; import { invalidateUserData } from "@/lib/cache/invalidate"; +import { + resolveWipeDelegate, + USER_RESET, + wipeDelegateKey, + WIPE_MODELS, +} from "@/lib/data-wipe/wipe-plan"; export const dynamic = "force-dynamic"; /** - * Delete all user-owned health/integration data while keeping the account. + * Budget for the wipe transaction. + * + * Prisma's default interactive-transaction timeout is 5 s, which was already + * optimistic for thirteen `deleteMany` statements and is not survivable for + * the full user-scoped set on an account with a large import behind it. The + * whole action is one transaction on purpose — a half-applied wipe is worse + * than none, because the person is told it succeeded either way. + */ +const WIPE_TRANSACTION_TIMEOUT_MS = 120_000; +const WIPE_TRANSACTION_MAX_WAIT_MS = 15_000; + +/** + * Delete every row this account owns, keeping the account and the credentials + * that sign into it. + * + * The set of tables is not written here. It is declared once in + * `@/lib/data-wipe/wipe-plan` and held against `prisma/schema.prisma` by + * `src/__tests__/data-wipe-completeness.test.ts`, so a model added to the + * schema cannot quietly land outside the wipe — which is exactly how this + * route came to delete thirteen tables out of eighty-odd while telling people + * it had deleted all of them. * * v1.23 — gated by a fresh step-up for MFA-enrolled accounts (see the account * deletion route); single-factor accounts keep the typed-confirmation-only @@ -40,92 +66,50 @@ export const DELETE = apiHandler(async (request: NextRequest) => { const userId = user.id; - const result = await prisma.$transaction(async (tx) => { - const measurements = await tx.measurement.deleteMany({ - where: { userId }, - }); - const intakeEvents = await tx.medicationIntakeEvent.deleteMany({ - where: { userId }, - }); - const medications = await tx.medication.deleteMany({ - where: { userId }, - }); - const moodEntries = await tx.moodEntry.deleteMany({ - where: { userId }, - }); - // v1.4.39 W-MOOD — wipe the persisted rollup partition for this - // user so the next analytics read returns an empty envelope. - await tx.moodEntryRollup.deleteMany({ where: { userId } }); - const apiTokens = await tx.apiToken.deleteMany({ - where: { userId }, - }); - const withingsConnections = await tx.withingsConnection.deleteMany({ - where: { userId }, - }); - const notificationChannels = await tx.notificationChannel.deleteMany({ - where: { userId }, - }); - const pushSubscriptions = await tx.pushSubscription.deleteMany({ - where: { userId }, - }); - const dataBackups = await tx.dataBackup.deleteMany({ - where: { userId }, - }); - const achievements = await tx.userAchievement.deleteMany({ - where: { userId }, - }); - const telegramDeletions = await tx.telegramScheduledDeletion.deleteMany({ - where: { userId }, - }); - const auditLogs = await tx.auditLog.deleteMany({ - where: { userId }, - }); + const counts = await prisma.$transaction( + async (tx) => { + const perModel: Record = {}; + for (const model of WIPE_MODELS) { + const { count } = await resolveWipeDelegate(tx, model).deleteMany({ + where: { userId }, + }); + if (count > 0) perModel[wipeDelegateKey(model)] = count; + } - await tx.user.update({ - where: { id: userId }, - data: { - heightCm: null, - dateOfBirth: null, - gender: null, - insightsPrivacyMode: "aggregated", - insightsCachedAt: null, - insightsCachedText: null, - telegramBotToken: null, - telegramChatId: null, - telegramEnabled: false, - withingsClientIdEncrypted: null, - withingsClientSecretEncrypted: null, - onboardingCompletedAt: null, - }, - }); + // The account row survives; the personal data carried on its own columns + // does not. Same classification contract as the model list. + await tx.user.update({ where: { id: userId }, data: USER_RESET }); - return { - measurements: measurements.count, - intakeEvents: intakeEvents.count, - medications: medications.count, - moodEntries: moodEntries.count, - apiTokens: apiTokens.count, - withingsConnections: withingsConnections.count, - notificationChannels: notificationChannels.count, - pushSubscriptions: pushSubscriptions.count, - dataBackups: dataBackups.count, - achievements: achievements.count, - telegramDeletions: telegramDeletions.count, - auditLogs: auditLogs.count, - }; - }); + return perModel; + }, + { + timeout: WIPE_TRANSACTION_TIMEOUT_MS, + maxWait: WIPE_TRANSACTION_MAX_WAIT_MS, + }, + ); + const deletedRows = Object.values(counts).reduce((sum, n) => sum + n, 0); + + // Written after the transaction commits, so the row cannot be removed by the + // AuditLog delete inside it. await auditLog("user.data.clear", { userId, ipAddress: getClientIp(request), - details: result, + details: { deletedRows, models: counts }, }); - annotate({ action: { name: "settings.data.clear" }, meta: result }); + annotate({ + action: { name: "settings.data.clear" }, + meta: { + deleted_rows: deletedRows, + models_wiped: WIPE_MODELS.length, + models_with_rows: Object.keys(counts).length, + }, + }); // v1.16.9 — every cached payload was built on the rows just deleted; // hard-evict the user's buckets so no surface serves pre-wipe data. invalidateUserData(userId); - return apiSuccess({ cleared: true, ...result }); + return apiSuccess({ cleared: true, deletedRows, models: counts }); }); diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts new file mode 100644 index 000000000..f14986763 --- /dev/null +++ b/src/lib/data-wipe/wipe-plan.ts @@ -0,0 +1,403 @@ +/** + * The one declaration of what "Delete All Data" means. + * + * The route this feeds used to carry its delete calls inline. That list was + * correct on the day it was written and then stopped being correct, silently, + * every time a model was added to `prisma/schema.prisma` — a new table is + * user-scoped from birth and outside the wipe by default, and nothing anywhere + * noticed. Sixty-odd tables accumulated on the wrong side of a confirmation + * dialog that says "permanently delete all your health data". + * + * So the list lives here, next to its own exemption list, and + * `src/__tests__/data-wipe-completeness.test.ts` enumerates the schema and + * fails when a model belongs to neither. Adding a model to the schema now + * forces a decision instead of granting a default. + * + * ── The boundary ─────────────────────────────────────────────────────────── + * + * This action erases everything the account owns EXCEPT what is needed to sign + * in again. Removing the sign-in credentials too would not be data deletion, + * it would be account deletion — and that has its own route + * (`src/app/api/settings/account/route.ts`, which deletes the `User` row and + * lets the schema's `onDelete: Cascade` do the rest). + * + * Everything in {@link WIPE_MODELS} is hard-deleted, including rows that were + * only soft-deleted before: `deleteMany({ where: { userId } })` carries no + * `deletedAt` filter, so a tombstone goes with the record. + * + * ── Why the list is ordered ──────────────────────────────────────────────── + * + * Correctness does not depend on the order — every child of a wiped model + * cascades at the database level. The order exists so the per-model counts in + * the audit row are truthful: delete a parent first and its children are gone + * before their own `deleteMany` runs, which would report 0 for rows that did + * exist. Children therefore come before their parents, and the completeness + * test asserts that property rather than trusting the author. + */ +import { Prisma } from "@/generated/prisma/client"; + +/** + * Every user-scoped model this action deletes, children before parents. + * + * Names are Prisma model names; {@link wipeDelegateKey} maps each to its + * client delegate. Keeping model names (not delegate keys) is what lets the + * completeness test compare this list against `schema.prisma` literally. + */ +export const WIPE_MODELS = [ + // ── Measurements and their derived tiers ──────────────────────────────── + "Measurement", + // The rollup tier is a per-day, per-metric reconstruction of the same + // history. A wipe that leaves it behind is not a wipe. + "MeasurementRollup", + "MeasurementReminder", + + // ── Medication ────────────────────────────────────────────────────────── + "MedicationIntakeEvent", + "MedicationPauseEra", + "MedicationSideEffect", + "MedicationInventoryItem", + "MedicationIntakeImportJob", + "MedicationComplianceRollup", + + // ── Mood ──────────────────────────────────────────────────────────────── + "MoodEntry", + "MoodEntryRollup", + "MoodReminderDispatch", + "MoodTagHidden", + + // ── Clinical record ───────────────────────────────────────────────────── + "MentalHealthAssessment", + "LabResult", + "Biomarker", + "CustomMetricEntry", + "MenstrualCycle", + "CycleDayLog", + "CycleSymptom", + "CycleProfile", + "CyclePrediction", + "Allergy", + "FamilyHistoryEntry", + "IllnessDayLog", + "EcgRecording", + "NutrientIntakeDay", + + // ── Activity ──────────────────────────────────────────────────────────── + "WorkoutInsight", + "WorkoutInsightGenerationClaim", + "PersonalRecord", + "StrainTrimpCache", + + // ── Context and engagement ────────────────────────────────────────────── + "EnvironmentContext", + "EnvironmentTravelLocation", + "UserHealthProfile", + "UserAchievement", + "ArrivalReaction", + "DismissedPriorityItem", + "RecommendationFeedback", + + // ── Documents ─────────────────────────────────────────────────────────── + "DocumentContentIndex", + "DocumentThumbnail", + "DocumentConditionLink", + "ExtractedFact", + "ImportJob", + + // ── AI ────────────────────────────────────────────────────────────────── + "CoachConversation", + "CoachFact", + "CoachPlan", + "CoachReminder", + "CoachUsage", + "InsightNarrative", + "ProviderHealth", + + // ── Sharing ───────────────────────────────────────────────────────────── + // Deleting the row is the revocation: `resolveShareToken` / + // `resolveShareGateState` both resolve an unknown `tokenHash` to `null`, and + // null is the same blunt 404 a revoked link gets. + "ClinicianShareLink", + + // ── Integrations ──────────────────────────────────────────────────────── + "WithingsConnection", + "WithingsOAuthState", + "WhoopConnection", + "WhoopOAuthState", + "WhoopConnectTicket", + "FitbitConnection", + "FitbitOAuthState", + "GoogleHealthConnection", + "GoogleHealthOAuthState", + "McpOAuthConnection", + "IntegrationStatus", + + // ── Notification transports ───────────────────────────────────────────── + "NotificationChannel", + "PushSubscription", + "PushAttempt", + "Device", + "TelegramScheduledDeletion", + "TelegramPromptContext", + + // ── API surface and operational trails ────────────────────────────────── + "StepUpElevation", + // Cached response envelopes — the bodies are the health payloads the API + // just returned. + "IdempotencyKey", + "DataBackup", + "AuditLog", + "ConsentReceipt", + "UserKnownDevice", + + // ── Parents, last ─────────────────────────────────────────────────────── + "Medication", + "MoodTag", + "MoodTagCategory", + "CustomMetric", + "IllnessEpisode", + "Workout", + "InboundDocument", + "ApiToken", +] as const; + +export type WipeModel = (typeof WIPE_MODELS)[number]; + +/** + * User-scoped models this action deliberately keeps, each with the reason. + * + * The rule is narrow and stated in the confirmation copy: what the account + * needs to sign in again survives. Anything added here that is not a sign-in + * credential needs a reason that survives being read aloud to the person who + * typed the confirmation. + */ +export const WIPE_EXEMPT: Readonly> = { + Session: + "the signed-in session performing the wipe; deleting it signs the person out mid-action and the account is preserved", + RefreshToken: + "per-device sign-in continuity; deleting it signs every paired device out and is account deletion by another name", + Passkey: "a sign-in credential — without it the account may be unreachable", + WebauthnMfaCredential: + "a second-factor credential; removing it weakens the account rather than clearing data", + MfaRecoveryCode: + "the second-factor recovery path; removing it can lock the account out permanently", + MfaChallenge: + "an in-flight authentication challenge, seconds-lived and swept by its own expiry", + AuthChallenge: + "an in-flight WebAuthn challenge, seconds-lived and swept by its own expiry", + TrustedDevice: + "the remember-this-device grant that suppresses a second-factor prompt; a sign-in credential", + OidcNativeHandoff: + "an in-flight one-time sign-in code, minutes-lived and swept by its own expiry", + InviteRedemption: + "the instance's registration ledger, not the person's record; even account deletion only detaches it (onDelete: SetNull) rather than removing it", +}; + +/** + * Models that carry no `userId` and belong to the instance rather than to any + * account, so no wipe of a single account may touch them. + * + * Everything else without a `userId` reaches a wiped or exempt model through a + * chain of `onDelete: Cascade` relations and needs no entry here; the + * completeness test proves that reachability rather than assuming it. + */ +export const INSTANCE_SCOPED: Readonly> = { + AppSettings: "instance-wide configuration singleton", + RateLimit: "instance-wide rate-limit buckets, keyed by string, self-expiring", + HostMetric: "instance-wide host telemetry, not attributable to an account", + CycleSymptomCategory: + "seeded catalogue shared by every account; a user's custom symptoms live on CycleSymptom", + IllnessSymptom: "seeded symptom catalogue shared by every account", + InviteToken: + "the instance's invite ledger; the creator relation cascades on account deletion, and a data wipe keeps the account", + User: "the account itself — preserved by definition; its columns are handled by USER_RESET / USER_KEPT_FIELDS", +}; + +/** + * The `User` columns this action clears, and the value each is reset to. + * + * The same rot applies here: a column added to `User` is personal data from + * birth and outside the reset by default. The completeness test compares these + * keys against the model's scalar columns, so a new column has to be + * classified. + * + * `Prisma.DbNull` rather than `null` for the JSON columns — on a nullable JSON + * field `null` is the JSON value `null`, not SQL NULL. + */ +export const USER_RESET = { + // Body and identity-of-the-record + heightCm: null, + dateOfBirth: null, + gender: null, + hasDiabetes: false, + fullName: null, + displayName: null, + insurerName: null, + insuranceNumberEncrypted: null, + insurerIkNumber: null, + lastReportPracticeName: null, + avatarBytes: null, + avatarContentType: null, + avatarUpdatedAt: null, + + // Where the person lives — an environment-context input, and a location + homeLat: null, + homeLon: null, + homeLabel: null, + homeTimezone: null, + homeSince: null, + + // Derived AI output cached on the row + insightsPrivacyMode: "aggregated", + insightsCachedAt: null, + insightsCachedText: null, + insightsSnapshotHash: null, + insightsWarmFailedAt: null, + insightsBriefingRerollDate: null, + morningDigestRefreshedOn: null, + insightsExcludeMetrics: [], + insightsLayoutJson: Prisma.DbNull, + + // AI provider configuration and keys + aiProvider: null, + aiModel: null, + aiBaseUrl: null, + aiAnthropicKeyEncrypted: null, + aiLocalKeyEncrypted: null, + aiOpenaiKeyEncrypted: null, + aiProviderChain: Prisma.DbNull, + aiResponseTimeoutSeconds: null, + disableCoach: false, + coachPrefsJson: Prisma.DbNull, + coachLastSeenAt: null, + documentsAutoAiRead: false, + labsLocalOcrEnabled: false, + codexAccessTokenEncrypted: null, + codexRefreshTokenEncrypted: null, + codexTokenExpiresAt: null, + codexConnectedAt: null, + codexConnectionStatus: "disconnected", + useCentralCodex: false, + + // Integration credentials and per-integration state + telegramBotToken: null, + telegramChatId: null, + telegramEnabled: false, + withingsClientIdEncrypted: null, + withingsClientSecretEncrypted: null, + whoopClientIdEncrypted: null, + whoopClientSecretEncrypted: null, + fitbitClientIdEncrypted: null, + fitbitClientSecretEncrypted: null, + googleHealthClientIdEncrypted: null, + googleHealthClientSecretEncrypted: null, + nightscoutUrlEncrypted: null, + nightscoutTokenEncrypted: null, + nightscoutAllowPrivateHost: false, + polarAccessTokenEncrypted: null, + polarUserIdEncrypted: null, + polarClientIdEncrypted: null, + polarClientSecretEncrypted: null, + ouraAccessTokenEncrypted: null, + ouraRefreshTokenEncrypted: null, + ouraClientIdEncrypted: null, + ouraClientSecretEncrypted: null, + stravaClientIdEncrypted: null, + stravaClientSecretEncrypted: null, + stravaAccessTokenEncrypted: null, + stravaRefreshTokenEncrypted: null, + stravaAthleteId: null, + stravaLastActivityAt: null, + stravaBackfillCompletedAt: null, + healthKitConfigJson: Prisma.DbNull, + healthKitLastSyncedAt: null, + lastSyncedAt: null, + sourcePriorityJson: Prisma.DbNull, + + // Choices that describe the record rather than the interface + thresholdsJson: Prisma.DbNull, + dashboardWidgetsJson: Prisma.DbNull, + medicationListLayoutJson: Prisma.DbNull, + moodTagLayoutJson: Prisma.DbNull, + reportSelectionJson: Prisma.DbNull, + modulePreferencesJson: Prisma.DbNull, + globalExcludedInjectionSites: [], + moodReminderEnabled: false, + notificationPrefs: Prisma.DbNull, + researchModeEnabled: false, + researchModeAcknowledgedAt: null, + researchModeAcknowledgedVersion: null, + + // Back to the start of the tour — the record it was built around is gone + onboardingCompletedAt: null, + onboardingStep: 0, + onboardingGoals: [], + onboardingTourCompleted: false, + onboardingTourProgressJson: Prisma.DbNull, +} satisfies Prisma.UserUpdateInput; + +/** + * `User` columns the wipe leaves alone, each with the reason. Same contract as + * {@link WIPE_EXEMPT}: sign-in, identity, and how the interface is rendered. + */ +export const USER_KEPT_FIELDS: Readonly> = { + id: "the account's primary key", + username: "the account's sign-in identity", + email: "the account's sign-in identity and recovery address", + passwordHash: "a sign-in credential — the account password", + role: "the account's authorisation level, set by the operator", + oidcIssuer: "a sign-in credential — the external identity binding", + oidcSub: "a sign-in credential — the external identity binding", + createdAt: "when the account was created; the account is preserved", + updatedAt: "row bookkeeping maintained by Prisma", + totpSecretEncrypted: "a second-factor credential", + totpConfirmedAt: "second-factor enrolment state", + totpLastStep: + "second-factor replay guard; clearing it re-opens a replay window", + mfaEnforced: "the operator's per-account second-factor policy", + passkeyUpgradeNudgeDismissed: + "an interface nudge the person already dismissed", + timezone: "an interface preference — how dates and times are rendered", + locale: "an interface preference — which language the interface speaks", + unitPreference: "an interface preference — metric or imperial rendering", + timeFormat: "an interface preference — 12- or 24-hour rendering", + dateFormat: "an interface preference — date-order rendering", + glucoseUnit: "an interface preference — mg/dL or mmol/L rendering", + disclaimerAcknowledgedAt: + "the account's acknowledgement of the medical disclaimer, which is about the app and not about the record", + disclaimerAcknowledgedVersion: + "the version of the medical disclaimer the account acknowledged", + documentQuotaBytes: + "a per-account storage quota set by the operator, not by the person", +}; + +/** Prisma client delegate key for a model name (`MoodEntry` → `moodEntry`). */ +export function wipeDelegateKey(model: string): string { + return model.charAt(0).toLowerCase() + model.slice(1); +} + +/** The narrow slice of a Prisma delegate the wipe loop uses. */ +export interface UserScopedDeleteDelegate { + deleteMany(args: { where: { userId: string } }): Promise<{ count: number }>; +} + +/** + * Resolve a model name to its delegate on a client or transaction handle. + * + * One documented cast: `WIPE_MODELS` is a literal list of model names checked + * against `schema.prisma` by the completeness test, and every entry indexes a + * delegate that exposes `deleteMany` over a `userId`. The test also asserts + * every entry resolves, so an unresolvable name fails the build rather than + * the request. + */ +export function resolveWipeDelegate( + client: object, + model: string, +): UserScopedDeleteDelegate { + const delegate = (client as Record)[ + wipeDelegateKey(model) + ]; + if (!delegate || typeof delegate.deleteMany !== "function") { + throw new Error(`No Prisma delegate for wipe model "${model}"`); + } + return delegate; +} From 5e66ebaa54fe4bbe7603a34585972b5bb4b71063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 01:52:27 +0200 Subject: [PATCH 02/52] fix(jobs): let the retention purges finish, and let them fail loudly The nightly tombstone purge asked for `deleted_at IS NOT NULL AND deleted_at < cutoff` and the audit-log purge asked for `created_at < cutoff`. Neither predicate had an index behind it: every deleted_at index in the schema is partial on IS NULL, the live-rows predicate, and every audit_logs index leads with user_id or action. Both purges planned a sequential scan and then paid index maintenance per removed row, in one unbounded statement, on a connection carrying statement_timeout=60000. Past a certain backlog the statement aborts, the transaction rolls back, nothing is purged, and the next night runs the identical statement against the same rows. The mechanism that gives deletion a horizon could not close on the instances that needed it. Migration 0277 adds the indexes the predicates ask for; the three tombstone ones are partial on IS NOT NULL, so on a healthy instance they cover only the tombstones and shrink as the purge drains. The purges now walk the backlog in batches of 5 000 by primary key, capped at 200 000 rows a run. Both handlers used to swallow every error into a wide-event warning and return, so pg-boss recorded a successful job either way: a purge failing every night looked exactly like a purge with nothing to do. They now rethrow, and a run that stops at the cap records that a backlog remains. --- .../migration.sql | 46 +++++++ prisma/schema.prisma | 26 ++++ .../jobs/__tests__/audit-log-cleanup.test.ts | 49 ++++++- .../measurement-tombstone-cleanup.test.ts | 46 ++++++- src/lib/jobs/audit-log-cleanup.ts | 29 +++- src/lib/jobs/measurement-tombstone-cleanup.ts | 84 ++++++++--- src/lib/jobs/purge-batch.ts | 81 +++++++++++ .../cleanup-handlers-visibility.test.ts | 130 ++++++++++++++++++ src/lib/jobs/reminder/cleanup-handlers.ts | 49 +++++-- 9 files changed, 493 insertions(+), 47 deletions(-) create mode 100644 prisma/migrations/0277_retention_purge_indexes/migration.sql create mode 100644 src/lib/jobs/purge-batch.ts create mode 100644 src/lib/jobs/reminder/__tests__/cleanup-handlers-visibility.test.ts diff --git a/prisma/migrations/0277_retention_purge_indexes/migration.sql b/prisma/migrations/0277_retention_purge_indexes/migration.sql new file mode 100644 index 000000000..acff53d8d --- /dev/null +++ b/prisma/migrations/0277_retention_purge_indexes/migration.sql @@ -0,0 +1,46 @@ +-- v1.32.40 — indexes for the retention purges. +-- +-- The nightly tombstone purge asks for `deleted_at IS NOT NULL AND deleted_at +-- < cutoff`, and the audit-log purge asks for `created_at < cutoff`. Neither +-- predicate had an index: every `deleted_at` index in the schema is PARTIAL on +-- `deleted_at IS NULL` (the live-rows predicate, the exact opposite of what a +-- purge needs), and every `audit_logs` index leads with `user_id` or `action`. +-- Both purges therefore planned a sequential scan and paid index maintenance +-- on every removed row. Each connection carries `statement_timeout=60000` +-- (`src/lib/db.ts`), so past a certain backlog the statement aborts, the +-- transaction rolls back, nothing is purged, and the next night runs the same +-- statement against the same rows. The purge that was meant to give deletion a +-- horizon could not close on the instances that needed it most. +-- +-- The purges are batched as of the same release, so no single statement is on +-- the hook for a whole backlog; these indexes are what make each batch's +-- lookup cheap instead of a fresh sequential scan per batch. +-- +-- The three tombstone indexes are PARTIAL on `deleted_at IS NOT NULL`, which +-- is the shape that matters here: on a healthy instance the index covers only +-- the tombstones, so it stays tiny and shrinks as the purge drains. Prisma +-- cannot express a partial predicate, so these stay raw-SQL-only — the same +-- convention as migrations 0087 / 0108 / 0243, noted at the model in +-- `schema.prisma`. `audit_logs(created_at)` is expressible and is declared on +-- the model. +-- +-- Additive-only: every statement is `CREATE INDEX IF NOT EXISTS`, no table +-- rewrite, no data change. NON-CONCURRENT on purpose — Prisma Migrate wraps a +-- migration file in a transaction and `CREATE INDEX CONCURRENTLY` cannot run +-- in one; these build at boot, where docker-entrypoint runs `migrate deploy` +-- before the app serves traffic, so the build lock lands pre-traffic. + +CREATE INDEX IF NOT EXISTS "measurements_tombstone_purge_idx" + ON "measurements" ("deleted_at") + WHERE "deleted_at" IS NOT NULL; + +CREATE INDEX IF NOT EXISTS "mood_entries_tombstone_purge_idx" + ON "mood_entries" ("deleted_at") + WHERE "deleted_at" IS NOT NULL; + +CREATE INDEX IF NOT EXISTS "medication_intake_events_tombstone_purge_idx" + ON "medication_intake_events" ("deleted_at") + WHERE "deleted_at" IS NOT NULL; + +CREATE INDEX IF NOT EXISTS "audit_logs_created_at_idx" + ON "audit_logs" ("created_at"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 90ca68d2b..b625607b2 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1763,6 +1763,13 @@ model Measurement { // (v1.28.46 H4). It makes the note-encryption boot-discovery `SELECT DISTINCT // user_id` an index-only scan that converges to empty. Same partial-predicate // reason it is omitted here. + // `measurements_tombstone_purge_idx` is a partial index over `(deleted_at) + // WHERE deleted_at IS NOT NULL` — see migration 0277 (v1.32.40). Every + // other deleted_at index here is partial on IS NULL, which is the live-rows + // predicate and the opposite of what the nightly tombstone purge asks for; + // without this one the purge planned a sequential scan and could not finish + // inside statement_timeout. Raw SQL only (partial predicate not expressible + // in Prisma). @@map("measurements") } @@ -2854,6 +2861,13 @@ model MedicationIntakeEvent { // scanned the whole intake history. These two let both index-seek instead. @@index([userId, scheduledFor]) @@index([userId, medicationId, takenAt]) + // `medication_intake_events_tombstone_purge_idx` is a partial index over `(deleted_at) + // WHERE deleted_at IS NOT NULL` — see migration 0277 (v1.32.40). Every + // other deleted_at index here is partial on IS NULL, which is the live-rows + // predicate and the opposite of what the nightly tombstone purge asks for; + // without this one the purge planned a sequential scan and could not finish + // inside statement_timeout. Raw SQL only (partial predicate not expressible + // in Prisma). @@map("medication_intake_events") } @@ -3852,6 +3866,13 @@ model MoodEntry { // WHERE note IS NOT NULL AND note_encrypted IS NULL` — see migration 0243 // (v1.28.46 H4), serving the note-encryption boot-discovery scan. Raw SQL // only (partial predicate not expressible in Prisma). + // `mood_entries_tombstone_purge_idx` is a partial index over `(deleted_at) + // WHERE deleted_at IS NOT NULL` — see migration 0277 (v1.32.40). Every + // other deleted_at index here is partial on IS NULL, which is the live-rows + // predicate and the opposite of what the nightly tombstone purge asks for; + // without this one the purge planned a sequential scan and could not finish + // inside statement_timeout. Raw SQL only (partial predicate not expressible + // in Prisma). @@map("mood_entries") } @@ -4277,6 +4298,11 @@ model AuditLog { // this three-column index serves both the findFirst and the take:5 as // an index range scan over the highest-churn table. See migration 0123. @@index([userId, action, createdAt(sort: Desc)]) + // v1.32.40 — the retention purge filters on `createdAt` alone. Every index + // above leads with `userId` or `action`, so the purge planned a sequential + // scan and could not finish inside statement_timeout on an instance with + // enough history. See migration 0277. + @@index([createdAt]) @@map("audit_logs") } diff --git a/src/lib/jobs/__tests__/audit-log-cleanup.test.ts b/src/lib/jobs/__tests__/audit-log-cleanup.test.ts index c7b431543..25e2eb1a3 100644 --- a/src/lib/jobs/__tests__/audit-log-cleanup.test.ts +++ b/src/lib/jobs/__tests__/audit-log-cleanup.test.ts @@ -1,14 +1,28 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { + describe, + it, + expect, + vi, + beforeEach, + afterEach, + type Mock, +} from "vitest"; import type { PrismaClient } from "@/generated/prisma/client"; import { cleanupOldAuditLogs, getAuditLogRetentionDays, DEFAULT_AUDIT_LOG_RETENTION_DAYS, } from "../audit-log-cleanup"; +import { PURGE_BATCH_SIZE, PURGE_MAX_BATCHES } from "../purge-batch"; function makePrismaMock(deletedCount: number) { return { auditLog: { + findMany: vi + .fn() + .mockResolvedValue( + Array.from({ length: deletedCount }, (_, i) => ({ id: `row-${i}` })), + ), deleteMany: vi.fn().mockResolvedValue({ count: deletedCount }), }, } as unknown as PrismaClient; @@ -56,14 +70,23 @@ describe("cleanupOldAuditLogs", () => { it("deletes audit log rows older than 365 days by default", async () => { const prisma = makePrismaMock(42); const now = new Date("2026-05-04T00:00:00Z"); - const deleted = await cleanupOldAuditLogs(prisma, now); + const outcome = await cleanupOldAuditLogs(prisma, now); - expect(deleted).toBe(42); - expect(prisma.auditLog.deleteMany).toHaveBeenCalledTimes(1); + expect(outcome).toEqual({ deleted: 42, drained: true }); + // The predicate is applied on the bounded lookup; the delete itself is by + // primary key so no statement is on the hook for the whole backlog. + expect(prisma.auditLog.findMany).toHaveBeenCalledTimes(1); const cutoff = new Date(now.getTime() - 365 * 86_400_000); - expect(prisma.auditLog.deleteMany).toHaveBeenCalledWith({ + expect(prisma.auditLog.findMany).toHaveBeenCalledWith({ where: { createdAt: { lt: cutoff } }, + select: { id: true }, + take: PURGE_BATCH_SIZE, }); + expect(prisma.auditLog.deleteMany).toHaveBeenCalledTimes(1); + expect( + (prisma.auditLog.deleteMany as unknown as Mock).mock.calls[0][0].where.id + .in, + ).toHaveLength(42); }); it("uses the configured retention", async () => { @@ -73,8 +96,22 @@ describe("cleanupOldAuditLogs", () => { await cleanupOldAuditLogs(prisma, now); const cutoff = new Date(now.getTime() - 90 * 86_400_000); - expect(prisma.auditLog.deleteMany).toHaveBeenCalledWith({ + expect(prisma.auditLog.findMany).toHaveBeenCalledWith({ where: { createdAt: { lt: cutoff } }, + select: { id: true }, + take: PURGE_BATCH_SIZE, }); }); + + it("stops at the batch cap and says the backlog is not drained", async () => { + const prisma = makePrismaMock(PURGE_BATCH_SIZE); + const outcome = await cleanupOldAuditLogs( + prisma, + new Date("2026-05-04T00:00:00Z"), + ); + + expect(prisma.auditLog.findMany).toHaveBeenCalledTimes(PURGE_MAX_BATCHES); + expect(outcome.drained).toBe(false); + expect(outcome.deleted).toBe(PURGE_BATCH_SIZE * PURGE_MAX_BATCHES); + }); }); diff --git a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts index 5733f252c..ed96efe98 100644 --- a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts +++ b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts @@ -16,6 +16,7 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { cleanupExpiredMeasurementTombstones } from "../measurement-tombstone-cleanup"; +import { PURGE_BATCH_SIZE, PURGE_MAX_BATCHES } from "../purge-batch"; import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client"; const DAY_MS = 86_400_000; @@ -23,16 +24,19 @@ const DAY_MS = 86_400_000; describe("cleanupExpiredMeasurementTombstones", () => { it("prunes only tombstones older than the retention horizon", async () => { const now = new Date("2026-06-01T00:00:00.000Z"); + const findMany = vi + .fn() + .mockResolvedValueOnce([{ id: "a" }, { id: "b" }, { id: "c" }]); const deleteMany = vi.fn().mockResolvedValue({ count: 3 }); const prisma = { - measurement: { deleteMany }, + measurement: { findMany, deleteMany }, } as unknown as Parameters[0]; - const pruned = await cleanupExpiredMeasurementTombstones(prisma, now); - expect(pruned).toBe(3); + const outcome = await cleanupExpiredMeasurementTombstones(prisma, now); + expect(outcome).toEqual({ deleted: 3, drained: true }); - expect(deleteMany).toHaveBeenCalledTimes(1); - const where = deleteMany.mock.calls[0][0].where as { + expect(findMany).toHaveBeenCalledTimes(1); + const where = findMany.mock.calls[0][0].where as { deletedAt: { not: null; lt: Date }; }; // Only soft-deleted rows are eligible … @@ -42,6 +46,38 @@ describe("cleanupExpiredMeasurementTombstones", () => { now.getTime() - TOMBSTONE_RETENTION_DAYS * DAY_MS, ); expect(where.deletedAt.lt.getTime()).toBe(expectedCutoff.getTime()); + + // The delete is by primary key, so no single statement is on the hook for + // the whole backlog. + expect(deleteMany).toHaveBeenCalledTimes(1); + expect(deleteMany.mock.calls[0][0].where).toEqual({ + id: { in: ["a", "b", "c"] }, + }); + }); + + it("keeps walking a backlog that exceeds one batch, and reports when it stops short", async () => { + const now = new Date("2026-06-01T00:00:00.000Z"); + // Always a full batch back — the predicate never exhausts, so the run has + // to stop at the cap rather than loop forever. + const fullBatch = Array.from({ length: PURGE_BATCH_SIZE }, (_, i) => ({ + id: `row-${i}`, + })); + const findMany = vi.fn().mockResolvedValue(fullBatch); + const deleteMany = vi.fn().mockResolvedValue({ count: PURGE_BATCH_SIZE }); + const prisma = { + measurement: { findMany, deleteMany }, + } as unknown as Parameters[0]; + + const outcome = await cleanupExpiredMeasurementTombstones(prisma, now); + + expect(findMany).toHaveBeenCalledTimes(PURGE_MAX_BATCHES); + expect(outcome.deleted).toBe(PURGE_BATCH_SIZE * PURGE_MAX_BATCHES); + expect(outcome.drained).toBe(false); + // Every lookup is bounded — an unbounded one is the shape that could not + // finish inside statement_timeout. + for (const call of findMany.mock.calls) { + expect(call[0].take).toBe(PURGE_BATCH_SIZE); + } }); }); diff --git a/src/lib/jobs/audit-log-cleanup.ts b/src/lib/jobs/audit-log-cleanup.ts index 5fb51179c..ddb890b24 100644 --- a/src/lib/jobs/audit-log-cleanup.ts +++ b/src/lib/jobs/audit-log-cleanup.ts @@ -11,6 +11,7 @@ * `deleteMany`; runs daily via pg-boss. */ import type { PrismaClient } from "@/generated/prisma/client"; +import { purgeInBatches, type PurgeOutcome } from "@/lib/jobs/purge-batch"; export const DEFAULT_AUDIT_LOG_RETENTION_DAYS = 365; @@ -27,14 +28,34 @@ export function getAuditLogRetentionDays(): number { return parsed; } +/** + * Delete audit rows past the retention horizon, in bounded batches. + * + * Until v1.32.40 this was one unbounded `deleteMany` on `created_at < cutoff`. + * Every index on the table leads with `user_id` or `action`, so the predicate + * had none and the statement planned a sequential scan plus index maintenance + * on every removed row. Past the 60-second `statement_timeout` that aborts and + * rolls back, and the next night repeats it against the same rows — a + * retention window that can never close on an instance that has accumulated + * enough of them. Migration 0277 adds the `created_at` index; the batching + * keeps any single statement bounded regardless. + */ export async function cleanupOldAuditLogs( prisma: PrismaClient, now: Date = new Date(), -): Promise { +): Promise { const days = getAuditLogRetentionDays(); const cutoff = new Date(now.getTime() - days * 86_400_000); - const { count } = await prisma.auditLog.deleteMany({ - where: { createdAt: { lt: cutoff } }, + return purgeInBatches({ + findIds: async (take) => + ( + await prisma.auditLog.findMany({ + where: { createdAt: { lt: cutoff } }, + select: { id: true }, + take, + }) + ).map((row) => row.id), + deleteIds: async (ids) => + (await prisma.auditLog.deleteMany({ where: { id: { in: ids } } })).count, }); - return count; } diff --git a/src/lib/jobs/measurement-tombstone-cleanup.ts b/src/lib/jobs/measurement-tombstone-cleanup.ts index 681b3e2f6..d627059d9 100644 --- a/src/lib/jobs/measurement-tombstone-cleanup.ts +++ b/src/lib/jobs/measurement-tombstone-cleanup.ts @@ -20,50 +20,90 @@ */ import type { PrismaClient } from "@/generated/prisma/client"; import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client"; +import { purgeInBatches, type PurgeOutcome } from "@/lib/jobs/purge-batch"; const DAY_MS = 86_400_000; +function tombstoneCutoff(now: Date): Date { + return new Date(now.getTime() - TOMBSTONE_RETENTION_DAYS * DAY_MS); +} + /** - * Hard-delete soft-deleted measurement rows whose `deletedAt` is older - * than the retention horizon. Returns the number of rows pruned. + * Hard-delete soft-deleted measurement rows whose `deletedAt` is older than + * the retention horizon. + * + * Batched. Until v1.32.40 this was one unbounded `deleteMany` over a predicate + * that no index supported, which meant a backlog large enough to exceed the + * 60-second `statement_timeout` could never be drained: the statement aborted, + * the transaction rolled back, and the next night ran the identical statement + * against the same rows. `measurements` is the densest table in the schema and + * a bulk source delete can tombstone a six-figure row count in one action, so + * this is the one most likely to have been stuck. Migration 0277 adds the + * partial index the predicate needs. */ export async function cleanupExpiredMeasurementTombstones( prisma: PrismaClient, now: Date = new Date(), -): Promise { - const cutoff = new Date(now.getTime() - TOMBSTONE_RETENTION_DAYS * DAY_MS); - const { count } = await prisma.measurement.deleteMany({ - where: { deletedAt: { not: null, lt: cutoff } }, +): Promise { + const cutoff = tombstoneCutoff(now); + return purgeInBatches({ + findIds: async (take) => + ( + await prisma.measurement.findMany({ + where: { deletedAt: { not: null, lt: cutoff } }, + select: { id: true }, + take, + }) + ).map((row) => row.id), + deleteIds: async (ids) => + (await prisma.measurement.deleteMany({ where: { id: { in: ids } } })) + .count, }); - return count; } -/** - * Hard-delete soft-deleted mood-entry rows past the retention horizon. - * Returns the number of rows pruned. - */ +/** Hard-delete soft-deleted mood-entry rows past the retention horizon. */ export async function cleanupExpiredMoodTombstones( prisma: PrismaClient, now: Date = new Date(), -): Promise { - const cutoff = new Date(now.getTime() - TOMBSTONE_RETENTION_DAYS * DAY_MS); - const { count } = await prisma.moodEntry.deleteMany({ - where: { deletedAt: { not: null, lt: cutoff } }, +): Promise { + const cutoff = tombstoneCutoff(now); + return purgeInBatches({ + findIds: async (take) => + ( + await prisma.moodEntry.findMany({ + where: { deletedAt: { not: null, lt: cutoff } }, + select: { id: true }, + take, + }) + ).map((row) => row.id), + deleteIds: async (ids) => + (await prisma.moodEntry.deleteMany({ where: { id: { in: ids } } })).count, }); - return count; } /** * Hard-delete soft-deleted medication-intake-event rows past the retention - * horizon. Returns the number of rows pruned. + * horizon. */ export async function cleanupExpiredIntakeTombstones( prisma: PrismaClient, now: Date = new Date(), -): Promise { - const cutoff = new Date(now.getTime() - TOMBSTONE_RETENTION_DAYS * DAY_MS); - const { count } = await prisma.medicationIntakeEvent.deleteMany({ - where: { deletedAt: { not: null, lt: cutoff } }, +): Promise { + const cutoff = tombstoneCutoff(now); + return purgeInBatches({ + findIds: async (take) => + ( + await prisma.medicationIntakeEvent.findMany({ + where: { deletedAt: { not: null, lt: cutoff } }, + select: { id: true }, + take, + }) + ).map((row) => row.id), + deleteIds: async (ids) => + ( + await prisma.medicationIntakeEvent.deleteMany({ + where: { id: { in: ids } }, + }) + ).count, }); - return count; } diff --git a/src/lib/jobs/purge-batch.ts b/src/lib/jobs/purge-batch.ts new file mode 100644 index 000000000..8db2b5fac --- /dev/null +++ b/src/lib/jobs/purge-batch.ts @@ -0,0 +1,81 @@ +/** + * Batched retention purges. + * + * The retention jobs used to issue one unbounded `deleteMany` over a predicate + * no index supported. Every `deleted_at` index in the schema is PARTIAL on + * `IS NULL` — the live-rows predicate, the exact opposite of what a purge asks + * for — so the purge planned a sequential scan and then paid index maintenance + * on every row it removed. Below the 60-second `statement_timeout` the + * connection carries (`src/lib/db.ts`) that is invisible; above it the + * statement aborts, the transaction rolls back, nothing is purged, and the job + * runs the identical statement again the next night. A backlog that grows past + * the timeout can therefore never be drained by the mechanism meant to drain + * it, and the job that was supposed to make deletion real never completes. + * + * The fix has three parts and this file is one of them: + * - the predicates get indexes that match them (migration 0277); + * - the delete runs in bounded batches, so no single statement is on the + * hook for the whole backlog; + * - a run that stops at the cap says so, so "still draining" and "nothing + * left to do" stop looking identical from the outside. + */ + +/** Rows removed per statement. */ +export const PURGE_BATCH_SIZE = 5_000; + +/** + * Batches per run. 40 × 5 000 = 200 000 rows a night, which drains a large + * backlog over a handful of nights instead of asking one statement to survive + * it. The cap exists so a nightly job cannot hold connections and bloat WAL + * for an unbounded stretch on a instance that has accumulated years of rows. + */ +export const PURGE_MAX_BATCHES = 40; + +export interface PurgeOutcome { + /** Rows removed this run. */ + deleted: number; + /** + * `false` when the run stopped at {@link PURGE_MAX_BATCHES} with rows still + * matching. The next run continues; an operator watching this stay `false` + * for many nights is watching a backlog that needs a wider window. + */ + drained: boolean; +} + +export interface PurgeBatchOptions { + /** Ids of up to `take` rows matching the retention predicate. */ + findIds: (take: number) => Promise; + /** Remove exactly those ids; returns the row count actually removed. */ + deleteIds: (ids: string[]) => Promise; + batchSize?: number; + maxBatches?: number; +} + +/** + * Delete rows matching a retention predicate in bounded batches. + * + * Selecting ids first and deleting by primary key keeps each statement small + * and predictable regardless of how selective the predicate is, and keeps the + * scan on the index that supports the predicate rather than on the delete. + */ +export async function purgeInBatches({ + findIds, + deleteIds, + batchSize = PURGE_BATCH_SIZE, + maxBatches = PURGE_MAX_BATCHES, +}: PurgeBatchOptions): Promise { + let deleted = 0; + + for (let batch = 0; batch < maxBatches; batch++) { + const ids = await findIds(batchSize); + if (ids.length === 0) return { deleted, drained: true }; + + deleted += await deleteIds(ids); + + // A short final batch means the predicate is exhausted; asking again would + // cost a round-trip to learn nothing. + if (ids.length < batchSize) return { deleted, drained: true }; + } + + return { deleted, drained: false }; +} diff --git a/src/lib/jobs/reminder/__tests__/cleanup-handlers-visibility.test.ts b/src/lib/jobs/reminder/__tests__/cleanup-handlers-visibility.test.ts new file mode 100644 index 000000000..766971dab --- /dev/null +++ b/src/lib/jobs/reminder/__tests__/cleanup-handlers-visibility.test.ts @@ -0,0 +1,130 @@ +/** + * v1.32.40 — the retention purges have to be able to fail out loud. + * + * Both handlers used to catch every error into a wide-event warning and return + * normally. pg-boss then recorded a successful job, so a purge that aborted on + * `statement_timeout` every night for a year looked exactly like a purge with + * nothing to do. That is the failure this whole change exists to remove, so + * the propagation is pinned rather than left to convention. + * + * The second half pins the other half of the same problem: a run that stopped + * at the batch cap still has a backlog, and without the flag that is also + * indistinguishable from a clean run. + * + * Mutation check: drop the `throw err` from either handler and the matching + * "surfaces the failure" test goes red; drop the `drained` meta and the + * matching backlog test goes red. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const h = vi.hoisted(() => ({ + cleanupOldAuditLogs: vi.fn(), + cleanupExpiredMeasurementTombstones: vi.fn(), + cleanupExpiredMoodTombstones: vi.fn(), + cleanupExpiredIntakeTombstones: vi.fn(), + meta: {} as Record, + warnings: [] as string[], +})); + +vi.mock("@/lib/jobs/audit-log-cleanup", () => ({ + cleanupOldAuditLogs: h.cleanupOldAuditLogs, +})); +vi.mock("@/lib/jobs/measurement-tombstone-cleanup", () => ({ + cleanupExpiredMeasurementTombstones: h.cleanupExpiredMeasurementTombstones, + cleanupExpiredMoodTombstones: h.cleanupExpiredMoodTombstones, + cleanupExpiredIntakeTombstones: h.cleanupExpiredIntakeTombstones, +})); +vi.mock("../shared", async (importOriginal) => ({ + ...(await importOriginal>()), + getWorkerPrisma: () => ({}), +})); +vi.mock("@/lib/logging/background", () => ({ + withBackgroundEvent: async ( + _name: string, + fn: (evt: { + addMeta: (k: string, v: unknown) => void; + addWarning: (w: string) => void; + }) => Promise, + ) => + fn({ + addMeta: (k, v) => { + h.meta[k] = v; + }, + addWarning: (w) => { + h.warnings.push(w); + }, + }), +})); + +import { + handleAuditLogCleanup, + handleMeasurementTombstoneCleanup, +} from "../cleanup-handlers"; + +const DRAINED = { deleted: 1, drained: true }; + +beforeEach(() => { + vi.clearAllMocks(); + for (const key of Object.keys(h.meta)) delete h.meta[key]; + h.warnings.length = 0; + h.cleanupOldAuditLogs.mockResolvedValue(DRAINED); + h.cleanupExpiredMeasurementTombstones.mockResolvedValue(DRAINED); + h.cleanupExpiredMoodTombstones.mockResolvedValue(DRAINED); + h.cleanupExpiredIntakeTombstones.mockResolvedValue(DRAINED); +}); + +describe("audit-log cleanup handler", () => { + it("surfaces the failure instead of recording a successful job", async () => { + h.cleanupOldAuditLogs.mockRejectedValue(new Error("statement timeout")); + await expect(handleAuditLogCleanup([])).rejects.toThrow( + "statement timeout", + ); + expect(h.warnings.join(" ")).toContain("audit-log-cleanup failed"); + }); + + it("reports a backlog left behind by the batch cap", async () => { + h.cleanupOldAuditLogs.mockResolvedValue({ + deleted: 200_000, + drained: false, + }); + await handleAuditLogCleanup([]); + expect(h.meta.audit_log_cleanup_deleted).toBe(200_000); + expect(h.meta.audit_log_cleanup_drained).toBe(false); + expect(h.warnings.join(" ")).toContain("backlog remains"); + }); + + it("stays quiet when the backlog is drained", async () => { + await handleAuditLogCleanup([]); + expect(h.meta.audit_log_cleanup_drained).toBe(true); + expect(h.warnings).toEqual([]); + }); +}); + +describe("tombstone cleanup handler", () => { + it("surfaces the failure instead of recording a successful job", async () => { + h.cleanupExpiredMeasurementTombstones.mockRejectedValue( + new Error("statement timeout"), + ); + await expect(handleMeasurementTombstoneCleanup([])).rejects.toThrow( + "statement timeout", + ); + expect(h.warnings.join(" ")).toContain("tombstone-cleanup failed"); + }); + + it("reports a backlog when any of the three legs stopped short", async () => { + h.cleanupExpiredMoodTombstones.mockResolvedValue({ + deleted: 200_000, + drained: false, + }); + await handleMeasurementTombstoneCleanup([]); + expect(h.meta.tombstone_cleanup_drained).toBe(false); + expect(h.meta.mood_tombstone_cleanup_pruned).toBe(200_000); + expect(h.warnings.join(" ")).toContain("backlog remains"); + }); + + it("stays quiet when all three legs drained", async () => { + await handleMeasurementTombstoneCleanup([]); + expect(h.meta.tombstone_cleanup_drained).toBe(true); + expect(h.warnings).toEqual([]); + }); +}); diff --git a/src/lib/jobs/reminder/cleanup-handlers.ts b/src/lib/jobs/reminder/cleanup-handlers.ts index 4513975c1..a3b1e6f43 100644 --- a/src/lib/jobs/reminder/cleanup-handlers.ts +++ b/src/lib/jobs/reminder/cleanup-handlers.ts @@ -184,16 +184,36 @@ export async function handleMeasurementTombstoneCleanup( try { // v1.7.0 sync — prune tombstones across all three sync domains on // the same retention horizon. - const [measurements, mood, intakes] = await Promise.all([ - cleanupExpiredMeasurementTombstones(p), - cleanupExpiredMoodTombstones(p), - cleanupExpiredIntakeTombstones(p), - ]); - evt.addMeta("measurement_tombstone_cleanup_pruned", measurements); - evt.addMeta("mood_tombstone_cleanup_pruned", mood); - evt.addMeta("intake_tombstone_cleanup_pruned", intakes); + // + // v1.32.40 — sequential rather than parallel. Each leg now walks its + // backlog in batches, and three concurrent walks against the three + // densest tables competed for the worker pool for no gain: the job has + // the whole night. + const measurements = await cleanupExpiredMeasurementTombstones(p); + const mood = await cleanupExpiredMoodTombstones(p); + const intakes = await cleanupExpiredIntakeTombstones(p); + evt.addMeta( + "measurement_tombstone_cleanup_pruned", + measurements.deleted, + ); + evt.addMeta("mood_tombstone_cleanup_pruned", mood.deleted); + evt.addMeta("intake_tombstone_cleanup_pruned", intakes.deleted); + // A run that stopped at the batch cap has a backlog left. Without this + // flag a partial run and a complete one are indistinguishable from + // outside, which is how a purge that never finished stayed invisible. + const drained = measurements.drained && mood.drained && intakes.drained; + evt.addMeta("tombstone_cleanup_drained", drained); + if (!drained) { + evt.addWarning( + "tombstone-cleanup stopped at the batch cap; a backlog remains for the next run", + ); + } } catch (err) { + // Rethrow. A retention purge that fails silently every night looks + // exactly like one that has nothing to do; pg-boss recording a failed + // job is the only signal an operator can act on. evt.addWarning(`tombstone-cleanup failed: ${err}`); + throw err; } }, ); @@ -268,10 +288,19 @@ export async function handleAuditLogCleanup( await withBackgroundEvent("job.audit_log_cleanup", async (evt) => { const p = getWorkerPrisma(); try { - const deleted = await cleanupOldAuditLogs(p); - evt.addMeta("audit_log_cleanup_deleted", deleted); + const outcome = await cleanupOldAuditLogs(p); + evt.addMeta("audit_log_cleanup_deleted", outcome.deleted); + evt.addMeta("audit_log_cleanup_drained", outcome.drained); + if (!outcome.drained) { + evt.addWarning( + "audit-log-cleanup stopped at the batch cap; a backlog remains for the next run", + ); + } } catch (err) { + // Rethrow — see the tombstone handler. Storage-limitation retention that + // fails quietly is retention that is not happening. evt.addWarning(`audit-log-cleanup failed: ${err}`); + throw err; } }); } From dccd2f320baf090e46be3b8b4c8a77eb0a3e1a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 02:14:07 +0200 Subject: [PATCH 03/52] test(settings): prove the wipe against real Postgres, and state what deletion means The defect is about rows that survive, so a mocked client proves nothing. The new integration test seeds a row into every table in the schema, driven by information_schema and pg_constraint rather than a fixture, runs the wipe and asks the database what is left. A table added later is seeded and asserted without anyone remembering to. It holds all four halves: nothing in the wipe list survives, everything in the exemption list does, everything reached only by a database cascade is gone, and the account row comes back reset but able to sign in. A second file pins the allergy and checkup-reminder deletes actually removing their row. The reproductive-health note gains a Deletion section. It taught people to reason carefully about a host being imaged and never said what pressing delete does; now it says which of the three deletions removes the row and which only hides it. --- docs/self-hosting/reproductive-health-data.md | 25 + src/__tests__/data-wipe-completeness.test.ts | 2 +- src/app/api/allergies/[id]/route.ts | 4 +- .../api/measurement-reminders/[id]/route.ts | 5 +- src/app/api/settings/data/route.ts | 5 +- src/lib/data-wipe/wipe-plan.ts | 2 - tests/integration/settings-data-wipe.test.ts | 522 ++++++++++++++++++ .../soft-delete-copy-parity.test.ts | 120 ++++ 8 files changed, 678 insertions(+), 7 deletions(-) create mode 100644 tests/integration/settings-data-wipe.test.ts create mode 100644 tests/integration/soft-delete-copy-parity.test.ts diff --git a/docs/self-hosting/reproductive-health-data.md b/docs/self-hosting/reproductive-health-data.md index 051c5bdfd..557f96633 100644 --- a/docs/self-hosting/reproductive-health-data.md +++ b/docs/self-hosting/reproductive-health-data.md @@ -81,6 +81,31 @@ These are cited as factual public record to explain _why_ the "who holds the data" question is the one that matters for this category. HealthLog makes no claim about any other product's current practices. +## What deleting actually does + +Worth knowing before you rely on it, because "deleted" means two +different things depending on which button you press. + +- **Deleting one cycle or day-log entry** hides it. Every read filters + it out and it is gone from the app, from exports, and from the doctor + report. The row itself stays in PostgreSQL as a tombstone, which is + what lets a phone that was offline at delete time learn that the entry + went away. No job removes cycle tombstones on a schedule, so the row + stays until one of the two actions below. +- **Settings → Delete All Data** removes the rows outright, tombstones + included. Everything the account owns goes except what you sign in + with; the confirmation dialog lists it. +- **Deleting the account** removes the account row, and the database + cascades every record with it. + +Two consequences follow, and they matter for the seizure threat model +above. A host imaged while a per-entry deletion is the only deletion you +have performed still holds that entry. And a backup taken before any of +these actions still holds whatever it captured, because a backup is a +copy of a moment. If your reason for deleting is that the data must not +be recoverable from the machine, use one of the two whole-record actions +and rotate your backups. + ## Practical hardening If reproductive-health privacy is the reason you self-host, the diff --git a/src/__tests__/data-wipe-completeness.test.ts b/src/__tests__/data-wipe-completeness.test.ts index 98315046d..58f9cf752 100644 --- a/src/__tests__/data-wipe-completeness.test.ts +++ b/src/__tests__/data-wipe-completeness.test.ts @@ -169,7 +169,7 @@ describe("data-wipe completeness", () => { } const orphans = [...models] - .filter(([name, shape]) => !shape.scalars.includes("userId")) + .filter(([, shape]) => !shape.scalars.includes("userId")) .map(([name]) => name) .filter( (name) => diff --git a/src/app/api/allergies/[id]/route.ts b/src/app/api/allergies/[id]/route.ts index 10ee3ccb2..a2591bf40 100644 --- a/src/app/api/allergies/[id]/route.ts +++ b/src/app/api/allergies/[id]/route.ts @@ -1,7 +1,9 @@ /** * `GET /api/allergies/{id}` — one owned, live allergy record. * `PATCH /api/allergies/{id}` — edit it (partial; field-by-field). - * `DELETE /api/allergies/{id}` — soft-delete it (idempotent; `{ deleted: true }`). + * `DELETE /api/allergies/{id}` — delete it outright (`{ deleted: true }`; 404 + * once it is gone). The confirmation dialog promises it cannot be undone, and + * nothing here needs a tombstone to keep that promise honest. * * Owner-scoped. `userId` is narrowed from auth and fed to the Prisma `where`; * the body never carries it. The free-text `reaction` + `note` are encrypted diff --git a/src/app/api/measurement-reminders/[id]/route.ts b/src/app/api/measurement-reminders/[id]/route.ts index 6f37d2fd6..e591c7481 100644 --- a/src/app/api/measurement-reminders/[id]/route.ts +++ b/src/app/api/measurement-reminders/[id]/route.ts @@ -2,8 +2,9 @@ * v1.17.1 — Vorsorge (measurement) reminder by id: get + patch + delete. * * PATCH re-derives the server-authoritative `nextDueAt` after applying - * the (mutually-exclusive cadence) edit. DELETE soft-deletes (tombstone) - * for parity with the rest of the tree. + * the (mutually-exclusive cadence) edit. DELETE removes the row; the + * confirmation dialog says the reminder is permanently deleted, and nothing + * here needs a tombstone to keep that promise honest. */ import { NextRequest } from "next/server"; diff --git a/src/app/api/settings/data/route.ts b/src/app/api/settings/data/route.ts index 38a744be4..482fefb4a 100644 --- a/src/app/api/settings/data/route.ts +++ b/src/app/api/settings/data/route.ts @@ -91,7 +91,10 @@ export const DELETE = apiHandler(async (request: NextRequest) => { const deletedRows = Object.values(counts).reduce((sum, n) => sum + n, 0); // Written after the transaction commits, so the row cannot be removed by the - // AuditLog delete inside it. + // AuditLog delete inside it. The person's whole audit history goes; this one + // row stays as the receipt for the erasure they asked for. Account deletion + // takes the other branch and purges even this, because there is no longer an + // account for it to belong to. await auditLog("user.data.clear", { userId, ipAddress: getClientIp(request), diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index f14986763..7ad5d753b 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -160,8 +160,6 @@ export const WIPE_MODELS = [ "ApiToken", ] as const; -export type WipeModel = (typeof WIPE_MODELS)[number]; - /** * User-scoped models this action deliberately keeps, each with the reason. * diff --git a/tests/integration/settings-data-wipe.test.ts b/tests/integration/settings-data-wipe.test.ts new file mode 100644 index 000000000..1edbeef11 --- /dev/null +++ b/tests/integration/settings-data-wipe.test.ts @@ -0,0 +1,522 @@ +/** + * Integration guard for `DELETE /api/settings/data` — "Delete All Data". + * + * The whole defect is about rows that survive, so a mocked Prisma proves + * nothing: it can only show that the calls the route makes were made. This + * runs against real Postgres, seeds a row into every table in the schema, runs + * the wipe, and then asks the database what is left. + * + * The seeding is driven by `information_schema` and `pg_constraint` rather + * than a hand-written fixture, for the same reason the wipe list is derived + * rather than hand-written: a fixture written today covers the schema of + * today. A table added next year is seeded automatically and therefore + * asserted automatically. + * + * What it asserts: + * - every model in `WIPE_MODELS` has zero rows for the account afterwards; + * - every model in `WIPE_EXEMPT` still has its row, so the "kept" half of + * the confirmation copy is true too; + * - every table the wipe reaches only through a database cascade is empty; + * - the instance-scoped tables are untouched; + * - the `User` row survives with `USER_RESET` applied and + * `USER_KEPT_FIELDS` intact. + * + * Its honest limit: it proves the wipe removes what the plan declares. The + * plan being right is what `src/__tests__/data-wipe-completeness.test.ts` + * holds against `schema.prisma`. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import { + INSTANCE_SCOPED, + USER_KEPT_FIELDS, + USER_RESET, + WIPE_EXEMPT, + WIPE_MODELS, +} from "@/lib/data-wipe/wipe-plan"; +import type { PrismaClient } from "@/generated/prisma/client"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +// ── schema.prisma model → table name ──────────────────────────────────────── + +function modelTableMap(): Map { + const schema = readFileSync( + join(process.cwd(), "prisma", "schema.prisma"), + "utf8", + ); + const map = new Map(); + for (const block of schema.matchAll(/^model\s+(\w+)\s*\{([\s\S]*?)^\}/gm)) { + const mapped = /@@map\("([^"]+)"\)/.exec(block[2]); + map.set(block[1], mapped ? mapped[1] : block[1]); + } + return map; +} + +// ── generic seeder ────────────────────────────────────────────────────────── + +interface ColumnRow { + table_name: string; + column_name: string; + is_nullable: string; + column_default: string | null; + udt_name: string; +} + +interface ForeignKeyRow { + table_name: string; + column_name: string; + ref_table: string; + ref_column: string; + cascades: boolean; +} + +/** Tables the seeder never writes: Prisma's own ledger, and the account row. */ +const SEED_SKIP = new Set(["_prisma_migrations", "users"]); + +function synthesise(udt: string, enums: Map, tag: string) { + if (udt.startsWith("_")) return `'{}'::${udt.slice(1)}[]`; + switch (udt) { + case "text": + case "varchar": + case "bpchar": + return `'${tag}'`; + case "int2": + case "int4": + case "int8": + case "numeric": + case "float4": + case "float8": + return "1"; + case "bool": + return "false"; + case "timestamp": + case "timestamptz": + case "date": + return "now()"; + case "bytea": + return "'\\x00'::bytea"; + case "json": + case "jsonb": + return `'{}'::${udt}`; + case "uuid": + return "gen_random_uuid()"; + default: { + const label = enums.get(udt); + if (label) return `'${label}'::"${udt}"`; + throw new Error(`seeder has no value for column type "${udt}"`); + } + } +} + +/** + * Insert one row into every table in `public`, parents first, wiring every + * foreign key to the row already created for its target and every `user_id` to + * the account under test. + */ +async function seedEveryTable( + prisma: PrismaClient, + userId: string, +): Promise { + const columns = await prisma.$queryRaw` + SELECT table_name, column_name, is_nullable, column_default, udt_name + FROM information_schema.columns + WHERE table_schema = 'public'`; + + const foreignKeys = await prisma.$queryRaw` + SELECT con.conrelid::regclass::text AS table_name, + att.attname AS column_name, + con.confrelid::regclass::text AS ref_table, + ratt.attname AS ref_column, + con.confdeltype = 'c' AS cascades + FROM pg_constraint con + JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS ck(attnum, ord) ON true + JOIN LATERAL unnest(con.confkey) WITH ORDINALITY AS rk(attnum, ord) + ON rk.ord = ck.ord + JOIN pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = ck.attnum + JOIN pg_attribute ratt ON ratt.attrelid = con.confrelid AND ratt.attnum = rk.attnum + WHERE con.contype = 'f' AND con.connamespace = 'public'::regnamespace`; + + const enumRows = await prisma.$queryRaw< + Array<{ typname: string; enumlabel: string }> + >` + SELECT t.typname, e.enumlabel + FROM pg_type t + JOIN pg_enum e ON e.enumtypid = t.oid + ORDER BY t.typname, e.enumsortorder`; + const enums = new Map(); + for (const row of enumRows) { + if (!enums.has(row.typname)) enums.set(row.typname, row.enumlabel); + } + + const byTable = new Map(); + for (const column of columns) { + if (SEED_SKIP.has(column.table_name)) continue; + const bucket = byTable.get(column.table_name) ?? []; + bucket.push(column); + byTable.set(column.table_name, bucket); + } + + const fkByTable = new Map(); + for (const fk of foreignKeys) { + const bucket = fkByTable.get(fk.table_name) ?? []; + bucket.push(fk); + fkByTable.set(fk.table_name, bucket); + } + + // Parents before children. A self-reference or an optional cycle is skipped + // by only ordering on the columns the insert actually fills. + const pending = new Set(byTable.keys()); + const ordered: string[] = []; + while (pending.size > 0) { + let progressed = false; + for (const table of [...pending]) { + const blocked = (fkByTable.get(table) ?? []).some( + (fk) => fk.ref_table !== table && pending.has(fk.ref_table), + ); + if (blocked) continue; + ordered.push(table); + pending.delete(table); + progressed = true; + } + if (!progressed) { + throw new Error( + `foreign-key cycle among: ${[...pending].sort().join(", ")}`, + ); + } + } + + const insertedId = new Map(); + let tag = 0; + + for (const table of ordered) { + const tableColumns = byTable.get(table)!; + const fks = fkByTable.get(table) ?? []; + const names: string[] = []; + const values: string[] = []; + + for (const column of tableColumns) { + const fk = fks.find((f) => f.column_name === column.column_name); + const required = + column.is_nullable === "NO" && column.column_default === null; + const isUserColumn = column.column_name === "user_id"; + + if (fk?.ref_table === "users") { + // Always bind to the account under test, nullable or not — a NULL + // owner would put the row outside the wipe and make the assertion + // vacuous. + names.push(column.column_name); + values.push(`'${userId}'`); + continue; + } + if (!required && !isUserColumn) continue; + if (fk) { + const parent = insertedId.get(fk.ref_table); + if (!parent) { + throw new Error( + `${table}.${column.column_name} references ${fk.ref_table}, which was not seeded`, + ); + } + names.push(column.column_name); + values.push(`'${parent}'`); + continue; + } + names.push(column.column_name); + values.push(synthesise(column.udt_name, enums, `seed-${tag++}`)); + } + + const sql = + names.length === 0 + ? `INSERT INTO "${table}" DEFAULT VALUES ON CONFLICT DO NOTHING` + : `INSERT INTO "${table}" (${names.map((n) => `"${n}"`).join(", ")}) VALUES (${values.join(", ")}) ON CONFLICT DO NOTHING`; + await prisma.$executeRawUnsafe(sql); + + const hasId = tableColumns.some((c) => c.column_name === "id"); + if (hasId) { + const rows = await prisma.$queryRawUnsafe>( + `SELECT id FROM "${table}" LIMIT 1`, + ); + if (rows[0]) insertedId.set(table, rows[0].id); + } + } + + return ordered; +} + +async function countRows( + prisma: PrismaClient, + table: string, + userId?: string, +): Promise { + const rows = userId + ? await prisma.$queryRawUnsafe>( + `SELECT count(*)::bigint AS n FROM "${table}" WHERE user_id = $1`, + userId, + ) + : await prisma.$queryRawUnsafe>( + `SELECT count(*)::bigint AS n FROM "${table}"`, + ); + return Number(rows[0].n); +} + +describe("DELETE /api/settings/data leaves nothing it promised to delete", () => { + beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + }); + + it("wipes every in-scope table and keeps exactly the declared set", async () => { + const prisma = getPrismaClient(); + const tables = modelTableMap(); + + const user = await prisma.user.create({ + data: { + username: "wipe-subject", + email: "wipe-subject@example.test", + heightCm: 180, + dateOfBirth: new Date("1990-01-01T00:00:00Z"), + gender: "OTHER", + fullName: "Seeded Name", + insurerName: "Seeded Insurer", + stravaAthleteId: "12345", + thresholdsJson: { sys: 130 }, + onboardingCompletedAt: new Date(), + onboardingStep: 4, + passwordHash: "argon2-placeholder", + locale: "en", + unitPreference: "metric", + }, + }); + + const seeded = await seedEveryTable(prisma, user.id); + expect( + seeded.length, + "the seeder should reach essentially the whole schema", + ).toBeGreaterThan(100); + + // The session has to be created after the generic seed so its id is the + // one the cookie carries. `mfaVerifiedAt` is set because the seeder also + // planted a WebAuthn MFA credential, which makes the account MFA-enrolled + // and puts the route behind a fresh step-up. + const session = await prisma.session.create({ + data: { + userId: user.id, + expiresAt: new Date(Date.now() + 60_000), + mfaVerifiedAt: new Date(), + }, + }); + cookieJar.set("healthlog_session", session.id); + + // ── act ── + const { DELETE } = await import("@/app/api/settings/data/route"); + const response = await DELETE( + new Request("http://localhost/api/settings/data", { + method: "DELETE", + body: JSON.stringify({ confirm: "DELETE" }), + }) as never, + ); + expect(response.status).toBe(200); + + // ── assert: nothing the plan claims to delete is left ── + const survivors: string[] = []; + for (const model of WIPE_MODELS) { + const table = tables.get(model)!; + // `AuditLog` is wiped inside the transaction and then receives exactly + // one new row after it commits: the receipt for the wipe itself. The + // history is gone; the record that the person asked for it is not. + const expected = model === "AuditLog" ? 1 : 0; + if ((await countRows(prisma, table, user.id)) > expected) { + survivors.push(`${model} (${table})`); + } + } + expect( + survivors, + `rows survived a wipe that told the person everything was deleted:\n${survivors.join("\n")}`, + ).toEqual([]); + + // The one surviving audit row is the receipt, not leftover history. + const auditRows = await prisma.auditLog.findMany({ + where: { userId: user.id }, + select: { action: true }, + }); + expect(auditRows.map((r) => r.action)).toEqual(["user.data.clear"]); + + // ── assert: the kept set is exactly what the copy promises ── + const missing: string[] = []; + for (const model of Object.keys(WIPE_EXEMPT)) { + const table = tables.get(model)!; + if ((await countRows(prisma, table, user.id)) === 0) { + missing.push(`${model} (${table})`); + } + } + expect( + missing, + `the copy says these are kept, and they are gone:\n${missing.join("\n")}`, + ).toEqual([]); + + // ── assert: the instance's own tables are untouched ── + for (const model of Object.keys(INSTANCE_SCOPED)) { + if (model === "User") continue; + const table = tables.get(model)!; + expect( + await countRows(prisma, table), + `${model} belongs to the instance and must survive one account's wipe`, + ).toBeGreaterThan(0); + } + + // ── assert: everything reached only by cascade is gone too ── + const wipedTables = new Set(WIPE_MODELS.map((m) => tables.get(m)!)); + const exemptTables = new Set( + Object.keys(WIPE_EXEMPT).map((m) => tables.get(m)!), + ); + const instanceTables = new Set( + Object.keys(INSTANCE_SCOPED).map((m) => tables.get(m)!), + ); + const leftovers: string[] = []; + for (const table of seeded) { + if (wipedTables.has(table)) continue; + if (exemptTables.has(table)) continue; + if (instanceTables.has(table)) continue; + if ((await countRows(prisma, table)) > 0) leftovers.push(table); + } + expect( + leftovers, + `tables the wipe reaches only through a database cascade still hold rows:\n${leftovers.join("\n")}`, + ).toEqual([]); + + // ── assert: the account survives, reset but signed-in-able ── + const after = await prisma.user.findUniqueOrThrow({ + where: { id: user.id }, + }); + const record = after as unknown as Record; + + expect(record.username).toBe("wipe-subject"); + expect(record.passwordHash).toBe("argon2-placeholder"); + for (const column of Object.keys(USER_KEPT_FIELDS)) { + expect(record).toHaveProperty(column); + } + expect(record.locale).toBe("en"); + expect(record.unitPreference).toBe("metric"); + + const stillSet: string[] = []; + for (const column of Object.keys(USER_RESET)) { + const value = record[column]; + const cleared = + value === null || + value === false || + value === 0 || + (Array.isArray(value) && value.length === 0) || + value === "aggregated" || + value === "disconnected"; + if (!cleared) stillSet.push(`${column} = ${JSON.stringify(value)}`); + } + expect( + stillSet, + `User columns the plan resets still carry a value:\n${stillSet.join("\n")}`, + ).toEqual([]); + }); + + it("refuses without the typed confirmation and leaves the record alone", async () => { + const prisma = getPrismaClient(); + + const user = await prisma.user.create({ + data: { username: "wipe-refused", email: "wipe-refused@example.test" }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + await prisma.measurement.create({ + data: { + userId: user.id, + type: "WEIGHT", + value: 80, + unit: "kg", + measuredAt: new Date(), + }, + }); + + const { DELETE } = await import("@/app/api/settings/data/route"); + const response = await DELETE( + new Request("http://localhost/api/settings/data", { + method: "DELETE", + body: JSON.stringify({ confirm: "nope" }), + }) as never, + ); + + expect(response.status).toBe(422); + expect(await prisma.measurement.count({ where: { userId: user.id } })).toBe( + 1, + ); + }); + + it("revokes a clinician share link so the token resolves to nothing", async () => { + const prisma = getPrismaClient(); + + const user = await prisma.user.create({ + data: { username: "wipe-sharer", email: "wipe-sharer@example.test" }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + + const { hashToken } = await import("@/lib/auth/hmac"); + const rawToken = `hls_${"a".repeat(48)}`; + await prisma.clinicianShareLink.create({ + data: { + userId: user.id, + tokenHash: hashToken(rawToken), + label: "Practice", + rangeStart: new Date("2026-01-01T00:00:00Z"), + sectionsJson: {}, + expiresAt: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000), + }, + }); + + const { resolveShareToken } = + await import("@/lib/clinician-share/resolve-share-token"); + expect(await resolveShareToken(rawToken)).not.toBeNull(); + + const { DELETE } = await import("@/app/api/settings/data/route"); + const response = await DELETE( + new Request("http://localhost/api/settings/data", { + method: "DELETE", + body: JSON.stringify({ confirm: "DELETE" }), + }) as never, + ); + expect(response.status).toBe(200); + + // A clinician holding the link now reads nothing, not a stale record. + expect(await resolveShareToken(rawToken)).toBeNull(); + }); +}); diff --git a/tests/integration/soft-delete-copy-parity.test.ts b/tests/integration/soft-delete-copy-parity.test.ts new file mode 100644 index 000000000..052cc1c63 --- /dev/null +++ b/tests/integration/soft-delete-copy-parity.test.ts @@ -0,0 +1,120 @@ +/** + * Two confirmation dialogs promised permanence over a soft delete. + * + * `records.allergies.deleteConfirmDescription` says it can't be undone and + * `measurementReminders.deleteConfirmDescription` says the reminder is + * permanently deleted, while both handlers stamped `deletedAt` and kept the + * row — with no restore route, no undo affordance, and no purge job to give + * the tombstone a horizon. The row then rode the nightly disaster-recovery + * backup off the host, indefinitely. + * + * Neither tombstone bought anything: neither model is in the + * `/api/sync/changes` delta feed, nothing references either table, and every + * read already filtered `deletedAt: null`. So the behaviour moved to match + * the copy rather than the copy retreating. + * + * Asserted against the real database because the whole question is whether a + * row is still there. `src/__tests__/copy-matches-behaviour-guard.test.ts` + * holds the other end — that the copy keeps claiming permanence. + * + * Mutation check: put either handler back on `update({ data: { deletedAt } })` + * and the matching test goes red on the surviving row. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +async function signIn(username: string): Promise { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username, email: `${username}@example.test` }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return user.id; +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +describe("deletes that promise permanence remove the row", () => { + it("an allergy is gone from the database, not hidden", async () => { + const prisma = getPrismaClient(); + const userId = await signIn("allergy-deleter"); + + const allergy = await prisma.allergy.create({ + data: { userId, substance: "Penicillin" }, + }); + + const { DELETE } = await import("@/app/api/allergies/[id]/route"); + const response = await DELETE( + new Request(`http://localhost/api/allergies/${allergy.id}`, { + method: "DELETE", + }) as never, + { params: Promise.resolve({ id: allergy.id }) } as never, + ); + expect(response.status).toBe(200); + + expect(await prisma.allergy.count({ where: { id: allergy.id } })).toBe(0); + }); + + it("a checkup reminder is gone from the database, not hidden", async () => { + const prisma = getPrismaClient(); + const userId = await signIn("reminder-deleter"); + + const reminder = await prisma.measurementReminder.create({ + data: { + userId, + label: "Dental checkup", + intervalDays: 180, + notifyHour: 9, + nextDueAt: new Date(Date.now() + 86_400_000), + }, + }); + + const { DELETE } = + await import("@/app/api/measurement-reminders/[id]/route"); + const response = await DELETE( + new Request(`http://localhost/api/measurement-reminders/${reminder.id}`, { + method: "DELETE", + }) as never, + { params: Promise.resolve({ id: reminder.id }) } as never, + ); + expect(response.status).toBe(200); + + expect( + await prisma.measurementReminder.count({ where: { id: reminder.id } }), + ).toBe(0); + }); +}); From fb9a4ee616bad9230d4a1507aba202f27d431ac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 02:28:14 +0200 Subject: [PATCH 04/52] test(sync): follow the tombstone purge's batched outcome shape --- tests/integration/sync-changes.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/sync-changes.test.ts b/tests/integration/sync-changes.test.ts index ef29461fe..64185964c 100644 --- a/tests/integration/sync-changes.test.ts +++ b/tests/integration/sync-changes.test.ts @@ -377,8 +377,10 @@ describe("GET /api/sync/changes (real Postgres)", () => { const pruned = await cleanupExpiredMeasurementTombstones(prisma); // Only the ancient row (deletedAt past the retention horizon) is - // pruned; the recent reachable tombstone survives. - expect(pruned).toBe(1); + // pruned; the recent reachable tombstone survives. `drained` says the + // batched walk reached the end of the backlog rather than stopping at + // its per-run cap. + expect(pruned).toEqual({ deleted: 1, drained: true }); const survivors = await prisma.measurement.findMany({ where: { userId: TEST_USER_ID }, select: { externalId: true }, From 84e38ab11023ba79f431fa5b158497c4cc23781f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 14:04:56 +0200 Subject: [PATCH 05/52] chore(db): number the retention indexes for the slot they actually take --- .../migration.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename prisma/migrations/{0277_retention_purge_indexes => 0276_retention_purge_indexes}/migration.sql (100%) diff --git a/prisma/migrations/0277_retention_purge_indexes/migration.sql b/prisma/migrations/0276_retention_purge_indexes/migration.sql similarity index 100% rename from prisma/migrations/0277_retention_purge_indexes/migration.sql rename to prisma/migrations/0276_retention_purge_indexes/migration.sql From 527dd6c1fa58724367e2e0a292ff68f6af0b2c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 01:02:35 +0200 Subject: [PATCH 06/52] fix(integrations): escalate a long transient failure streak, and scope a success to its own leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sync failure the classifier bucketed as `transient` had no exit. The park test read `kind === "persistent"` and the persistent streak anchor was only ever stamped by a persistent failure, so a streak could run for a thousand attempts with the persistent counter at zero, the state frozen at `error_transient`, and no ladder it could climb. `transient` is a hypothesis about retryability, and the classifier defaults every timeout, 429, 5xx and unrecognised response to it precisely because the wire cannot tell a reboot from a permanent break. That default is right for one attempt and wrong for a thousand: an unbroken streak retried hourly for a full day has refuted it by experiment. Record it as `persistent` from there, which feeds the park ladder that already exists — 48 h of unbroken failure end to end before an integration stops attempting. The counters could not accumulate anyway. `IntegrationStatus` is keyed (user, integration) while Withings runs four legs on four crons and WHOOP four resources on four more, and a success cleared the error, every bucket, the streak anchor and the state whatever leg had recorded them. The ECG tick at :41 wiped what the sleep tick recorded at :15 — including a reauth park — so the strike ladder was reset several times an hour and the card painted a green "connected · 12 minutes ago" over a data class that had stopped arriving. A success now clears only what it is entitled to clear. Two columns carry it: `failing_since_at` dates the streak, `failing_leg` names its owner. The migration backfills the anchor for rows already in an error state so a connection broken for weeks escalates on its next attempt rather than restarting its clock. --- messages/de.json | 2 +- messages/en.json | 2 +- messages/es.json | 2 +- messages/fr.json | 2 +- messages/it.json | 2 +- messages/pl.json | 2 +- .../migration.sql | 30 ++ prisma/schema.prisma | 10 + src/app/api/integrations/status/route.ts | 8 + .../settings/integration-status-pill.tsx | 11 +- .../settings/integrations/shared.tsx | 11 +- src/lib/integrations/__tests__/status.test.ts | 8 + src/lib/integrations/status.ts | 215 ++++++++++- src/lib/integrations/sync-verdict.ts | 23 +- src/lib/jobs/reminder/whoop-sync.ts | 16 +- src/lib/mcp/__tests__/tools.test.ts | 2 + src/lib/mcp/tools.ts | 1 + src/lib/polar/__tests__/sync.test.ts | 8 +- src/lib/polar/sync-workouts.ts | 5 + src/lib/polar/sync.ts | 6 +- .../whoop/__tests__/sync-dead-token.test.ts | 22 +- .../whoop/__tests__/sync-user-whoop.test.ts | 28 +- src/lib/whoop/sync-core.ts | 10 +- .../withings/__tests__/sync-activity.test.ts | 4 +- src/lib/withings/__tests__/sync-ecg.test.ts | 15 +- .../__tests__/sync-measure-watermark.test.ts | 8 +- src/lib/withings/__tests__/sync-sleep.test.ts | 8 +- src/lib/withings/sync-activity.ts | 15 +- src/lib/withings/sync-ecg.ts | 17 +- src/lib/withings/sync-sleep.ts | 13 +- src/lib/withings/sync.ts | 24 +- .../integration-sync-escalation.test.ts | 358 ++++++++++++++++++ 32 files changed, 823 insertions(+), 65 deletions(-) create mode 100644 prisma/migrations/0275_integration_failure_streak_and_leg/migration.sql create mode 100644 tests/integration/integration-sync-escalation.test.ts diff --git a/messages/de.json b/messages/de.json index ee218f830..861e0a940 100644 --- a/messages/de.json +++ b/messages/de.json @@ -5155,7 +5155,7 @@ "integrationPill": { "connected": "Verbunden", "errorReconnect": "Fehler — neu verbinden", - "warningServerError": "Verbunden, Sync-Problem", + "warningServerError": "Sync fehlgeschlagen", "parkedReconnect": "Pausiert — manuell wieder verbinden", "notConnected": "Nicht verbunden", "justNow": "gerade eben", diff --git a/messages/en.json b/messages/en.json index 4c498d987..7edc56190 100644 --- a/messages/en.json +++ b/messages/en.json @@ -5155,7 +5155,7 @@ "integrationPill": { "connected": "Connected", "errorReconnect": "Error — reconnect", - "warningServerError": "Connected, sync issue", + "warningServerError": "Sync failing", "parkedReconnect": "Paused — reconnect manually", "notConnected": "Not connected", "justNow": "just now", diff --git a/messages/es.json b/messages/es.json index 8c609dfcd..a4c955d17 100644 --- a/messages/es.json +++ b/messages/es.json @@ -5155,7 +5155,7 @@ "integrationPill": { "connected": "Conectado", "errorReconnect": "Error — reconectar", - "warningServerError": "Conectado, problema de sincronización", + "warningServerError": "Sincronización con errores", "parkedReconnect": "Pausado — reconectar manualmente", "notConnected": "No conectado", "justNow": "ahora mismo", diff --git a/messages/fr.json b/messages/fr.json index 678b46765..cd3cc3217 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -5155,7 +5155,7 @@ "integrationPill": { "connected": "Connecté", "errorReconnect": "Erreur — reconnecter", - "warningServerError": "Connecté, problème de synchronisation", + "warningServerError": "Échec de synchronisation", "parkedReconnect": "En pause — reconnecter manuellement", "notConnected": "Non connecté", "justNow": "à l’instant", diff --git a/messages/it.json b/messages/it.json index 5a791cdc9..858cab7f8 100644 --- a/messages/it.json +++ b/messages/it.json @@ -5155,7 +5155,7 @@ "integrationPill": { "connected": "Connesso", "errorReconnect": "Errore — riconnetti", - "warningServerError": "Connesso, problema di sincronizzazione", + "warningServerError": "Sincronizzazione in errore", "parkedReconnect": "In pausa — riconnettere manualmente", "notConnected": "Non connesso", "justNow": "proprio ora", diff --git a/messages/pl.json b/messages/pl.json index 92b747092..4a8f61594 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -5155,7 +5155,7 @@ "integrationPill": { "connected": "Połączono", "errorReconnect": "Błąd — połącz ponownie", - "warningServerError": "Połączono, problem z synchronizacją", + "warningServerError": "Synchronizacja nie działa", "parkedReconnect": "Wstrzymane — połącz ręcznie", "notConnected": "Nie połączono", "justNow": "przed chwilą", diff --git a/prisma/migrations/0275_integration_failure_streak_and_leg/migration.sql b/prisma/migrations/0275_integration_failure_streak_and_leg/migration.sql new file mode 100644 index 000000000..d473ff3a2 --- /dev/null +++ b/prisma/migrations/0275_integration_failure_streak_and_leg/migration.sql @@ -0,0 +1,30 @@ +-- Integration sync ledger: date the failure streak, and name the leg that owns it. +-- +-- `failing_since_at` is the start of the current unbroken failure streak, +-- whatever kind the classifier assigned. The per-kind buckets counted attempts +-- but nothing dated them, so a streak the classifier had called `transient` +-- could run for a thousand attempts without ever becoming anything: the park +-- test read `kind = 'persistent'` and the persistent anchor was only ever +-- stamped by a persistent failure. +-- +-- `failing_leg` records which sync leg the error came from. The table is keyed +-- (user_id, integration) while Withings runs four legs on four crons and WHOOP +-- runs four resources on four more, so a sibling leg's success was clearing an +-- error it had not caused — several times an hour, which is why the strike +-- ladder never climbed. +ALTER TABLE "integration_statuses" + ADD COLUMN "failing_since_at" TIMESTAMP(3), + ADD COLUMN "failing_leg" TEXT; + +-- Backfill for rows that are already failing. Without this a connection that +-- has been broken for weeks would restart its clock at the next attempt and +-- need another two days before it escalated. `last_success_at` is the honest +-- anchor where it exists (the streak began at most one cron tick after it); +-- rows that never succeeded fall back to the last attempt, then to creation. +-- `failing_leg` is left NULL: which leg broke is not recoverable from the row, +-- and NULL means "unattributed", under which any success still clears — the +-- pre-existing behaviour, applied only to the backfilled rows. +UPDATE "integration_statuses" +SET "failing_since_at" = COALESCE("last_success_at", "last_attempt_at", "created_at") +WHERE "state" IN ('error_transient', 'error_reauth', 'parked') + AND "failing_since_at" IS NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index b625607b2..b54bdac41 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -4137,6 +4137,16 @@ model IntegrationStatus { // sole source of truth for the alert ladder. consecutiveFailuresByKind Json? @map("consecutive_failures_by_kind") // v1.4.43 W14 — { transient, reauth_required, persistent } persistentFailureStartedAt DateTime? @map("persistent_failure_started_at") // v1.4.43 W14 — start of the current persistent streak (drives >24h park) + // Start of the current unbroken failure streak, whatever the kind. The + // per-kind buckets count attempts; this dates them, which is what tells a + // blip apart from a dead pipe. Cleared by a success that is entitled to + // clear it, by a reconnect, a disconnect, and a resume. + failingSinceAt DateTime? @map("failing_since_at") + // Which sync leg owns the error currently on the row. `null` = the provider + // has a single leg, or the failure predates leg attribution. A success from + // a DIFFERENT leg may not clear an error this one did not cause — see + // `recordSyncSuccess`. + failingLeg String? @map("failing_leg") alertedAt DateTime? @map("alerted_at") // last time admin Telegram alert fired (idempotency guard) createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") diff --git a/src/app/api/integrations/status/route.ts b/src/app/api/integrations/status/route.ts index e74ad777f..9611e5c57 100644 --- a/src/app/api/integrations/status/route.ts +++ b/src/app/api/integrations/status/route.ts @@ -168,6 +168,7 @@ export const GET = apiHandler(async () => { state: string; lastSuccessAt: string | null; lastAttemptAt: string | null; + failingSince: string | null; }, opts: { connected?: boolean; @@ -181,6 +182,7 @@ export const GET = apiHandler(async () => { state: snapshot.state, lastSuccessAt: snapshot.lastSuccessAt, lastAttemptAt: snapshot.lastAttemptAt, + failingSinceAt: snapshot.failingSince, legacyLastSyncedAt: opts.legacyLastSyncedAt ?? null, cadence: INTEGRATION_CADENCE[integration], now, @@ -381,6 +383,12 @@ interface IntegrationViewModel { lastSuccessAt: string | null; lastAttemptAt: string | null; lastError: string | null; + /** + * Start of the current unbroken failure streak, or null when the provider is + * not failing. `syncHealth.since` is derived from it; the raw instant rides + * along so a client can render the age without re-deriving the verdict. + */ + failingSince: string | null; /** * The server-resolved liveness verdict. Present on EVERY entry, whether or * not a card renders it — the unit of coverage is the envelope entry, not diff --git a/src/components/settings/integration-status-pill.tsx b/src/components/settings/integration-status-pill.tsx index 35d8ef14d..0850617cc 100644 --- a/src/components/settings/integration-status-pill.tsx +++ b/src/components/settings/integration-status-pill.tsx @@ -110,11 +110,20 @@ const WARNING_STATES = new Set([ "parked", ]); -/** The states whose label carries an inline relative timestamp. */ +/** + * The states whose label carries an inline relative timestamp. + * + * `warning` is in the set because leaving it out was the whole lie: the pill + * said "sync issue" identically on hour one and on day seventeen, and the age + * — which the mapper computes and passes in — was dropped on the floor. How + * long it has been broken is the only part of that sentence a person can act + * on. + */ const RELATIVE_STATES = new Set([ "connected", "stale", "stalled", + "warning", ]); export function IntegrationStatusPill({ diff --git a/src/components/settings/integrations/shared.tsx b/src/components/settings/integrations/shared.tsx index 24583bec8..a7fb3a2d6 100644 --- a/src/components/settings/integrations/shared.tsx +++ b/src/components/settings/integrations/shared.tsx @@ -161,14 +161,19 @@ export function pillStateFor( /** * The timestamp the pill shows inline. `connected` reads the last success (a * connection row's own `lastSyncedAt` wins when it is newer, which is what - * repaints a fresh "just now" straight after a manual sync); the two aging - * states read the instant that triggered the verdict. + * repaints a fresh "just now" straight after a manual sync); the aging and + * failing states read the instant that triggered the verdict. + * + * `failing` must NOT fall through to the `legacyLastSyncedAt` arm below. That + * field is a connection row's own clock, kept fresh by the very sync that is + * erroring, so a provider that had delivered nothing for a fortnight painted a + * "just now" beside its warning. */ export function pillTimestampFor( status: IntegrationStatusViewModel | undefined, ): string | null { const verdict = status?.syncHealth?.verdict; - if (verdict === "stale" || verdict === "stalled") { + if (verdict === "stale" || verdict === "stalled" || verdict === "failing") { return status?.syncHealth?.since ?? null; } return status?.legacyLastSyncedAt ?? status?.lastSuccessAt ?? null; diff --git a/src/lib/integrations/__tests__/status.test.ts b/src/lib/integrations/__tests__/status.test.ts index 4eea74911..e8a95ac1b 100644 --- a/src/lib/integrations/__tests__/status.test.ts +++ b/src/lib/integrations/__tests__/status.test.ts @@ -108,6 +108,8 @@ describe("recordSyncSuccess", () => { persistent: 0, }, persistentFailureStartedAt: null, + failingSinceAt: null, + failingLeg: null, alertedAt: null, }); }); @@ -493,6 +495,8 @@ describe("markReauthRequired / markDisconnected / markReconnected", () => { persistent: 0, }, persistentFailureStartedAt: null, + failingSinceAt: null, + failingLeg: null, alertedAt: null, }); }); @@ -516,6 +520,8 @@ describe("markReauthRequired / markDisconnected / markReconnected", () => { persistent: 0, }, persistentFailureStartedAt: null, + failingSinceAt: null, + failingLeg: null, alertedAt: null, }); }); @@ -833,6 +839,8 @@ describe("resumeIntegrationFromPark", () => { persistent: 0, }, persistentFailureStartedAt: null, + failingSinceAt: null, + failingLeg: null, alertedAt: null, }); diff --git a/src/lib/integrations/status.ts b/src/lib/integrations/status.ts index b844cb4ed..8edc70e9f 100644 --- a/src/lib/integrations/status.ts +++ b/src/lib/integrations/status.ts @@ -34,7 +34,7 @@ import { prisma } from "@/lib/db"; import { encrypt, decrypt } from "@/lib/crypto"; import { auditLog } from "@/lib/auth/audit"; -import { getEvent } from "@/lib/logging/context"; +import { annotate, getEvent } from "@/lib/logging/context"; import { dispatchNotification } from "@/lib/notifications/dispatcher"; import type { IntegrationClassification } from "@/lib/integrations/http-status-classifier"; @@ -194,6 +194,41 @@ const ALERT_REPEAT_WINDOW_MS = 24 * 60 * 60 * 1000; */ const PARK_PERSISTENT_FAILURE_AFTER_MS = 24 * 60 * 60 * 1000; +/** + * How long an unbroken failure streak may keep claiming to be `transient` + * before the claim is treated as refuted and the failure is recorded as + * `persistent` instead. + * + * `transient` is a hypothesis, not an observation: the classifier defaults a + * timeout, a 429, a 5xx and any unrecognised response to it precisely because + * the wire cannot tell "the upstream is rebooting" from "this will never work + * again". That default is right for one attempt and wrong for a thousand — a + * streak that has been retried hourly for a full day without a single success + * has falsified the hypothesis by experiment, and continuing to call it + * transient is the ledger asserting a certainty the wire never gave it. + * + * 24 h is the same window the persistent park already uses. An upstream + * maintenance night, a rate-limit cool-off, a redeploy and a DNS wobble all + * fit comfortably inside it, so a genuine blip recovers and resets the streak + * long before this trips — nothing that heals on its own takes a day. The + * escalation is additionally gated on `getPersistentFailureThreshold()` + * consecutive failures, so a provider whose cron ticks rarely cannot escalate + * on its second-ever attempt just because wall-clock time passed. + * + * Escalating feeds the EXISTING ladder rather than a second one: the + * `persistent` bucket starts counting, `persistentFailureStartedAt` anchors, + * and `PARK_PERSISTENT_FAILURE_AFTER_MS` later the row parks and stops + * retrying — 48 h of unbroken failure end to end before anything is disabled. + */ +const TRANSIENT_ESCALATION_AFTER_MS = 24 * 60 * 60 * 1000; + +/** Row states that mean "an error is currently held on this row". */ +const ERROR_STATES: ReadonlySet = new Set([ + "error_transient", + "error_reauth", + "parked", +]); + export interface IntegrationStatusSnapshot { integration: IntegrationKey; state: IntegrationState; @@ -205,6 +240,13 @@ export interface IntegrationStatusSnapshot { * of truth now that the legacy `consecutiveFailures` column is * gone. */ consecutiveFailuresByKind: ConsecutiveFailuresByKind | null; + /** + * Start of the current unbroken failure streak, whatever the kind. The + * buckets count attempts; this dates them, which is what tells a blip apart + * from a pipe that has been dead for weeks. `null` whenever the row is not + * currently failing. + */ + failingSince: string | null; } /** @@ -271,6 +313,7 @@ export async function getIntegrationStatus( lastAttemptAt: null, lastError: null, consecutiveFailuresByKind: null, + failingSince: null, }; } return { @@ -280,6 +323,7 @@ export async function getIntegrationStatus( lastAttemptAt: row.lastAttemptAt?.toISOString() ?? null, lastError: row.lastError ? safeDecryptError(row.lastError) : null, consecutiveFailuresByKind: readBucketColumn(row.consecutiveFailuresByKind), + failingSince: row.failingSinceAt?.toISOString() ?? null, }; } @@ -305,17 +349,79 @@ export async function isReauthRequired( return row?.state === "error_reauth" || row?.state === "parked"; } +export interface RecordSyncSuccessOptions { + /** + * The sync leg that succeeded, for a provider whose legs run on separate + * schedules (Withings measures / activity / sleep / ECG, the four WHOOP + * resources, the two Polar legs). Omit it for a single-leg provider or for a + * full pass that covered every leg — an omitted leg clears the row whole, + * which is the pre-existing behaviour and the correct one for those callers. + */ + leg?: string; +} + /** * Record a successful sync. Resets the failure counter, clears any * prior error message, and (importantly) flips state back to * `connected` even from `error_reauth` — the Withings flow re-enters * after the OAuth callback writes a new refresh token. + * + * A success only clears what it is entitled to clear. `IntegrationStatus` is + * keyed `(userId, integration)`, but Withings runs four legs on four crons and + * WHOOP runs four resources on four more — so an unconditional clear meant the + * ECG leg succeeding at :41 wiped the error, the buckets, the streak anchor and + * the state that the sleep leg had recorded at :15. The failing leg's strike + * ladder was reset several times an hour, so it could never reach the alert + * threshold or the park window, and the card painted a green "connected · 12 + * minutes ago" over a data class that had stopped arriving. When the row holds + * an error owned by a DIFFERENT named leg, this success records its attempt and + * leaves the error standing. + * + * The partial path deliberately does NOT advance `lastSuccessAt`. That field is + * what the freshness arms and the card's "connected · X ago" read, and moving + * it forward while another leg is dead is exactly the sentence this change + * exists to stop the ledger from saying. `lastAttemptAt` does advance, so the + * row still reads as "being tried" rather than ageing into `stalled`. */ export async function recordSyncSuccess( userId: string, integration: IntegrationKey, + options?: RecordSyncSuccessOptions, ): Promise { const now = new Date(); + const leg = options?.leg ?? null; + + if (leg !== null) { + const existing = await prisma.integrationStatus.findUnique({ + where: { userId_integration: { userId, integration } }, + select: { state: true, failingLeg: true }, + }); + const heldByAnotherLeg = + existing != null && + existing.failingLeg != null && + existing.failingLeg !== leg && + ERROR_STATES.has(existing.state); + + if (heldByAnotherLeg) { + await prisma.integrationStatus.update({ + where: { userId_integration: { userId, integration } }, + data: { lastAttemptAt: now }, + }); + // Meta only — the enclosing wide event owns its own action name (the + // cron task), and overwriting it here would rename the job. + annotate({ + meta: { + integration_partial_success: { + integration, + succeeded_leg: leg, + failing_leg: existing.failingLeg, + }, + }, + }); + return; + } + } + // v1.4.43 W14 — a success resets ALL per-kind buckets back to zero // and clears the persistent-streak start timestamp. // v1.4.47 W1 — the legacy `consecutiveFailures` column was dropped @@ -337,6 +443,8 @@ export async function recordSyncSuccess( lastError: null, consecutiveFailuresByKind: zeroBuckets(), persistentFailureStartedAt: null, + failingSinceAt: null, + failingLeg: null, alertedAt: null, }, }); @@ -349,6 +457,12 @@ export interface RecordSyncFailureInput { message: string; /** Optional structured error code (e.g. "invalid_grant", "401"). */ errorCode?: string; + /** + * The sync leg that failed, for a provider whose legs run on separate + * schedules. Recorded on the row so a sibling leg's success cannot clear an + * error it did not cause. Omit it for a single-leg provider. + */ + leg?: string; } /** @@ -375,23 +489,35 @@ export interface RecordSyncFailureInput { * v1.4.47 W1 — the legacy single-column `consecutiveFailures` integer * was dropped (migration 0077); the bucket is now the sole counter * and the back-fill branch is gone. + * + * A `transient` failure whose streak has run unbroken for longer than + * `TRANSIENT_ESCALATION_AFTER_MS` is recorded as `persistent` instead. Before + * that, a `transient` streak had no exit at all: the park test read + * `kind === "persistent"`, `persistentFailureStartedAt` was only stamped on a + * persistent failure, and so a streak the classifier had bucketed as transient + * could run for a thousand attempts with the persistent counter at zero, the + * state frozen at `error_transient`, and no ladder it could ever climb. See + * the constant for why time, rather than count, is the honest signal. */ export async function recordSyncFailure( input: RecordSyncFailureInput, ): Promise { const { userId, integration, kind, message, errorCode } = input; + const leg = input.leg ?? null; const now = new Date(); const encryptedError = safeEncryptError(message); // Read the current row first so we can: // (a) compute the new per-kind bucket value - // (b) decide whether the persistent streak has exceeded the + // (b) decide whether the streak has outlived the `transient` claim + // (c) decide whether the persistent streak has exceeded the // park threshold const existing = await prisma.integrationStatus.findUnique({ where: { userId_integration: { userId, integration } }, select: { consecutiveFailuresByKind: true, persistentFailureStartedAt: true, + failingSinceAt: true, alertedAt: true, }, }); @@ -402,6 +528,30 @@ export async function recordSyncFailure( // or for the first-ever write of this (user, integration) pair const startingBuckets = readBucketColumn(existing?.consecutiveFailuresByKind); + // Streak anchor: the first failure of the current unbroken run. Every path + // that resolves a failure (success entitled to clear, reconnect, disconnect, + // resume) nulls it, so a non-null value always dates a live streak. + const failingSinceAt = existing?.failingSinceAt ?? now; + const streakAgeMs = now.getTime() - failingSinceAt.getTime(); + + // The `transient` hypothesis, tested against the clock. Both conditions are + // required: the wall-clock window rules out a blip, and the consecutive-count + // gate keeps a rarely-polled provider from escalating on its second attempt + // just because a day of real time went by. `+ 1` because this failure has not + // been written into the bucket yet. + const transientStreakAfterThis = (startingBuckets?.transient ?? 0) + 1; + const escalatedFromTransient = + kind === "transient" && + streakAgeMs >= TRANSIENT_ESCALATION_AFTER_MS && + transientStreakAfterThis >= getPersistentFailureThreshold(); + + // Everything downstream — bucket, state mapping, park test, audit detail, + // alert copy — reads the effective kind, so the escalation feeds the ladder + // that already exists instead of growing a parallel one. + const effectiveKind: FailureKind = escalatedFromTransient + ? "persistent" + : kind; + // Snapshot the persistent-bucket count BEFORE we increment so the // streak-anchor decision below can see the pre-increment value. // Otherwise we'd inspect the freshly-incremented bucket and never @@ -419,14 +569,21 @@ export async function recordSyncFailure( // Increment only the bucket matching this failure's kind. // The other two buckets stay at their current value so a persistent // streak isn't reset by an intervening transient hiccup. - buckets[kind] = (buckets[kind] ?? 0) + 1; + buckets[effectiveKind] = (buckets[effectiveKind] ?? 0) + 1; // Track the persistent-streak start so the >24h park check has a // wall-clock anchor. Only stamped on the FIRST persistent failure of // a streak; cleared on success or when the persistent bucket goes // back to zero (which today only happens via success, but the logic // is symmetric for future "transient drained the streak" rules). - const isPersistent = kind === "persistent"; + // + // An escalated transient anchors here too, at the moment of escalation + // rather than at the streak start. That is deliberate: it dates when the + // failure started being treated as unrecoverable, which is what the park + // window measures, and it puts a full second day between "this is no longer + // transient" and "stop retrying" — 48 h of unbroken failure before an + // integration disables itself over a verdict the wire never actually gave. + const isPersistent = effectiveKind === "persistent"; let persistentFailureStartedAt: Date | null = existing?.persistentFailureStartedAt ?? null; if (isPersistent && persistentStreakBefore === 0) { @@ -455,7 +612,7 @@ export async function recordSyncFailure( // grep for contract-bug bursts) const newState: IntegrationState = shouldPark ? "parked" - : kind === "reauth_required" + : effectiveKind === "reauth_required" ? "error_reauth" : "error_transient"; @@ -469,6 +626,8 @@ export async function recordSyncFailure( lastError: encryptedError, consecutiveFailuresByKind: buckets, persistentFailureStartedAt: isPersistent ? now : null, + failingSinceAt, + failingLeg: leg, }, update: { state: newState, @@ -476,6 +635,8 @@ export async function recordSyncFailure( lastError: encryptedError, consecutiveFailuresByKind: buckets, persistentFailureStartedAt, + failingSinceAt, + failingLeg: leg, }, }); @@ -496,11 +657,18 @@ export async function recordSyncFailure( userId, details: { integration, - kind, + kind: effectiveKind, + // Non-null only when the clock overrode the classifier, so an operator + // reading the trail can tell "the upstream said this was permanent" from + // "we concluded it was after a day of retrying". + escalatedFrom: escalatedFromTransient ? kind : null, + failingLeg: leg, + failingSinceAt: failingSinceAt.toISOString(), + streakAgeMs, errorCode: errorCode ?? null, message, attemptNumber: bucketTotal, - bucketCount: buckets[kind], + bucketCount: buckets[effectiveKind], state: newState, }, }); @@ -528,7 +696,7 @@ export async function recordSyncFailure( await maybeAlertAdmins({ userId, integration, - kind, + kind: effectiveKind, message, errorCode, consecutiveFailures: alertSignal, @@ -560,6 +728,10 @@ export async function recordSyncFailure( persistentFailureStartedAt: existing.persistentFailureStartedAt.toISOString(), persistentStreakAgeMs, + // The whole streak, which for an escalated transient is a full day + // longer than the persistent window above. + failingSinceAt: failingSinceAt.toISOString(), + streakAgeMs, errorCode: errorCode ?? null, message, }, @@ -603,6 +775,8 @@ export async function resumeIntegrationFromPark( lastError: null, consecutiveFailuresByKind: zeroBuckets(), persistentFailureStartedAt: null, + failingSinceAt: null, + failingLeg: null, alertedAt: null, }, }); @@ -646,8 +820,11 @@ export async function parkIntegrationAtReauth(opts: { integration: IntegrationKey; message: string; errorCode: string; + /** The leg whose scope gap forced the park, for multi-leg providers. */ + leg?: string; }): Promise { const { userId, integration, message, errorCode } = opts; + const leg = opts.leg ?? null; const now = new Date(); const encryptedError = safeEncryptError(message); @@ -656,7 +833,7 @@ export async function parkIntegrationAtReauth(opts: { // audit log when the state or error changes. const existing = await prisma.integrationStatus.findUnique({ where: { userId_integration: { userId, integration } }, - select: { state: true, lastError: true }, + select: { state: true, lastError: true, failingSinceAt: true }, }); const isFreshPark = existing?.state !== "error_reauth" || existing.lastError !== encryptedError; @@ -673,6 +850,8 @@ export async function parkIntegrationAtReauth(opts: { // zero so a later genuine transient burst still has the full // 3-strike runway before paging. consecutiveFailuresByKind: zeroBuckets(), + failingSinceAt: now, + failingLeg: leg, }, update: { state: "error_reauth", @@ -681,6 +860,8 @@ export async function parkIntegrationAtReauth(opts: { // Deliberately omit `consecutiveFailuresByKind` — the existing // bucket values are preserved exactly. This is the whole point // of the helper. + failingSinceAt: existing?.failingSinceAt ?? now, + failingLeg: leg, }, }); @@ -703,6 +884,12 @@ export async function markReauthRequired( integration: IntegrationKey, message: string, ): Promise { + const now = new Date(); + const existing = await prisma.integrationStatus.findUnique({ + where: { userId_integration: { userId, integration } }, + select: { failingSinceAt: true }, + }); + // v1.4.47 W1 — the legacy `consecutiveFailures` column was dropped // (migration 0077). Out-of-band reauth detection (proactive token // refresh) seeds the `reauth_required` bucket at 1 on a first-ever @@ -720,11 +907,15 @@ export async function markReauthRequired( reauth_required: 1, persistent: 0, }, - lastAttemptAt: new Date(), + lastAttemptAt: now, + failingSinceAt: now, }, update: { state: "error_reauth", lastError: safeEncryptError(message), + // Never re-stamp a streak already running — the anchor dates the + // failure, not the detection. + failingSinceAt: existing?.failingSinceAt ?? now, }, }); @@ -756,6 +947,8 @@ export async function markDisconnected( lastError: null, consecutiveFailuresByKind: zeroBuckets(), persistentFailureStartedAt: null, + failingSinceAt: null, + failingLeg: null, alertedAt: null, }, }); @@ -783,6 +976,8 @@ export async function markReconnected( lastError: null, consecutiveFailuresByKind: zeroBuckets(), persistentFailureStartedAt: null, + failingSinceAt: null, + failingLeg: null, alertedAt: null, }, }); diff --git a/src/lib/integrations/sync-verdict.ts b/src/lib/integrations/sync-verdict.ts index f4ab30bce..d0313902b 100644 --- a/src/lib/integrations/sync-verdict.ts +++ b/src/lib/integrations/sync-verdict.ts @@ -132,6 +132,14 @@ export interface SyncVerdictInput { lastAttemptAt?: string | Date | null; /** A connection row's own `lastSyncedAt`, for rows predating the ledger. */ legacyLastSyncedAt?: string | Date | null; + /** + * Start of the current unbroken failure streak, from the ledger. This is what + * the error verdicts date themselves by. `lastAttemptAt` is the wrong anchor + * — it is refreshed every hour by the retry itself, so a connection dead for + * a fortnight reported "failing, just now". `lastSuccessAt` is nearly right + * but a multi-leg provider can advance it while one leg stays broken. + */ + failingSinceAt?: string | Date | null; /** Newest observed data instant, when the caller knows it (Apple Health). */ lastDataAt?: string | Date | null; cadence: SyncCadence; @@ -183,6 +191,7 @@ export function resolveSyncVerdict(input: SyncVerdictInput): SyncHealth { const lastAttempt = toMillis(input.lastAttemptAt); const legacy = toMillis(input.legacyLastSyncedAt); const lastData = toMillis(input.lastDataAt); + const failingSince = toMillis(input.failingSinceAt); // 1. No live connection, or one the user explicitly ended. if (input.connected === false) @@ -196,10 +205,13 @@ export function resolveSyncVerdict(input: SyncVerdictInput): SyncHealth { // 2. States the user has to act on, in order of specificity. if (input.state === "parked") { - return { verdict: "parked", since: iso(lastAttempt) }; + return { verdict: "parked", since: iso(failingSince ?? lastAttempt) }; } if (input.state === "error_reauth") { - return { verdict: "reauth_required", since: iso(lastAttempt) }; + return { + verdict: "reauth_required", + since: iso(failingSince ?? lastAttempt), + }; } // 3. Nothing has ever run and nothing has ever arrived — the honest reading @@ -218,9 +230,12 @@ export function resolveSyncVerdict(input: SyncVerdictInput): SyncHealth { } } - // 5. Trying and erroring. + // 5. Trying and erroring. `since` dates the streak, not the last retry: this + // is the field the card renders, and "failing since a minute ago" on a + // connection that last delivered a fortnight back is the sentence this + // whole path exists to stop. if (input.state === "error_transient") { - return { verdict: "failing", since: iso(lastSuccess) }; + return { verdict: "failing", since: iso(failingSince ?? lastSuccess) }; } // 6. Running, but nothing new is arriving. The primary arm for push sources. diff --git a/src/lib/jobs/reminder/whoop-sync.ts b/src/lib/jobs/reminder/whoop-sync.ts index b8df364f5..10f07df56 100644 --- a/src/lib/jobs/reminder/whoop-sync.ts +++ b/src/lib/jobs/reminder/whoop-sync.ts @@ -24,6 +24,7 @@ import { import { recordWhoopSyncFailure, syncWhoopResourceWithStatus, + type WhoopSyncLeg, } from "@/lib/whoop/sync-core"; import { isSyncFailureRecorded } from "@/lib/integrations/status"; import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; @@ -68,6 +69,7 @@ type ByIdSync = (userId: string, resourceId: string) => Promise; */ export async function runWhoopResourceSync( taskName: string, + leg: WhoopSyncLeg, jobs: Job[], syncFn: CollectionSync, byIdFn?: ByIdSync, @@ -101,7 +103,7 @@ export async function runWhoopResourceSync( // Stamp `IntegrationStatus.lastSuccessAt` on a genuine per-resource // import — a webhook-driven refresh advanced the cursor but used to // leave the sync-status ledger frozen until the next full poll. - const imported = await syncWhoopResourceWithStatus(userId, () => + const imported = await syncWhoopResourceWithStatus(userId, leg, () => resourceId && byIdFn ? byIdFn(userId, resourceId) : syncFn(userId), ); measurementsImported += imported; @@ -119,7 +121,7 @@ export async function runWhoopResourceSync( // marked in `handleCollectionFetchError`, so record only the UNMARKED // escapes here. if (!isSyncFailureRecorded(err)) { - await recordWhoopSyncFailure(userId, err); + await recordWhoopSyncFailure(userId, err, leg); } evt.addWarning(`${taskName} failed for user ${userId}: ${err}`); } @@ -144,6 +146,7 @@ export async function runWhoopResourceSync( export function handleWhoopRecoverySync(jobs: Job[]) { return runWhoopResourceSync( "job.whoop_recovery_sync", + "recovery", jobs, syncUserRecovery, syncWhoopRecoveryById, @@ -153,6 +156,7 @@ export function handleWhoopRecoverySync(jobs: Job[]) { export function handleWhoopSleepSync(jobs: Job[]) { return runWhoopResourceSync( "job.whoop_sleep_sync", + "sleep", jobs, syncWhoopSleep, syncWhoopSleepById, @@ -162,6 +166,7 @@ export function handleWhoopSleepSync(jobs: Job[]) { export function handleWhoopWorkoutSync(jobs: Job[]) { return runWhoopResourceSync( "job.whoop_workout_sync", + "workout", jobs, syncUserWorkout, syncWhoopWorkoutById, @@ -170,5 +175,10 @@ export function handleWhoopWorkoutSync(jobs: Job[]) { export function handleWhoopCycleSync(jobs: Job[]) { // Cycle has no webhook (poll-only), so no fetch-by-id path. - return runWhoopResourceSync("job.whoop_cycle_sync", jobs, syncUserCycle); + return runWhoopResourceSync( + "job.whoop_cycle_sync", + "cycle", + jobs, + syncUserCycle, + ); } diff --git a/src/lib/mcp/__tests__/tools.test.ts b/src/lib/mcp/__tests__/tools.test.ts index e5c7526c5..ddf9c85f8 100644 --- a/src/lib/mcp/__tests__/tools.test.ts +++ b/src/lib/mcp/__tests__/tools.test.ts @@ -572,6 +572,7 @@ describe("get_integration_status", () => { lastAttemptAt: "2026-06-27T00:00:00.000Z", lastError: "token revoked", consecutiveFailuresByKind: null, + failingSince: null, }); const result = (await tool("get_integration_status").run(CTX, {})) as { @@ -611,6 +612,7 @@ describe("get_integration_status", () => { lastAttemptAt, lastError: null, consecutiveFailuresByKind: null, + failingSince: null, }); return tool("get_integration_status") diff --git a/src/lib/mcp/tools.ts b/src/lib/mcp/tools.ts index 46be574db..13abed92e 100644 --- a/src/lib/mcp/tools.ts +++ b/src/lib/mcp/tools.ts @@ -1518,6 +1518,7 @@ export const MCP_TOOLS: McpToolDefinition[] = [ state: s.state, lastSuccessAt: s.lastSuccessAt, lastAttemptAt: s.lastAttemptAt, + failingSinceAt: s.failingSince, cadence: INTEGRATION_CADENCE[s.integration], }); return { diff --git a/src/lib/polar/__tests__/sync.test.ts b/src/lib/polar/__tests__/sync.test.ts index 56e33a343..f5d12ff9c 100644 --- a/src/lib/polar/__tests__/sync.test.ts +++ b/src/lib/polar/__tests__/sync.test.ts @@ -196,7 +196,9 @@ describe("syncUserPolar", () => { externalId: "recharge:2026-06-10:recovery", value: 100, }); - expect(recordSuccessMock).toHaveBeenCalledWith("u1", "polar"); + expect(recordSuccessMock).toHaveBeenCalledWith("u1", "polar", { + leg: "vitals", + }); }); it("maps a cardio-load row with source POLAR + stable externalId", async () => { @@ -482,7 +484,9 @@ describe("syncUserPolar", () => { expect(recordFailureMock).not.toHaveBeenCalled(); expect(recordSuccessMock).toHaveBeenCalledTimes(1); - expect(recordSuccessMock).toHaveBeenCalledWith("u1", "polar"); + expect(recordSuccessMock).toHaveBeenCalledWith("u1", "polar", { + leg: "vitals", + }); }); it.each([ diff --git a/src/lib/polar/sync-workouts.ts b/src/lib/polar/sync-workouts.ts index 0f7e942c0..7084ae511 100644 --- a/src/lib/polar/sync-workouts.ts +++ b/src/lib/polar/sync-workouts.ts @@ -40,6 +40,8 @@ import { fetchExercises, mapExercise, type PolarWorkoutRow } from "./client"; import { getPolarConnection } from "./credentials"; import { PolarApiError, classifyPolarError } from "./response-classifier"; +const POLAR_LEG_WORKOUTS = "workouts"; + async function recordPolarWorkoutFailure( userId: string, err: unknown, @@ -47,6 +49,9 @@ async function recordPolarWorkoutFailure( await recordSyncFailure({ userId, integration: "polar", + // The workouts leg rides the same hourly tick as vitals but owns its own + // errors: the vitals success stamp must not clear a failure from here. + leg: POLAR_LEG_WORKOUTS, // Same classification the vitals leg uses (`classifyPolarFailure` in // `./sync`); resolved directly off the shared classifier so this leg does // not have to import the heavier vitals-sync module. diff --git a/src/lib/polar/sync.ts b/src/lib/polar/sync.ts index 048445a4f..e6f62a920 100644 --- a/src/lib/polar/sync.ts +++ b/src/lib/polar/sync.ts @@ -91,6 +91,8 @@ export function classifyPolarFailure(err: unknown): FailureKind { * record a future escape with the SAME provider-owned classification the inline * catch already applies. */ +export const POLAR_LEG_VITALS = "vitals"; + export async function recordPolarSyncFailure( userId: string, err: unknown, @@ -98,6 +100,7 @@ export async function recordPolarSyncFailure( await recordSyncFailure({ userId, integration: "polar", + leg: POLAR_LEG_VITALS, kind: classifyPolarFailure(err), message: err instanceof Error ? err.message : String(err), errorCode: @@ -288,6 +291,7 @@ export async function syncUserPolar(userId: string): Promise { await recordSyncFailure({ userId, integration: "polar", + leg: POLAR_LEG_VITALS, kind: classifyPolarFailure(firstErr), message: `partial sync failure (${names}): ${ firstErr instanceof Error ? firstErr.message : String(firstErr) @@ -300,7 +304,7 @@ export async function syncUserPolar(userId: string): Promise { return imported; } - await recordSyncSuccess(userId, "polar"); + await recordSyncSuccess(userId, "polar", { leg: POLAR_LEG_VITALS }); return imported; } diff --git a/src/lib/whoop/__tests__/sync-dead-token.test.ts b/src/lib/whoop/__tests__/sync-dead-token.test.ts index 3065f74a2..cea0a9b4e 100644 --- a/src/lib/whoop/__tests__/sync-dead-token.test.ts +++ b/src/lib/whoop/__tests__/sync-dead-token.test.ts @@ -259,19 +259,29 @@ describe("syncWhoopResourceWithStatus — dominant per-resource cron path", () = // The hourly-cron arm Fitbit has no twin for: a dead token noted by // `getValidToken` inside `run` must block the per-resource success stamp, // or the release is inert on the path that runs most often. - const imported = await syncWhoopResourceWithStatus("user-1", async () => { - noteHardFailure(WHOOP_TOKEN_HARD_FAIL); - return 0; - }); + const imported = await syncWhoopResourceWithStatus( + "user-1", + "recovery", + async () => { + noteHardFailure(WHOOP_TOKEN_HARD_FAIL); + return 0; + }, + ); expect(imported).toBe(0); expect(recordSyncSuccessMock).not.toHaveBeenCalled(); }); it("stamps success on a clean per-resource run (no hard failure)", async () => { - const imported = await syncWhoopResourceWithStatus("user-1", async () => 3); + const imported = await syncWhoopResourceWithStatus( + "user-1", + "recovery", + async () => 3, + ); expect(imported).toBe(3); - expect(recordSyncSuccessMock).toHaveBeenCalledWith("user-1", "whoop"); + expect(recordSyncSuccessMock).toHaveBeenCalledWith("user-1", "whoop", { + leg: "recovery", + }); }); }); diff --git a/src/lib/whoop/__tests__/sync-user-whoop.test.ts b/src/lib/whoop/__tests__/sync-user-whoop.test.ts index 2c5fcfbdc..00110e072 100644 --- a/src/lib/whoop/__tests__/sync-user-whoop.test.ts +++ b/src/lib/whoop/__tests__/sync-user-whoop.test.ts @@ -177,24 +177,38 @@ describe("syncUserWhoop — all-403 looks-healthy guard", () => { describe("syncWhoopResourceWithStatus — per-resource lastSuccessAt stamp", () => { it("stamps success when the resource imported rows", async () => { - const imported = await syncWhoopResourceWithStatus("user1", async () => 5); + const imported = await syncWhoopResourceWithStatus( + "user1", + "recovery", + async () => 5, + ); expect(imported).toBe(5); - expect(recordSyncSuccess).toHaveBeenCalledWith("user1", "whoop"); + expect(recordSyncSuccess).toHaveBeenCalledWith("user1", "whoop", { + leg: "recovery", + }); }); it("stamps success on a clean fetch that imported nothing new", async () => { // No 403 soft-skip → a genuine "nothing changed" webhook/cron tick still // proves the connection is alive, so `lastSuccessAt` advances. - const imported = await syncWhoopResourceWithStatus("user1", async () => 0); + const imported = await syncWhoopResourceWithStatus( + "user1", + "recovery", + async () => 0, + ); expect(imported).toBe(0); - expect(recordSyncSuccess).toHaveBeenCalledWith("user1", "whoop"); + expect(recordSyncSuccess).toHaveBeenCalledWith("user1", "whoop", { + leg: "recovery", + }); }); it("does NOT stamp success when the resource soft-skipped a 403 and imported nothing", async () => { - const imported = await syncWhoopResourceWithStatus("user1", () => - softSkip403("user1"), + const imported = await syncWhoopResourceWithStatus( + "user1", + "recovery", + () => softSkip403("user1"), ); expect(imported).toBe(0); @@ -204,7 +218,7 @@ describe("syncWhoopResourceWithStatus — per-resource lastSuccessAt stamp", () it("propagates a hard failure without stamping success", async () => { await expect( - syncWhoopResourceWithStatus("user1", async () => { + syncWhoopResourceWithStatus("user1", "recovery", async () => { throw new Error("boom"); }), ).rejects.toThrow("boom"); diff --git a/src/lib/whoop/sync-core.ts b/src/lib/whoop/sync-core.ts index dc54db379..2e2e41212 100644 --- a/src/lib/whoop/sync-core.ts +++ b/src/lib/whoop/sync-core.ts @@ -591,8 +591,11 @@ export async function markResourceSynced( * a genuine import clears the error state; a 403 soft-skip leaves the * "looks-healthy" window closed. */ +export type WhoopSyncLeg = "recovery" | "sleep" | "workout" | "cycle"; + export async function syncWhoopResourceWithStatus( userId: string, + leg: WhoopSyncLeg, run: () => Promise, ): Promise { // Run under BOTH trackers: soft-skip (a 403 scope gate) and hard-fail (a dead @@ -607,7 +610,10 @@ export async function syncWhoopResourceWithStatus( ); const softSkippedOnly = softSkipCount > 0 && imported === 0; if (!softSkippedOnly && hardFailTracker.failures.length === 0) { - await recordSyncSuccess(userId, "whoop"); + // Leg-scoped: WHOOP runs four resource crons against one ledger row, so a + // healthy recovery tick at :05 must not clear the error the workout tick + // recorded at :35 the hour before. + await recordSyncSuccess(userId, "whoop", { leg }); } return imported; } @@ -619,6 +625,7 @@ export async function syncWhoopResourceWithStatus( export async function recordWhoopSyncFailure( userId: string, err: unknown, + leg?: WhoopSyncLeg, ): Promise { const message = err instanceof Error ? err.message : String(err); await recordSyncFailure({ @@ -626,6 +633,7 @@ export async function recordWhoopSyncFailure( integration: "whoop", kind: classificationToFailureKind(classifyWhoopError(err)), message, + leg, errorCode: err instanceof WhoopApiError ? err.httpStatus?.toString() : undefined, }); diff --git a/src/lib/withings/__tests__/sync-activity.test.ts b/src/lib/withings/__tests__/sync-activity.test.ts index 0f9c2a80d..8c1ba17fe 100644 --- a/src/lib/withings/__tests__/sync-activity.test.ts +++ b/src/lib/withings/__tests__/sync-activity.test.ts @@ -370,7 +370,9 @@ describe("syncUserActivity — field mapping + idempotency", () => { } as never); await syncUserActivity("user-1"); - expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings"); + expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", { + leg: "activity", + }); }); it("W-2 — records a transient failure and refuses the success stamp when the batch write throws", async () => { diff --git a/src/lib/withings/__tests__/sync-ecg.test.ts b/src/lib/withings/__tests__/sync-ecg.test.ts index 92317bd21..5baca7a92 100644 --- a/src/lib/withings/__tests__/sync-ecg.test.ts +++ b/src/lib/withings/__tests__/sync-ecg.test.ts @@ -216,7 +216,9 @@ describe("syncUserEcg", () => { rhythmClassification: "IRREGULAR", externalId: "withings:ecg:user-1:111", }); - expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings"); + expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", { + leg: "ecg", + }); }); it("never narrows the broad catch-up to an ordinary webhook window", async () => { @@ -384,6 +386,7 @@ describe("syncUserEcg", () => { expect(recordWithingsSyncFailure).toHaveBeenCalledWith( "user-1", expect.objectContaining({ message: "source write failed" }), + "ecg", ); }); @@ -405,7 +408,11 @@ describe("syncUserEcg", () => { "IRREGULAR_RHYTHM_NOTIFICATION", new Date("2024-05-06T00:00:00.000Z"), ); - expect(recordWithingsSyncFailure).toHaveBeenCalledWith("user-1", poison); + expect(recordWithingsSyncFailure).toHaveBeenCalledWith( + "user-1", + poison, + "ecg", + ); expect(recordSyncSuccess).not.toHaveBeenCalled(); }); @@ -435,6 +442,8 @@ describe("syncUserEcg", () => { expect(imported).toBe(1); expect(prisma.measurement.upsert).toHaveBeenCalledTimes(1); expect(prisma.ecgRecording.upsert).not.toHaveBeenCalled(); - expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings"); + expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", { + leg: "ecg", + }); }); }); diff --git a/src/lib/withings/__tests__/sync-measure-watermark.test.ts b/src/lib/withings/__tests__/sync-measure-watermark.test.ts index c81a9d4a6..c597f80b4 100644 --- a/src/lib/withings/__tests__/sync-measure-watermark.test.ts +++ b/src/lib/withings/__tests__/sync-measure-watermark.test.ts @@ -201,7 +201,9 @@ describe("syncUserMeasurements — watermark hold on hard row failure (F4)", () // still healthy: stamp the watermark + success, record no failure. expect(imported).toBe(0); expect(prisma.withingsConnection.update).toHaveBeenCalledTimes(1); - expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings"); + expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", { + leg: "measures", + }); expect(recordSyncFailure).not.toHaveBeenCalled(); }); @@ -218,7 +220,9 @@ describe("syncUserMeasurements — watermark hold on hard row failure (F4)", () expect(imported).toBe(1); expect(prisma.withingsConnection.update).toHaveBeenCalledTimes(1); - expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings"); + expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", { + leg: "measures", + }); expect(recordSyncFailure).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/withings/__tests__/sync-sleep.test.ts b/src/lib/withings/__tests__/sync-sleep.test.ts index 98fcff22d..bf414bfe0 100644 --- a/src/lib/withings/__tests__/sync-sleep.test.ts +++ b/src/lib/withings/__tests__/sync-sleep.test.ts @@ -470,7 +470,9 @@ describe("syncUserSleep — segment writes + idempotency", () => { it("calls recordSyncSuccess after a clean round-trip", async () => { installFetchMock([]); await syncUserSleep("user-1"); - expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings"); + expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", { + leg: "sleep", + }); }); it("W-3 — records a transient failure and rethrows when a sleep write throws", async () => { @@ -580,7 +582,9 @@ describe("syncUserSleep — segment writes + idempotency", () => { // The one stage segment is still written; the summary failure is swallowed. expect(imported).toBe(1); expect(reconcileMock).toHaveBeenCalledTimes(1); - expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings"); + expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", { + leg: "sleep", + }); }); it("tolerates a ScanWatch night with no REM segment (no all-zeros row)", async () => { diff --git a/src/lib/withings/sync-activity.ts b/src/lib/withings/sync-activity.ts index 05431f7ef..869fb1d18 100644 --- a/src/lib/withings/sync-activity.ts +++ b/src/lib/withings/sync-activity.ts @@ -50,7 +50,11 @@ import { WithingsApiError, classifyWithingsResponse, } from "./response-classifier"; -import { getValidToken, recordWithingsSyncFailure } from "./sync"; +import { + WITHINGS_LEG_ACTIVITY, + getValidToken, + recordWithingsSyncFailure, +} from "./sync"; import { isReauthRequired, parkIntegrationAtReauth, @@ -253,6 +257,7 @@ export async function syncUserActivity( message: "Withings connection is missing the user.activity scope. Reconnect Withings in Settings to enable activity sync.", errorCode: "scope_missing", + leg: WITHINGS_LEG_ACTIVITY, }); return 0; } @@ -303,10 +308,11 @@ export async function syncUserActivity( kind: "reauth_required", message: err instanceof Error ? err.message : String(err), errorCode: "403", + leg: WITHINGS_LEG_ACTIVITY, }); throw err; } - await recordWithingsSyncFailure(userId, err); + await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_ACTIVITY); throw err; } @@ -468,6 +474,7 @@ export async function syncUserActivity( message: `activity batch write failed: ${ err instanceof Error ? err.message : String(err) }`, + leg: WITHINGS_LEG_ACTIVITY, }); } } @@ -500,7 +507,9 @@ export async function syncUserActivity( // W-2 — only a clean write stamps success. A swallowed batch-write left the // ledger honest via the transient failure recorded above. if (!writeFailed) { - await recordSyncSuccess(userId, "withings"); + await recordSyncSuccess(userId, "withings", { + leg: WITHINGS_LEG_ACTIVITY, + }); } return imported; } diff --git a/src/lib/withings/sync-ecg.ts b/src/lib/withings/sync-ecg.ts index 04c99bb1a..7f226913f 100644 --- a/src/lib/withings/sync-ecg.ts +++ b/src/lib/withings/sync-ecg.ts @@ -64,7 +64,11 @@ import { WithingsApiError, classifyWithingsResponse, } from "./response-classifier"; -import { getValidToken, recordWithingsSyncFailure } from "./sync"; +import { + WITHINGS_LEG_ECG, + getValidToken, + recordWithingsSyncFailure, +} from "./sync"; import { isReauthRequired, recordSyncFailure, @@ -408,10 +412,11 @@ export async function syncUserEcg( kind: "reauth_required", message: err instanceof Error ? err.message : String(err), errorCode: "403", + leg: WITHINGS_LEG_ECG, }); throw err; } - await recordWithingsSyncFailure(userId, err); + await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_ECG); throw err; } @@ -526,10 +531,14 @@ export async function syncUserEcg( } if (sourceWriteError) { - await recordWithingsSyncFailure(userId, sourceWriteError.cause); + await recordWithingsSyncFailure( + userId, + sourceWriteError.cause, + WITHINGS_LEG_ECG, + ); throw sourceWriteError.cause; } - await recordSyncSuccess(userId, "withings"); + await recordSyncSuccess(userId, "withings", { leg: WITHINGS_LEG_ECG }); return imported; } diff --git a/src/lib/withings/sync-sleep.ts b/src/lib/withings/sync-sleep.ts index 37ba27262..840b5d8d3 100644 --- a/src/lib/withings/sync-sleep.ts +++ b/src/lib/withings/sync-sleep.ts @@ -64,7 +64,11 @@ import { WithingsApiError, classifyWithingsResponse, } from "./response-classifier"; -import { getValidToken, recordWithingsSyncFailure } from "./sync"; +import { + WITHINGS_LEG_SLEEP, + getValidToken, + recordWithingsSyncFailure, +} from "./sync"; import { isReauthRequired, parkIntegrationAtReauth, @@ -355,6 +359,7 @@ export async function syncUserSleep( message: "Withings connection is missing the user.activity scope. Reconnect Withings in Settings to enable sleep sync.", errorCode: "scope_missing", + leg: WITHINGS_LEG_SLEEP, }); return 0; } @@ -394,10 +399,11 @@ export async function syncUserSleep( kind: "reauth_required", message: err instanceof Error ? err.message : String(err), errorCode: "403", + leg: WITHINGS_LEG_SLEEP, }); throw err; } - await recordWithingsSyncFailure(userId, err); + await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_SLEEP); throw err; } @@ -429,6 +435,7 @@ export async function syncUserSleep( message: `sleep write failed: ${ err instanceof Error ? err.message : String(err) }`, + leg: WITHINGS_LEG_SLEEP, }); throw err; }; @@ -581,6 +588,6 @@ export async function syncUserSleep( () => {}, ); - await recordSyncSuccess(userId, "withings"); + await recordSyncSuccess(userId, "withings", { leg: WITHINGS_LEG_SLEEP }); return imported; } diff --git a/src/lib/withings/sync.ts b/src/lib/withings/sync.ts index 5594b39f9..3da585814 100644 --- a/src/lib/withings/sync.ts +++ b/src/lib/withings/sync.ts @@ -156,7 +156,7 @@ export async function getValidToken( }, PROVIDER_REFRESH_TRANSACTION_OPTIONS); } catch (err) { getEvent()?.addWarning(`Token refresh failed for user ${userId}`); - await recordWithingsSyncFailure(userId, err); + await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_MEASURES); if (options.throwOnRefreshFailure) throw err; return null; } @@ -226,7 +226,7 @@ export async function syncUserMeasurements( try { measures = await fetchMeasurements(tokenInfo.accessToken, startDate); } catch (err) { - await recordWithingsSyncFailure(userId, err); + await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_MEASURES); throw err; } @@ -388,6 +388,7 @@ export async function syncUserMeasurements( await recordWithingsSyncFailure( userId, firstRowError ?? new Error("Withings measure row write failed"), + WITHINGS_LEG_MEASURES, ); return imported; } @@ -402,7 +403,9 @@ export async function syncUserMeasurements( // round-trip is the signal "the connection is healthy". A user who // doesn't weigh themselves for a week shouldn't see "error" on // their settings page. - await recordSyncSuccess(userId, "withings"); + await recordSyncSuccess(userId, "withings", { + leg: WITHINGS_LEG_MEASURES, + }); return imported; } @@ -423,9 +426,23 @@ export async function syncUserMeasurements( * `Error` instances (e.g. a pg-boss job retry that lost the prototype * during the JSON round-trip). */ +/** + * The four Withings sync legs. Each runs on its own cron at its own minute and + * writes the SAME `(user, withings)` ledger row, so a failure has to say which + * leg it came from — otherwise the next sibling to succeed clears an error it + * did not cause, which is what kept a broken leg invisible. + */ +export type WithingsSyncLeg = "measures" | "activity" | "sleep" | "ecg"; + +export const WITHINGS_LEG_MEASURES: WithingsSyncLeg = "measures"; +export const WITHINGS_LEG_ACTIVITY: WithingsSyncLeg = "activity"; +export const WITHINGS_LEG_SLEEP: WithingsSyncLeg = "sleep"; +export const WITHINGS_LEG_ECG: WithingsSyncLeg = "ecg"; + export async function recordWithingsSyncFailure( userId: string, err: unknown, + leg: WithingsSyncLeg, ): Promise { const message = err instanceof Error ? err.message : String(err); await recordSyncFailure({ @@ -433,6 +450,7 @@ export async function recordWithingsSyncFailure( integration: "withings", kind: classificationToFailureKind(classifyError(err)), message, + leg, errorCode: err instanceof WithingsApiError ? err.withingsStatus?.toString() diff --git a/tests/integration/integration-sync-escalation.test.ts b/tests/integration/integration-sync-escalation.test.ts new file mode 100644 index 000000000..fee900389 --- /dev/null +++ b/tests/integration/integration-sync-escalation.test.ts @@ -0,0 +1,358 @@ +/** + * Failure escalation and leg-scoped clearing on the integration ledger (real + * Postgres). + * + * Two defects are pinned here, and both are about state that only exists across + * several writes — a mocked Prisma cannot show either of them. + * + * 1. A `transient` streak had no exit. The park test read + * `kind === "persistent"` and the persistent anchor was only ever stamped + * by a persistent failure, so a streak the classifier had bucketed as + * transient could run indefinitely with the persistent counter at zero and + * the state frozen at `error_transient`. + * 2. `IntegrationStatus` is keyed `(userId, integration)` while Withings runs + * four legs on four crons and WHOOP four resources on four more. An + * unconditional success cleared the error, the buckets, the streak anchor + * and the state that a DIFFERENT leg had recorded, several times an hour, + * so the failing leg's strike ladder could never climb. + * + * The dispatcher is mocked at module level so the alert path resolves without + * dialling Telegram; the assertions are on ledger state, not the network. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/notifications/dispatcher", () => ({ + dispatchNotification: vi.fn().mockResolvedValue(undefined), +})); + +import { + getIntegrationStatus, + isReauthRequired, + recordSyncFailure, + recordSyncSuccess, +} from "@/lib/integrations/status"; + +const TEST_USER_ID = "user-sync-escalation"; +const HOUR_MS = 60 * 60 * 1000; + +function row(integration: string) { + return getPrismaClient().integrationStatus.findUnique({ + where: { userId_integration: { userId: TEST_USER_ID, integration } }, + }); +} + +/** + * Age the streak anchor backwards so a multi-day window can be exercised + * without a multi-day test. Only the anchor moves — every counter, the state + * and the error stay exactly as the production writes left them. + */ +async function ageStreakBy(integration: string, ms: number) { + const current = await row(integration); + const anchor = current!.failingSinceAt!; + await getPrismaClient().integrationStatus.update({ + where: { userId_integration: { userId: TEST_USER_ID, integration } }, + data: { failingSinceAt: new Date(anchor.getTime() - ms) }, + }); +} + +async function agePersistentAnchorBy(integration: string, ms: number) { + const current = await row(integration); + const anchor = current!.persistentFailureStartedAt!; + await getPrismaClient().integrationStatus.update({ + where: { userId_integration: { userId: TEST_USER_ID, integration } }, + data: { persistentFailureStartedAt: new Date(anchor.getTime() - ms) }, + }); +} + +async function failTransient(integration: string, leg?: string) { + await recordSyncFailure({ + userId: TEST_USER_ID, + integration: integration as "withings", + kind: "transient", + message: `${integration} sync error: 503 - upstream unavailable`, + errorCode: "503", + leg, + }); +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + await getPrismaClient().user.create({ + data: { + id: TEST_USER_ID, + username: "sync-escalation", + email: "sync-escalation@example.test", + }, + }); +}); + +describe("failure escalation", () => { + it("escalates an unbroken transient streak to persistent once it outlives the transient window", async () => { + // Three hourly failures — a real burst, still inside the window. + await failTransient("withings"); + await failTransient("withings"); + await failTransient("withings"); + + let current = await row("withings"); + expect(current!.state).toBe("error_transient"); + expect(current!.consecutiveFailuresByKind).toMatchObject({ + transient: 3, + persistent: 0, + }); + expect(current!.persistentFailureStartedAt).toBeNull(); + expect(current!.failingSinceAt).toBeInstanceOf(Date); + + // The upstream stays broken for a full day. Nothing else changes. + await ageStreakBy("withings", 25 * HOUR_MS); + await failTransient("withings"); + + current = await row("withings"); + // The transient bucket is frozen where the window closed; the failure now + // counts against the persistent bucket the park ladder reads. + expect(current!.consecutiveFailuresByKind).toMatchObject({ + transient: 3, + persistent: 1, + }); + expect(current!.persistentFailureStartedAt).toBeInstanceOf(Date); + + const audits = await getPrismaClient().auditLog.findMany({ + where: { action: "integrations.sync.failed", userId: TEST_USER_ID }, + orderBy: { createdAt: "asc" }, + }); + const last = JSON.parse(audits[audits.length - 1]!.details ?? "{}"); + expect(last.kind).toBe("persistent"); + // The trail keeps the classifier's own verdict, so an operator can tell + // "the upstream said this was permanent" from "we concluded it was". + expect(last.escalatedFrom).toBe("transient"); + }); + + it("parks the escalated streak a further window later and stops attempting", async () => { + await failTransient("withings"); + await failTransient("withings"); + await failTransient("withings"); + await ageStreakBy("withings", 25 * HOUR_MS); + await failTransient("withings"); + + // Escalated, still retrying — parking on the same tick would turn one long + // upstream outage into a manual reconnect for every account. + expect((await row("withings"))!.state).toBe("error_transient"); + expect(await isReauthRequired(TEST_USER_ID, "withings")).toBe(false); + + // A second day of the same. + await agePersistentAnchorBy("withings", 25 * HOUR_MS); + await failTransient("withings"); + + const current = await row("withings"); + expect(current!.state).toBe("parked"); + expect(await isReauthRequired(TEST_USER_ID, "withings")).toBe(true); + + const parked = await getPrismaClient().auditLog.findMany({ + where: { action: "integrations.parked", userId: TEST_USER_ID }, + }); + expect(parked).toHaveLength(1); + }); + + it("does not escalate a short outage that recovers", async () => { + // Well past the wall-clock window, but the streak keeps being broken by a + // success — which is exactly what a flapping upstream looks like. + for (let cycle = 0; cycle < 4; cycle++) { + await failTransient("withings"); + await failTransient("withings"); + await ageStreakBy("withings", 48 * HOUR_MS); + await recordSyncSuccess(TEST_USER_ID, "withings"); + } + + const current = await row("withings"); + expect(current!.state).toBe("connected"); + expect(current!.consecutiveFailuresByKind).toMatchObject({ + transient: 0, + reauth_required: 0, + persistent: 0, + }); + expect(current!.persistentFailureStartedAt).toBeNull(); + expect(current!.failingSinceAt).toBeNull(); + expect(current!.lastError).toBeNull(); + }); + + it("does not escalate on wall-clock alone below the consecutive-failure threshold", async () => { + // A rarely-polled provider: one failure, then a day passes, then a second. + // Time is necessary but not sufficient — otherwise a provider that ticks + // once a day would escalate on its second-ever attempt. + await failTransient("nightscout"); + await ageStreakBy("nightscout", 25 * HOUR_MS); + await failTransient("nightscout"); + + const current = await row("nightscout"); + expect(current!.state).toBe("error_transient"); + expect(current!.consecutiveFailuresByKind).toMatchObject({ + transient: 2, + persistent: 0, + }); + expect(current!.persistentFailureStartedAt).toBeNull(); + }); + + it("parks a rejected credential at error_reauth on the first failure instead of retrying it", async () => { + await recordSyncFailure({ + userId: TEST_USER_ID, + integration: "whoop", + kind: "reauth_required", + message: "WHOOP refresh error: 400 - invalid_grant", + errorCode: "invalid_grant", + leg: "recovery", + }); + + const current = await row("whoop"); + expect(current!.state).toBe("error_reauth"); + expect(current!.failingSinceAt).toBeInstanceOf(Date); + // The sync entry-points short-circuit — a revoked grant is not retried a + // thousand times. + expect(await isReauthRequired(TEST_USER_ID, "whoop")).toBe(true); + }); + + it("dates the streak from its first failure, not from the last retry", async () => { + await failTransient("withings"); + const firstAttempt = (await row("withings"))!.failingSinceAt!; + + await failTransient("withings"); + await failTransient("withings"); + + const current = await row("withings"); + expect(current!.failingSinceAt!.getTime()).toBe(firstAttempt.getTime()); + // `lastAttemptAt` keeps moving — that is the field that made a fortnight of + // silence read as "failing, just now". + expect(current!.lastAttemptAt!.getTime()).toBeGreaterThanOrEqual( + firstAttempt.getTime(), + ); + + const snapshot = await getIntegrationStatus(TEST_USER_ID, "withings"); + expect(snapshot.failingSince).toBe(firstAttempt.toISOString()); + }); +}); + +describe("leg-scoped success", () => { + it("does not let a sibling leg's success clear another leg's error", async () => { + await recordSyncFailure({ + userId: TEST_USER_ID, + integration: "withings", + kind: "transient", + message: "sleep write failed: reconciliation conflict", + leg: "sleep", + }); + const afterFailure = await row("withings"); + const streakStart = afterFailure!.failingSinceAt!; + + // The ECG leg ticks 26 minutes later and is perfectly healthy. + await recordSyncSuccess(TEST_USER_ID, "withings", { leg: "ecg" }); + + const current = await row("withings"); + expect(current!.state).toBe("error_transient"); + expect(current!.lastError).toBe(afterFailure!.lastError); + expect(current!.consecutiveFailuresByKind).toMatchObject({ transient: 1 }); + expect(current!.failingLeg).toBe("sleep"); + expect(current!.failingSinceAt!.getTime()).toBe(streakStart.getTime()); + // Not advanced: it is what the card's "connected · X ago" reads, and the + // sleep pipe has delivered nothing. + expect(current!.lastSuccessAt).toBeNull(); + // The attempt did happen, so the row must not age into `stalled`. + expect(current!.lastAttemptAt).toBeInstanceOf(Date); + }); + + it("lets the owning leg's own success clear the row whole", async () => { + await recordSyncFailure({ + userId: TEST_USER_ID, + integration: "withings", + kind: "transient", + message: "sleep write failed: reconciliation conflict", + leg: "sleep", + }); + await recordSyncSuccess(TEST_USER_ID, "withings", { leg: "ecg" }); + await recordSyncSuccess(TEST_USER_ID, "withings", { leg: "sleep" }); + + const current = await row("withings"); + expect(current!.state).toBe("connected"); + expect(current!.lastError).toBeNull(); + expect(current!.failingLeg).toBeNull(); + expect(current!.failingSinceAt).toBeNull(); + expect(current!.lastSuccessAt).toBeInstanceOf(Date); + expect(current!.consecutiveFailuresByKind).toMatchObject({ + transient: 0, + reauth_required: 0, + persistent: 0, + }); + }); + + it("keeps a leg's reauth park standing against a sibling leg's success", async () => { + await recordSyncFailure({ + userId: TEST_USER_ID, + integration: "withings", + kind: "reauth_required", + message: "Withings sleep error: 403", + errorCode: "403", + leg: "sleep", + }); + expect(await isReauthRequired(TEST_USER_ID, "withings")).toBe(true); + + await recordSyncSuccess(TEST_USER_ID, "withings", { leg: "ecg" }); + + // Un-parking here is the worst shape of the bug: the sleep leg's grant is + // gone, and the row went back to claiming a healthy connection. + expect(await isReauthRequired(TEST_USER_ID, "withings")).toBe(true); + expect((await row("withings"))!.state).toBe("error_reauth"); + }); + + it("lets a sibling success be counted while the strike ladder still climbs", async () => { + // The regression, end to end: the failing leg fails hourly, a healthy leg + // succeeds in between, and the streak must still reach the escalation. + for (let hour = 0; hour < 3; hour++) { + await failTransient("withings", "sleep"); + await recordSyncSuccess(TEST_USER_ID, "withings", { leg: "ecg" }); + } + await ageStreakBy("withings", 25 * HOUR_MS); + await failTransient("withings", "sleep"); + + const current = await row("withings"); + expect(current!.consecutiveFailuresByKind).toMatchObject({ + transient: 3, + persistent: 1, + }); + expect(current!.persistentFailureStartedAt).toBeInstanceOf(Date); + }); + + it("clears the row whole when a full pass names no leg", async () => { + // A run that covered every resource is entitled to clear everything, and a + // single-leg provider that never names a leg keeps its old behaviour. + await recordSyncFailure({ + userId: TEST_USER_ID, + integration: "whoop", + kind: "transient", + message: "WHOOP workout error: 503", + leg: "workout", + }); + await recordSyncSuccess(TEST_USER_ID, "whoop"); + + const current = await row("whoop"); + expect(current!.state).toBe("connected"); + expect(current!.lastError).toBeNull(); + expect(current!.failingLeg).toBeNull(); + expect(current!.failingSinceAt).toBeNull(); + }); + + it("lets any success clear an unattributed failure", async () => { + await failTransient("oura"); + await recordSyncSuccess(TEST_USER_ID, "oura", { leg: "vitals" }); + + const current = await row("oura"); + expect(current!.state).toBe("connected"); + expect(current!.lastError).toBeNull(); + expect(current!.failingSinceAt).toBeNull(); + }); +}); From 4e3b029c38ed1d04e2415c136a7d7509f816ad5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 01:10:44 +0200 Subject: [PATCH 07/52] test(integrations): pin the escalation ladder, leg-scoped clearing, and what the card says Mutation-checked against the production paths they cover: the escalation verdict, the leg guard in `recordSyncSuccess`, the streak anchor the verdict dates itself by, the status envelope threading it, and the pill's relative timestamp. Also give the four legacy per-provider status routes the verdict the consolidated envelope already carries. They answered `connected: true` plus a `state` string and nothing else, so a client reading one of them saw a live connection over a pipe that had delivered nothing for weeks. --- .../status/__tests__/route.test.ts | 38 ++++ src/app/api/integrations/status/route.ts | 42 ++-- src/app/api/nightscout/status/route.ts | 17 ++ src/app/api/oura/status/route.ts | 17 ++ src/app/api/polar/status/route.ts | 17 ++ src/app/api/strava/status/route.ts | 17 ++ .../integration-status-pill.test.tsx | 34 +++ src/lib/integrations/__tests__/status.test.ts | 209 ++++++++++++++++++ .../__tests__/sync-verdict.test.ts | 67 +++++- 9 files changed, 443 insertions(+), 15 deletions(-) diff --git a/src/app/api/integrations/status/__tests__/route.test.ts b/src/app/api/integrations/status/__tests__/route.test.ts index edba6a7c8..ec5d59e2e 100644 --- a/src/app/api/integrations/status/__tests__/route.test.ts +++ b/src/app/api/integrations/status/__tests__/route.test.ts @@ -29,6 +29,7 @@ const ledger: Record = { lastSuccessAt: null, lastAttemptAt: null, lastError: null, + failingSince: null, }; vi.mock("@/lib/integrations/status", () => ({ getIntegrationStatus: vi.fn(async (_u: string, integration: string) => ({ @@ -239,3 +240,40 @@ describe("/api/integrations/status — sync-health verdict on every entry", () = expect(nightscout!.syncHealth).toBeDefined(); }); }); + +describe("/api/integrations/status — a provider that has delivered nothing for weeks", () => { + beforeEach(() => { + vi.clearAllMocks(); + for (const key of Object.keys(perIntegrationLedger)) { + delete perIntegrationLedger[key]; + } + userFind.mockResolvedValue({}); + whoopFind.mockResolvedValue(null); + polarAvailable.mockResolvedValue(true); + ouraAvailable.mockResolvedValue(false); + }); + + it("reports it as failing and dates the verdict from the streak, not the retry", async () => { + const failingSince = new Date( + Date.now() - 17 * 24 * 60 * 60 * 1000, + ).toISOString(); + perIntegrationLedger.nightscout = { + state: "error_transient", + // The hourly cron is still running and a sibling leg can still stamp a + // success, so neither of these tells the user anything. + lastSuccessAt: new Date(Date.now() - 20 * 60_000).toISOString(), + lastAttemptAt: new Date(Date.now() - 4 * 60_000).toISOString(), + failingSince, + lastError: "Nightscout sync HTTP 502", + }; + userFind.mockResolvedValue({ nightscoutUrlEncrypted: "enc" }); + + const nightscout = (await fetchEntries()).find( + (entry) => entry.integration === "nightscout", + )!; + expect(nightscout.syncHealth).toEqual({ + verdict: "failing", + since: failingSince, + }); + }); +}); diff --git a/src/app/api/integrations/status/route.ts b/src/app/api/integrations/status/route.ts index 9611e5c57..fc6b657fc 100644 --- a/src/app/api/integrations/status/route.ts +++ b/src/app/api/integrations/status/route.ts @@ -22,6 +22,7 @@ import { getIntegrationStatus, getPersistentFailureThreshold, type IntegrationKey, + type IntegrationStatusSnapshot, } from "@/lib/integrations/status"; import { getSourceMetricFreshness, @@ -155,6 +156,25 @@ export const GET = apiHandler(async () => { const now = Date.now(); const whoopConnected = !!whoopConn?.whoopUserId; + /** + * The ledger fields the envelope publishes, named one by one. + * + * `failingSince` is deliberately not among them. It is the input to + * `syncHealth.since`, which every consumer already reads, and putting both on + * the wire would be two names for one fact — the second of which nothing + * renders. + */ + function publish(snapshot: IntegrationStatusSnapshot) { + return { + integration: snapshot.integration, + state: snapshot.state, + lastSuccessAt: snapshot.lastSuccessAt, + lastAttemptAt: snapshot.lastAttemptAt, + lastError: snapshot.lastError, + consecutiveFailuresByKind: snapshot.consecutiveFailuresByKind, + }; + } + /** * The sync-health verdict + the per-metric staleness flags, resolved from * data already in hand — no additional query. Every entry carries it, @@ -227,7 +247,7 @@ export const GET = apiHandler(async () => { threshold: getPersistentFailureThreshold(), integrations: [ { - ...withingsStatus, + ...publish(withingsStatus), configured: withingsConfigured, connected: !!withingsConn, connectedAt: withingsConn?.createdAt?.toISOString() ?? null, @@ -251,7 +271,7 @@ export const GET = apiHandler(async () => { }), } satisfies IntegrationViewModel & WithingsExtras, { - ...whoopStatus, + ...publish(whoopStatus), configured: whoopConfigured, connected: whoopConnected, connectedAt: whoopConnected @@ -274,7 +294,7 @@ export const GET = apiHandler(async () => { }), } satisfies IntegrationViewModel & WhoopExtras, { - ...fitbitStatus, + ...publish(fitbitStatus), configured: fitbitConfigured, connected: fitbitConnected, connectedAt: fitbitConn?.createdAt?.toISOString() ?? null, @@ -291,7 +311,7 @@ export const GET = apiHandler(async () => { }), } satisfies IntegrationViewModel & FitbitExtras, { - ...googleHealthStatus, + ...publish(googleHealthStatus), configured: googleHealthConfigured, connected: googleHealthConnected, connectedAt: googleHealthConn?.createdAt?.toISOString() ?? null, @@ -314,7 +334,7 @@ export const GET = apiHandler(async () => { }), } satisfies IntegrationViewModel & GoogleHealthExtras, { - ...polarStatus, + ...publish(polarStatus), // `connected` = a stored access token; `configured` mirrors it (the // OAuth card has no separate "credentials saved but disconnected" view // beyond `hasOwnCredentials`). The card greys out the connect button @@ -331,7 +351,7 @@ export const GET = apiHandler(async () => { }), } satisfies IntegrationViewModel & OAuthProviderExtras, { - ...ouraStatus, + ...publish(ouraStatus), connected: ouraConnected, configured: ouraConnected, available: ouraAvailable, @@ -344,7 +364,7 @@ export const GET = apiHandler(async () => { }), } satisfies IntegrationViewModel & OAuthProviderExtras, { - ...stravaStatus, + ...publish(stravaStatus), connected: stravaConnected, configured: stravaConnected, available: stravaAvailable, @@ -359,7 +379,7 @@ export const GET = apiHandler(async () => { }), } satisfies IntegrationViewModel & OAuthProviderExtras, { - ...nightscoutStatus, + ...publish(nightscoutStatus), // Nightscout is a URL the self-hoster pastes once; `configured` (a // stored URL) is the connect marker, and a fully-public instance has // no token. The card reads these off the envelope now instead of its @@ -383,12 +403,6 @@ interface IntegrationViewModel { lastSuccessAt: string | null; lastAttemptAt: string | null; lastError: string | null; - /** - * Start of the current unbroken failure streak, or null when the provider is - * not failing. `syncHealth.since` is derived from it; the raw instant rides - * along so a client can render the age without re-deriving the verdict. - */ - failingSince: string | null; /** * The server-resolved liveness verdict. Present on EVERY entry, whether or * not a card renders it — the unit of coverage is the envelope entry, not diff --git a/src/app/api/nightscout/status/route.ts b/src/app/api/nightscout/status/route.ts index 38096cd59..f2cda6aa0 100644 --- a/src/app/api/nightscout/status/route.ts +++ b/src/app/api/nightscout/status/route.ts @@ -3,6 +3,10 @@ import { apiHandler, requireAuth } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; import { apiSuccess } from "@/lib/api-response"; import { getIntegrationStatus } from "@/lib/integrations/status"; +import { + INTEGRATION_CADENCE, + resolveSyncVerdict, +} from "@/lib/integrations/sync-verdict"; /** * Nightscout connection status for the current user (v1.17.0). @@ -45,5 +49,18 @@ export const GET = apiHandler(async () => { lastSuccessAt: status.lastSuccessAt, lastAttemptAt: status.lastAttemptAt, lastError: status.lastError, + // `connected: true` here means "a connection exists", not "it is working" + // — a pipe dead for a fortnight still answers true. The verdict is the + // liveness truth, resolved by the same rule the Settings envelope uses so + // this route cannot drift into a second opinion. + syncHealth: resolveSyncVerdict({ + connected: true, + configured: true, + state: status.state, + lastSuccessAt: status.lastSuccessAt, + lastAttemptAt: status.lastAttemptAt, + failingSinceAt: status.failingSince, + cadence: INTEGRATION_CADENCE["nightscout"], + }), }); }); diff --git a/src/app/api/oura/status/route.ts b/src/app/api/oura/status/route.ts index f3d4d17f5..ba3abf574 100644 --- a/src/app/api/oura/status/route.ts +++ b/src/app/api/oura/status/route.ts @@ -4,6 +4,10 @@ import { annotate } from "@/lib/logging/context"; import { apiSuccess } from "@/lib/api-response"; import { getOuraClientCredentials } from "@/lib/oura/credentials"; import { getIntegrationStatus } from "@/lib/integrations/status"; +import { + INTEGRATION_CADENCE, + resolveSyncVerdict, +} from "@/lib/integrations/sync-verdict"; /** * v1.17.0 (F4) — Oura connection status for the current user. @@ -53,5 +57,18 @@ export const GET = apiHandler(async () => { lastSuccessAt: status.lastSuccessAt, lastAttemptAt: status.lastAttemptAt, lastError: status.lastError, + // `connected: true` here means "a connection exists", not "it is working" + // — a pipe dead for a fortnight still answers true. The verdict is the + // liveness truth, resolved by the same rule the Settings envelope uses so + // this route cannot drift into a second opinion. + syncHealth: resolveSyncVerdict({ + connected: true, + configured: true, + state: status.state, + lastSuccessAt: status.lastSuccessAt, + lastAttemptAt: status.lastAttemptAt, + failingSinceAt: status.failingSince, + cadence: INTEGRATION_CADENCE["oura"], + }), }); }); diff --git a/src/app/api/polar/status/route.ts b/src/app/api/polar/status/route.ts index 87d43742b..92cd26d79 100644 --- a/src/app/api/polar/status/route.ts +++ b/src/app/api/polar/status/route.ts @@ -4,6 +4,10 @@ import { annotate } from "@/lib/logging/context"; import { apiSuccess } from "@/lib/api-response"; import { getPolarClientCredentials } from "@/lib/polar/credentials"; import { getIntegrationStatus } from "@/lib/integrations/status"; +import { + INTEGRATION_CADENCE, + resolveSyncVerdict, +} from "@/lib/integrations/sync-verdict"; /** * v1.17.0 (F4) — Polar connection status for the current user. @@ -54,5 +58,18 @@ export const GET = apiHandler(async () => { lastSuccessAt: status.lastSuccessAt, lastAttemptAt: status.lastAttemptAt, lastError: status.lastError, + // `connected: true` here means "a connection exists", not "it is working" + // — a pipe dead for a fortnight still answers true. The verdict is the + // liveness truth, resolved by the same rule the Settings envelope uses so + // this route cannot drift into a second opinion. + syncHealth: resolveSyncVerdict({ + connected: true, + configured: true, + state: status.state, + lastSuccessAt: status.lastSuccessAt, + lastAttemptAt: status.lastAttemptAt, + failingSinceAt: status.failingSince, + cadence: INTEGRATION_CADENCE["polar"], + }), }); }); diff --git a/src/app/api/strava/status/route.ts b/src/app/api/strava/status/route.ts index 64df49fe4..c80c14a71 100644 --- a/src/app/api/strava/status/route.ts +++ b/src/app/api/strava/status/route.ts @@ -4,6 +4,10 @@ import { annotate } from "@/lib/logging/context"; import { apiSuccess } from "@/lib/api-response"; import { getStravaClientCredentials } from "@/lib/strava/credentials"; import { getIntegrationStatus } from "@/lib/integrations/status"; +import { + INTEGRATION_CADENCE, + resolveSyncVerdict, +} from "@/lib/integrations/sync-verdict"; /** * v1.28.x — Strava connection status for the current user. @@ -53,5 +57,18 @@ export const GET = apiHandler(async () => { lastSuccessAt: status.lastSuccessAt, lastAttemptAt: status.lastAttemptAt, lastError: status.lastError, + // `connected: true` here means "a connection exists", not "it is working" + // — a pipe dead for a fortnight still answers true. The verdict is the + // liveness truth, resolved by the same rule the Settings envelope uses so + // this route cannot drift into a second opinion. + syncHealth: resolveSyncVerdict({ + connected: true, + configured: true, + state: status.state, + lastSuccessAt: status.lastSuccessAt, + lastAttemptAt: status.lastAttemptAt, + failingSinceAt: status.failingSince, + cadence: INTEGRATION_CADENCE["strava"], + }), }); }); diff --git a/src/components/settings/__tests__/integration-status-pill.test.tsx b/src/components/settings/__tests__/integration-status-pill.test.tsx index 390bee332..466834869 100644 --- a/src/components/settings/__tests__/integration-status-pill.test.tsx +++ b/src/components/settings/__tests__/integration-status-pill.test.tsx @@ -196,3 +196,37 @@ describe("IntegrationStatusPill", () => { expect(html).toContain('data-testid="apple-health-status"'); }); }); + +describe("IntegrationStatusPill — a failing provider states its age", () => { + it("renders the failing state with how long it has been failing", () => { + const seventeenDaysAgo = new Date(Date.now() - 17 * 24 * 60 * 60 * 1000); + const html = render( + , + ); + // The label must not open with a claim of connection: the pipe has + // delivered nothing for a fortnight. + expect(html).not.toContain("Connected"); + expect(html).toContain("Sync failing"); + // And the age has to be on the pill. Without it, hour one and day + // seventeen render identically, which is how a dead sync stayed quiet. + expect(html).toMatch(/17\s*d\s+ago/); + expect(html).toContain('data-state="warning"'); + }); + + it("keeps the German label free of a connection claim too", () => { + const html = render( + , + "de", + ); + expect(html).not.toContain("Verbunden"); + expect(html).toContain("Sync fehlgeschlagen"); + }); +}); diff --git a/src/lib/integrations/__tests__/status.test.ts b/src/lib/integrations/__tests__/status.test.ts index e8a95ac1b..9be47a336 100644 --- a/src/lib/integrations/__tests__/status.test.ts +++ b/src/lib/integrations/__tests__/status.test.ts @@ -899,3 +899,212 @@ describe("sync-failure-recorded marker (§5)", () => { expect(isSyncFailureRecorded(frozen)).toBe(false); }); }); + +describe("transient-streak escalation", () => { + const DAY_MS = 24 * 60 * 60 * 1000; + + /** Seed the row the escalation reads: a streak of `transient` of a given age. */ + function seedStreak(opts: { transient: number; ageMs: number }) { + vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({ + consecutiveFailuresByKind: { + transient: opts.transient, + reauth_required: 0, + persistent: 0, + }, + persistentFailureStartedAt: null, + failingSinceAt: new Date(Date.now() - opts.ageMs), + alertedAt: null, + } as never); + vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce({ + alertedAt: new Date(), + } as never); + vi.mocked(prisma.user.findMany).mockResolvedValue([] as never); + } + + function upsertUpdate() { + return vi.mocked(prisma.integrationStatus.upsert).mock.calls[0]![0]! + .update as Record; + } + + it("records a transient failure as persistent once the streak outlives the window", async () => { + seedStreak({ transient: 3, ageMs: DAY_MS + 60_000 }); + + await recordSyncFailure({ + userId: "u1", + integration: "withings", + kind: "transient", + message: "Withings measure error: 503 - upstream", + }); + + // The transient bucket is frozen where the window closed; the persistent + // bucket — the only one the park ladder reads — starts counting. + expect(upsertUpdate().consecutiveFailuresByKind).toEqual({ + transient: 3, + reauth_required: 0, + persistent: 1, + }); + expect(upsertUpdate().persistentFailureStartedAt).toBeInstanceOf(Date); + expect(auditLog).toHaveBeenCalledWith( + "integrations.sync.failed", + expect.objectContaining({ + details: expect.objectContaining({ + kind: "persistent", + escalatedFrom: "transient", + }), + }), + ); + }); + + it("leaves a streak inside the window alone", async () => { + seedStreak({ transient: 9, ageMs: DAY_MS - 60_000 }); + + await recordSyncFailure({ + userId: "u1", + integration: "withings", + kind: "transient", + message: "Withings measure error: 503 - upstream", + }); + + expect(upsertUpdate().consecutiveFailuresByKind).toEqual({ + transient: 10, + reauth_required: 0, + persistent: 0, + }); + expect(upsertUpdate().persistentFailureStartedAt).toBeNull(); + expect(auditLog).toHaveBeenCalledWith( + "integrations.sync.failed", + expect.objectContaining({ + details: expect.objectContaining({ + kind: "transient", + escalatedFrom: null, + }), + }), + ); + }); + + it("needs the consecutive-failure threshold as well as the clock", async () => { + // A provider polled once a day: old streak, but only two attempts in it. + seedStreak({ transient: 1, ageMs: 5 * DAY_MS }); + + await recordSyncFailure({ + userId: "u1", + integration: "nightscout", + kind: "transient", + message: "Nightscout sync HTTP 502", + }); + + expect(upsertUpdate().consecutiveFailuresByKind).toEqual({ + transient: 2, + reauth_required: 0, + persistent: 0, + }); + expect(upsertUpdate().persistentFailureStartedAt).toBeNull(); + }); + + it("dates a fresh streak from this failure and never re-dates a running one", async () => { + const anchor = new Date(Date.now() - 3 * DAY_MS); + vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({ + consecutiveFailuresByKind: { + transient: 1, + reauth_required: 0, + persistent: 0, + }, + persistentFailureStartedAt: null, + failingSinceAt: anchor, + alertedAt: new Date(), + } as never); + vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce({ + alertedAt: new Date(), + } as never); + + await recordSyncFailure({ + userId: "u1", + integration: "withings", + kind: "transient", + message: "Withings measure error: 503 - upstream", + }); + + expect(upsertUpdate().failingSinceAt).toEqual(anchor); + }); +}); + +describe("recordSyncSuccess — leg scoping", () => { + it("leaves a sibling leg's error standing and only records the attempt", async () => { + vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({ + state: "error_transient", + failingLeg: "sleep", + } as never); + vi.mocked(prisma.integrationStatus.update).mockResolvedValueOnce( + {} as never, + ); + + await recordSyncSuccess("u1", "withings", { leg: "ecg" }); + + expect(prisma.integrationStatus.upsert).not.toHaveBeenCalled(); + const args = vi.mocked(prisma.integrationStatus.update).mock.calls[0]![0]!; + // `lastSuccessAt` deliberately absent: it is what the card's + // "connected · X ago" reads, and the sleep pipe has delivered nothing. + expect(args.data).toEqual({ lastAttemptAt: expect.any(Date) }); + }); + + it("clears the row whole when the failing leg is the one that succeeded", async () => { + vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({ + state: "error_transient", + failingLeg: "sleep", + } as never); + vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce( + {} as never, + ); + + await recordSyncSuccess("u1", "withings", { leg: "sleep" }); + + expect(prisma.integrationStatus.update).not.toHaveBeenCalled(); + const args = vi.mocked(prisma.integrationStatus.upsert).mock.calls[0]![0]!; + expect(args.update).toMatchObject({ + state: "connected", + lastError: null, + failingSinceAt: null, + failingLeg: null, + }); + }); + + it("clears the row whole when the failure was never attributed to a leg", async () => { + vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({ + state: "error_transient", + failingLeg: null, + } as never); + vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce( + {} as never, + ); + + await recordSyncSuccess("u1", "oura", { leg: "vitals" }); + + expect(prisma.integrationStatus.upsert).toHaveBeenCalledOnce(); + }); + + it("clears the row whole when the caller names no leg (a full pass)", async () => { + vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce( + {} as never, + ); + + await recordSyncSuccess("u1", "whoop"); + + // No probe read at all — an unnamed success is entitled to clear. + expect(prisma.integrationStatus.findUnique).not.toHaveBeenCalled(); + expect(prisma.integrationStatus.upsert).toHaveBeenCalledOnce(); + }); + + it("does not treat a healthy row's stale leg marker as a held error", async () => { + vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({ + state: "connected", + failingLeg: "sleep", + } as never); + vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce( + {} as never, + ); + + await recordSyncSuccess("u1", "withings", { leg: "ecg" }); + + expect(prisma.integrationStatus.upsert).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/integrations/__tests__/sync-verdict.test.ts b/src/lib/integrations/__tests__/sync-verdict.test.ts index c119ec25a..ef891110e 100644 --- a/src/lib/integrations/__tests__/sync-verdict.test.ts +++ b/src/lib/integrations/__tests__/sync-verdict.test.ts @@ -22,7 +22,8 @@ import { const NOW = new Date("2026-07-25T12:00:00.000Z"); const nowMs = NOW.getTime(); const ago = (ms: number) => new Date(nowMs - ms).toISOString(); -const HOUR = 60 * 60 * 1000; +const MINUTE = 60 * 1000; +const HOUR = 60 * MINUTE; const DAY = 24 * HOUR; describe("resolveSyncVerdict — the ladder", () => { @@ -205,3 +206,67 @@ describe("resolveSyncVerdict — the ladder", () => { } }); }); + +describe("resolveSyncVerdict — the error verdicts date the streak", () => { + it("dates a failing provider from the streak start, not the last retry", () => { + // The shape that hid a fortnight of silence: the hourly retry keeps + // `lastAttemptAt` fresh, and a multi-leg provider can keep advancing + // `lastSuccessAt` from a healthy leg while another one is dead. + const failingSinceAt = ago(17 * DAY); + expect( + resolveSyncVerdict({ + connected: true, + state: "error_transient", + lastSuccessAt: ago(30 * MINUTE), + lastAttemptAt: ago(4 * MINUTE), + failingSinceAt, + cadence: POLLED_CADENCE, + now: NOW, + }), + ).toEqual({ verdict: "failing", since: failingSinceAt }); + }); + + it("falls back to the last success when the ledger has no streak anchor", () => { + // Rows written before the anchor existed, and never re-failed since. + const lastSuccessAt = ago(3 * DAY); + expect( + resolveSyncVerdict({ + connected: true, + state: "error_transient", + lastSuccessAt, + lastAttemptAt: ago(2 * HOUR), + failingSinceAt: null, + cadence: POLLED_CADENCE, + now: NOW, + }), + ).toEqual({ verdict: "failing", since: lastSuccessAt }); + }); + + it("dates a parked provider from the streak start", () => { + const failingSinceAt = ago(6 * DAY); + expect( + resolveSyncVerdict({ + connected: true, + state: "parked", + lastAttemptAt: ago(10 * MINUTE), + failingSinceAt, + cadence: POLLED_CADENCE, + now: NOW, + }), + ).toEqual({ verdict: "parked", since: failingSinceAt }); + }); + + it("dates a reauth-required provider from the streak start", () => { + const failingSinceAt = ago(9 * DAY); + expect( + resolveSyncVerdict({ + connected: true, + state: "error_reauth", + lastAttemptAt: ago(10 * MINUTE), + failingSinceAt, + cadence: POLLED_CADENCE, + now: NOW, + }), + ).toEqual({ verdict: "reauth_required", since: failingSinceAt }); + }); +}); From e0ae866f320562c9b50eb0b9bf414f3c96a658a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 01:14:20 +0200 Subject: [PATCH 08/52] test(whoop): assert the per-resource failure carries its leg --- src/lib/jobs/__tests__/whoop-resource-sync.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/jobs/__tests__/whoop-resource-sync.test.ts b/src/lib/jobs/__tests__/whoop-resource-sync.test.ts index 295ae98e6..6359d2ecb 100644 --- a/src/lib/jobs/__tests__/whoop-resource-sync.test.ts +++ b/src/lib/jobs/__tests__/whoop-resource-sync.test.ts @@ -147,6 +147,9 @@ describe("runWhoopResourceSync — WH-4 write-path ledger honesty", () => { expect(recordWhoopSyncFailureMock).toHaveBeenCalledWith( "u1", expect.objectContaining({ message: "write failed" }), + // Attributed to the leg it came from, so the recovery leg's success at + // :05 cannot clear it. + "workout", ); }); From 054eb51e96e46d2b14cff4515a43415b3b62e112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 14:19:26 +0200 Subject: [PATCH 09/52] chore(db): number the failure-streak migration behind the ones already released --- .../migration.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename prisma/migrations/{0275_integration_failure_streak_and_leg => 0277_integration_failure_streak_and_leg}/migration.sql (100%) diff --git a/prisma/migrations/0275_integration_failure_streak_and_leg/migration.sql b/prisma/migrations/0277_integration_failure_streak_and_leg/migration.sql similarity index 100% rename from prisma/migrations/0275_integration_failure_streak_and_leg/migration.sql rename to prisma/migrations/0277_integration_failure_streak_and_leg/migration.sql From a8d798442ff968bfd740b1b05f1be491f296b28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 07:20:53 +0200 Subject: [PATCH 10/52] fix(environment): name the enum type the database actually has MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration 0217 created the type as `environment_location_source`; the datamodel declared `EnvironmentLocationSource` with no `@@map`, so the client compiled every write against `public."EnvironmentLocationSource"`. Postgres answered 42704, the upsert rolled the transaction back, and the module sat at "0 days recorded" from the day it shipped — one enum out of forty-nine, and the only one without a map. No migration follows. `prisma migrate diff` against a database built from the committed migrations reports the enum rename and the `environment_contexts.source` column rebuild before the map and nothing after it: the database was never the side that drifted, and renaming a live type to match a datamodel typo would be the wrong direction. Two guards, because the two questions are different. The unit guard replays CREATE TYPE / ALTER TYPE RENAME / DROP TYPE across the migration set and holds every enum to a type that survives it. The integration guard compares all three name classes Prisma emits — enum type, table, column — against a real migrated Postgres, which is the only place a name mismatch is observable at all: it survives generation, typecheck, and every mocked test untouched. Refs #646 --- prisma/schema.prisma | 10 + src/__tests__/helpers/prisma-schema-names.ts | 121 ++++++++++ .../prisma-enum-mapping-guard.test.ts | 145 ++++++++++++ .../prisma-schema-name-drift.test.ts | 222 ++++++++++++++++++ 4 files changed, 498 insertions(+) create mode 100644 src/__tests__/helpers/prisma-schema-names.ts create mode 100644 src/__tests__/prisma-enum-mapping-guard.test.ts create mode 100644 tests/integration/prisma-schema-name-drift.test.ts diff --git a/prisma/schema.prisma b/prisma/schema.prisma index b54bdac41..1139d4d90 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -919,6 +919,16 @@ enum EnvironmentLocationSource { HOME TRAVEL DEVICE + + /// Migration 0217 created the physical type as `environment_location_source`, + /// like every other enum in this schema. This declaration carried no `@@map`, + /// so the client compiled every write against `public."EnvironmentLocationSource"` + /// — a type no database has ever had — and each insert into `environment_contexts` + /// died on `42704` before a row could commit. The map names the type that + /// exists; no migration follows, because the database was never the side that + /// drifted. `prisma-enum-mapping-guard.test.ts` now holds the whole schema to + /// this rule. + @@map("environment_location_source") } /// A per-day environmental observation (daily weather / daylight) for the diff --git a/src/__tests__/helpers/prisma-schema-names.ts b/src/__tests__/helpers/prisma-schema-names.ts new file mode 100644 index 000000000..310ecaaf3 --- /dev/null +++ b/src/__tests__/helpers/prisma-schema-names.ts @@ -0,0 +1,121 @@ +/** + * A minimal reader for the physical names `prisma/schema.prisma` declares. + * + * Prisma emits three kinds of identifier straight into the SQL it sends: the + * enum type name, the table name, and the column name. Each one is the `@@map` + * / `@map` value when present and the declared name verbatim when absent. If + * any of the three names something the database does not have, the statement + * dies at the driver with `42704` / `42P01` / `42703` — and nothing earlier in + * the pipeline can see it, because generation, typecheck, and every mocked + * unit test read the logical name only. + * + * Consumers: `prisma-enum-mapping-guard.test.ts` (the convention, no database) + * and `tests/integration/prisma-schema-name-drift.test.ts` (the physical + * comparison against a migrated Postgres). + */ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const SCHEMA_PATH = resolve( + dirname(fileURLToPath(import.meta.url)), + "../../../prisma/schema.prisma", +); + +export interface DeclaredEnum { + /** The enum name in the datamodel. */ + name: string; + /** The `@@map` value, or null when the declaration carries none. */ + mappedName: string | null; + /** The name Prisma puts in the SQL — `@@map` when present, else `name`. */ + physicalName: string; +} + +export interface DeclaredField { + name: string; + /** The `@map` value, or null when the field carries none. */ + mappedName: string | null; + /** The column name Prisma puts in the SQL. */ + physicalName: string; +} + +export interface DeclaredModel { + name: string; + mappedName: string | null; + /** The table name Prisma puts in the SQL. */ + physicalName: string; + /** Scalar + enum fields only — relation fields carry no column. */ + fields: DeclaredField[]; +} + +export interface DeclaredSchema { + enums: DeclaredEnum[]; + models: DeclaredModel[]; +} + +const BLOCK_MAP = /^\s*@@map\("([^"]+)"\)/m; +const FIELD_MAP = /@map\("([^"]+)"\)/; +/** `name Type[]? @attr…` — the two leading tokens are what matters. */ +const FIELD_LINE = /^\s*(\w+)\s+(\w+)(?:\[\])?\??\s*(.*)$/; + +/** Drop `//` and `///` lines so prose can never read as a declaration. */ +function stripComments(block: string): string { + return block + .split("\n") + .filter((line) => !/^\s*\/\//.test(line)) + .join("\n"); +} + +/** + * Read the datamodel. Deliberately regex-shaped rather than a real parser: the + * guard has to run inside `pnpm test` with no database and no Prisma engine, + * and the three declarations it reads (`enum`, `model`, field lines) are the + * most stable syntax in the file. Both consumers assert a floor on how much + * they found, so a parser that silently stops matching fails loudly instead of + * passing vacuously. + */ +export function readDeclaredSchema( + schemaPath: string = SCHEMA_PATH, +): DeclaredSchema { + const source = readFileSync(schemaPath, "utf8"); + const enums: DeclaredEnum[] = []; + const rawModels: Array<{ name: string; body: string }> = []; + + const block = /^(model|enum)\s+(\w+)\s*\{([\s\S]*?)^\}/gm; + for (let m = block.exec(source); m !== null; m = block.exec(source)) { + const [, kind, name, body] = m; + const clean = stripComments(body); + const mappedName = BLOCK_MAP.exec(clean)?.[1] ?? null; + if (kind === "enum") { + enums.push({ name, mappedName, physicalName: mappedName ?? name }); + } else { + rawModels.push({ name, body: clean }); + } + } + + const modelNames = new Set(rawModels.map((m) => m.name)); + + const models = rawModels.map(({ name, body }) => { + const mappedName = BLOCK_MAP.exec(body)?.[1] ?? null; + const fields: DeclaredField[] = []; + for (const line of body.split("\n")) { + if (/^\s*@@/.test(line)) continue; + const match = FIELD_LINE.exec(line); + if (!match) continue; + const [, fieldName, baseType, rest] = match; + // A field whose type is another model is a relation, not a column. The + // back-reference side carries no `@relation` attribute at all, so the + // type is what decides it — keying on `@relation` would miss half. + if (modelNames.has(baseType)) continue; + const fieldMapped = FIELD_MAP.exec(rest)?.[1] ?? null; + fields.push({ + name: fieldName, + mappedName: fieldMapped, + physicalName: fieldMapped ?? fieldName, + }); + } + return { name, mappedName, physicalName: mappedName ?? name, fields }; + }); + + return { enums, models }; +} diff --git a/src/__tests__/prisma-enum-mapping-guard.test.ts b/src/__tests__/prisma-enum-mapping-guard.test.ts new file mode 100644 index 000000000..94a866bac --- /dev/null +++ b/src/__tests__/prisma-enum-mapping-guard.test.ts @@ -0,0 +1,145 @@ +/** + * Structural guard — a Prisma enum cannot name a type the migrations never + * created. + * + * `EnvironmentLocationSource` was declared without an `@@map` while migration + * 0217 created the type as `environment_location_source`. Prisma writes the + * enum's physical name straight into the SQL, so every insert into + * `environment_contexts` asked Postgres for `public."EnvironmentLocationSource"` + * and got `42704`. The module read "0 days recorded" from the day it shipped. + * + * Nothing upstream could see it: the type is right in the generated client, the + * datamodel is internally consistent, and every mocked unit test only ever + * exercises the logical name. One enum out of forty-nine drifted, and it took a + * self-hoster reading Postgres error codes to find it. + * + * So the physical name is checked against the only other place that states it — + * the migration SQL. `CREATE TYPE`, `ALTER TYPE … RENAME TO`, and `DROP TYPE` + * are replayed in migration order, and every declared enum has to land on a + * type that survives to the end. The `@@map`-present rule is asserted + * separately, because an unmapped PascalCase enum is the specific mistake that + * happened and the message should say so rather than "type not found". + * + * Limits, stated plainly. This reads SQL text, not a database: it proves the + * migrations create the type, not that any particular deployment's database + * matches its migration ledger. It covers enum types only. The comprehensive + * check — every enum, table, and column name compared against a real migrated + * Postgres — is `tests/integration/prisma-schema-name-drift.test.ts`, which + * this guard exists to front-run at unit speed. + */ +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, it, expect } from "vitest"; + +import { readDeclaredSchema } from "./helpers/prisma-schema-names"; + +const MIGRATIONS_DIR = resolve( + dirname(fileURLToPath(import.meta.url)), + "../../prisma/migrations", +); + +const CREATE_TYPE = /CREATE\s+TYPE\s+(?:IF\s+NOT\s+EXISTS\s+)?"?([\w.]+)"?/gi; +const RENAME_TYPE = + /ALTER\s+TYPE\s+"?([\w.]+)"?\s+RENAME\s+TO\s+"?([\w.]+)"?/gi; +const DROP_TYPE = /DROP\s+TYPE\s+(?:IF\s+EXISTS\s+)?"?([\w.]+)"?/gi; + +/** `public.foo` and `foo` are the same type; the guard compares bare names. */ +function bare(name: string): string { + return name.includes(".") ? name.slice(name.lastIndexOf(".") + 1) : name; +} + +/** + * Statements that are commented out (the rollback recipes several migrations + * carry in a `--` block) must not count as executed SQL. + */ +function executableSql(sql: string): string { + return sql + .split("\n") + .filter((line) => !/^\s*--/.test(line)) + .join("\n"); +} + +/** Replay every migration in order and return the types left standing. */ +function liveEnumTypes(): Set { + const live = new Set(); + const dirs = readdirSync(MIGRATIONS_DIR, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort(); + + for (const dir of dirs) { + let sql: string; + try { + sql = readFileSync(join(MIGRATIONS_DIR, dir, "migration.sql"), "utf8"); + } catch { + continue; + } + const statements = executableSql(sql); + + // Order within one file matters: a migration that drops and recreates a + // type under a new name has to be replayed as written, so each statement + // kind is applied at its own offset rather than kind-by-kind. + const events: Array<{ at: number; apply: () => void }> = []; + for (const m of statements.matchAll(CREATE_TYPE)) { + const name = bare(m[1]); + events.push({ at: m.index, apply: () => void live.add(name) }); + } + for (const m of statements.matchAll(RENAME_TYPE)) { + const from = bare(m[1]); + const to = bare(m[2]); + events.push({ + at: m.index, + apply: () => { + live.delete(from); + live.add(to); + }, + }); + } + for (const m of statements.matchAll(DROP_TYPE)) { + const name = bare(m[1]); + events.push({ at: m.index, apply: () => void live.delete(name) }); + } + for (const event of events.sort((a, b) => a.at - b.at)) event.apply(); + } + return live; +} + +describe("prisma enums — the declared type is the migrated type", () => { + const { enums } = readDeclaredSchema(); + + it("reads the schema (a parser that stopped matching cannot pass vacuously)", () => { + expect(enums.length).toBeGreaterThan(40); + }); + + it("declares an @@map on every enum", () => { + const unmapped = enums + .filter((declared) => declared.mappedName === null) + .map((declared) => declared.name); + expect( + unmapped, + unmapped.length > 0 + ? `Enum(s) without @@map — Prisma will ask Postgres for the PascalCase ` + + `type name, which no migration creates: ${unmapped.join(", ")}` + : undefined, + ).toEqual([]); + }); + + it("finds the migrated types (a broken replay cannot pass vacuously)", () => { + expect(liveEnumTypes().size).toBeGreaterThan(40); + }); + + it("points every enum at a type the migrations leave standing", () => { + const live = liveEnumTypes(); + const missing = enums + .filter((declared) => !live.has(declared.physicalName)) + .map((declared) => `${declared.name} → "${declared.physicalName}"`); + expect( + missing, + missing.length > 0 + ? `Enum(s) naming a type no migration creates:\n ${missing.join("\n ")}` + : undefined, + ).toEqual([]); + }); +}); diff --git a/tests/integration/prisma-schema-name-drift.test.ts b/tests/integration/prisma-schema-name-drift.test.ts new file mode 100644 index 000000000..9865cb93a --- /dev/null +++ b/tests/integration/prisma-schema-name-drift.test.ts @@ -0,0 +1,222 @@ +/** + * Every name `prisma/schema.prisma` will put into SQL exists in the migrated + * database. + * + * `EnvironmentLocationSource` shipped with no `@@map` while migration 0217 had + * created the type as `environment_location_source`. Prisma writes the enum's + * physical name into the statement, so every insert into `environment_contexts` + * asked for `public."EnvironmentLocationSource"` and got `42704`. The nightly + * job failed on every run from the day the module shipped and the screen read + * "0 days recorded" the whole time. + * + * The reason it survived that long is the reason this file is an integration + * test and not a unit test: generation succeeds, typecheck succeeds, and a + * mocked Prisma client never emits a type name at all. Only a real migrated + * Postgres can answer whether the datamodel's physical names exist. + * + * So it compares all three name classes Prisma emits — enum type, table, + * column — against `pg_type` and `information_schema`. One direction only: + * every DECLARED name must exist physically. The reverse is deliberately not + * asserted, because a column dropped from the datamodel but left in the + * database breaks nothing, while a name the client sends and the database + * lacks breaks every statement that touches it. + * + * Also verified here: the environment write the drift actually blocked, once + * per enum value, against the live type. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { + readDeclaredSchema, + type DeclaredModel, +} from "@/__tests__/helpers/prisma-schema-names"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const prisma = getPrismaClient(); + +async function physicalEnumTypes(): Promise> { + const rows = await prisma.$queryRaw>` + SELECT t.typname + FROM pg_type t + JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE t.typtype = 'e' AND n.nspname = 'public' + `; + return new Set(rows.map((row) => row.typname)); +} + +async function physicalColumns(): Promise>> { + const rows = await prisma.$queryRaw< + Array<{ table_name: string; column_name: string }> + >` + SELECT table_name, column_name + FROM information_schema.columns + WHERE table_schema = 'public' + `; + const byTable = new Map>(); + for (const { table_name, column_name } of rows) { + const columns = byTable.get(table_name) ?? new Set(); + columns.add(column_name); + byTable.set(table_name, columns); + } + return byTable; +} + +function describeModel(model: DeclaredModel): string { + return `${model.name} → "${model.physicalName}"`; +} + +describe("prisma datamodel — declared names exist in the migrated database", () => { + const { enums, models } = readDeclaredSchema(); + + it("read the datamodel (a parser that stopped matching cannot pass vacuously)", () => { + expect(enums.length).toBeGreaterThan(40); + expect(models.length).toBeGreaterThan(80); + expect( + models.reduce((sum, model) => sum + model.fields.length, 0), + ).toBeGreaterThan(500); + }); + + it("has every declared enum type", async () => { + const physical = await physicalEnumTypes(); + const missing = enums + .filter((declared) => !physical.has(declared.physicalName)) + .map((declared) => `${declared.name} → "${declared.physicalName}"`); + expect( + missing, + missing.length > 0 + ? `Enum type(s) the database does not have:\n ${missing.join("\n ")}` + : undefined, + ).toEqual([]); + }); + + it("has every declared table", async () => { + const byTable = await physicalColumns(); + const missing = models + .filter((model) => !byTable.has(model.physicalName)) + .map(describeModel); + expect( + missing, + missing.length > 0 + ? `Table(s) the database does not have:\n ${missing.join("\n ")}` + : undefined, + ).toEqual([]); + }); + + it("has every declared column", async () => { + const byTable = await physicalColumns(); + const missing: string[] = []; + for (const model of models) { + const columns = byTable.get(model.physicalName); + if (!columns) continue; // reported by the table assertion above + for (const field of model.fields) { + if (!columns.has(field.physicalName)) { + missing.push( + `${model.name}.${field.name} → "${model.physicalName}"."${field.physicalName}"`, + ); + } + } + } + expect( + missing, + missing.length > 0 + ? `Column(s) the database does not have:\n ${missing.join("\n ")}` + : undefined, + ).toEqual([]); + }); +}); + +describe("environment context — the write the enum drift blocked", () => { + beforeEach(async () => { + await truncateAllTables(prisma); + }); + + it("stores a row under every EnvironmentLocationSource value", async () => { + const user = await prisma.user.create({ + data: { username: "env-enum", email: "env-enum@example.test" }, + select: { id: true }, + }); + + for (const [index, source] of ( + ["HOME", "TRAVEL", "DEVICE"] as const + ).entries()) { + const date = `2026-03-0${index + 1}`; + await prisma.environmentContext.upsert({ + where: { userId_date: { userId: user.id, date } }, + create: { + userId: user.id, + date, + lat: 52.5, + lon: 13.4, + locationLabel: "Somewhere", + source, + tempMean: 12.5, + fetchedAt: new Date(), + }, + update: { source, tempMean: 12.5 }, + }); + } + + const stored = await prisma.environmentContext.findMany({ + where: { userId: user.id }, + orderBy: { date: "asc" }, + select: { date: true, source: true }, + }); + expect(stored).toEqual([ + { date: "2026-03-01", source: "HOME" }, + { date: "2026-03-02", source: "TRAVEL" }, + { date: "2026-03-03", source: "DEVICE" }, + ]); + + // Idempotent — the nightly lookback re-runs the same days every night. + await prisma.environmentContext.upsert({ + where: { userId_date: { userId: user.id, date: "2026-03-01" } }, + create: { + userId: user.id, + date: "2026-03-01", + lat: 52.5, + lon: 13.4, + locationLabel: "Somewhere", + source: "HOME", + fetchedAt: new Date(), + }, + update: { source: "TRAVEL", tempMean: 14 }, + }); + const reread = await prisma.environmentContext.findMany({ + where: { userId: user.id }, + }); + expect(reread).toHaveLength(3); + expect(reread.find((row) => row.date === "2026-03-01")?.source).toBe( + "TRAVEL", + ); + }); + + it("filters on the enum column (the read side sends the type name too)", async () => { + const user = await prisma.user.create({ + data: { username: "env-enum-read", email: "env-enum-read@example.test" }, + select: { id: true }, + }); + await prisma.environmentContext.create({ + data: { + userId: user.id, + date: "2026-04-01", + lat: 52.5, + lon: 13.4, + locationLabel: "Somewhere", + source: "TRAVEL", + fetchedAt: new Date(), + }, + }); + + await expect( + prisma.environmentContext.count({ + where: { userId: user.id, source: { in: ["HOME", "DEVICE"] } }, + }), + ).resolves.toBe(0); + await expect( + prisma.environmentContext.count({ + where: { userId: user.id, source: "TRAVEL" }, + }), + ).resolves.toBe(1); + }); +}); From f0c5b5ae94106f8fff13428abffd5a4a662d534e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 26 Jul 2026 07:46:26 +0200 Subject: [PATCH 11/52] fix(medications): identify an imported dose by when it was taken MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The intake import built its replay key from the row's optional `zaehler` field. That reads as a per-row counter — the documented example showed one — but nothing required it to be unique and nothing checked. A reported export carried the dose quantity there instead: 1 tablet, on all 28 rows. Every entry queued under the same key, one dose reached the ledger, and the other 27 came back reported as duplicates of a dose they had nothing to do with. The key is the instant now, and `zaehler` is out of the contract rather than accepted-and-ignored. Nothing is lost by dropping it: the live-row unique on `(user_id, medication_id, scheduled_for, source)` already makes an imported dose unique per instant, so a re-import of a file first imported under the old key still dedups. The telling is the other half. Two different facts were counted as one number: entries of a submission that collide with each other, and doses the ledger already holds. They are separate reasons now, grouped with a count, and the dialog renders them through the same classifier and the same outcome line as the CSV and JSON import cards — so an import that wrote nothing can no longer answer in success tone. That rule shipped for the settings import panel and this surface was not part of it; the guard now lists every import surface by path, so a fourth one has to be added deliberately. `skippedInvalid` is gone: it was hardcoded to zero on every path and reported a capability the importer never had. Alongside it, the other half of the environment report — a scheduled job that fails every run has to say so. `recordError()` increments one unnamed counter, so the admin status page could show a number and not which queue produced it, and the module's own screen showed a zero day count with nothing to distinguish "nothing stored yet" from "every run failed". pg-boss has held the answer the whole time: a job that exhausts its retries keeps its queue name, its error and its timestamp. The admin status now names the failing queues with counts and the last error, the environment overview reports its own account's failed runs, and neither needed a change at any of the thirty `recordError()` call sites — a visibility fix that has to be remembered per call site is the same failure one level up. Refs #650, #646 --- messages/de.json | 30 ++- messages/en.json | 30 ++- messages/es.json | 30 ++- messages/fr.json | 30 ++- messages/it.json | 30 ++- messages/pl.json | 30 ++- src/app/api/admin/status/route.ts | 14 + src/app/api/environment/route.ts | 75 +++--- .../fixtures/reported-intake-export.json | 30 +++ .../intake/import/__tests__/route.test.ts | 78 +++++- .../medications/[id]/intake/import/route.ts | 29 ++- src/components/admin/_shared.tsx | 15 ++ .../admin/system-status-section.tsx | 85 ++++++- .../__tests__/intake-import-result.test.tsx | 145 +++++++++++ .../medications/intake-import-dialog.tsx | 89 ++++--- .../medications/intake-import-result.tsx | 126 +++++++++ .../settings/environment-section.tsx | 23 +- .../__tests__/import-result-guard.test.ts | 75 +++++- .../medication-intake-import.test.ts | 73 +++++- src/lib/jobs/job-failures.ts | 153 +++++++++++ src/lib/jobs/medication-intake-import.ts | 132 ++++++++-- .../__tests__/intake-import-poll.test.ts | 8 +- src/lib/medications/intake-import-payload.ts | 42 +++ src/lib/medications/intake-import-poll.ts | 5 +- tests/integration/job-failures.test.ts | 151 +++++++++++ .../medication-intake-import-job.test.ts | 13 +- ...tion-intake-import-reported-export.test.ts | 239 ++++++++++++++++++ 27 files changed, 1626 insertions(+), 154 deletions(-) create mode 100644 src/app/api/medications/[id]/intake/import/__tests__/fixtures/reported-intake-export.json create mode 100644 src/components/medications/__tests__/intake-import-result.test.tsx create mode 100644 src/components/medications/intake-import-result.tsx create mode 100644 src/lib/jobs/job-failures.ts create mode 100644 src/lib/medications/intake-import-payload.ts create mode 100644 tests/integration/job-failures.test.ts create mode 100644 tests/integration/medication-intake-import-reported-export.test.ts diff --git a/messages/de.json b/messages/de.json index 861e0a940..206b9d8f2 100644 --- a/messages/de.json +++ b/messages/de.json @@ -911,9 +911,6 @@ "importFileLoaded": "Datei \"{name}\" geladen", "importInvalidJson": "Datei enthält kein gültiges JSON", "importNoArray": "Kein Array gefunden", - "importResult": "{imported} Einnahmen importiert", - "importDuplicatesSkipped": "{count} Duplikate übersprungen", - "importInvalidSkipped": "{count} ungültige Einträge übersprungen", "importFailed": "Import fehlgeschlagen", "importInvalidFormat": "Ungültiges JSON-Format", "requestExample": "Request-Beispiel", @@ -1909,6 +1906,18 @@ "scaleAbsolute": "Absolut", "scalePercentSinceStart": "% seit Start", "scaleToggleLabel": "Diagrammskala" + }, + "import": { + "resultSuccess": "{imported} Einnahmen importiert.", + "resultPartial": "{imported} importiert, {skipped} übersprungen.", + "resultNothing": "Nichts importiert: alle {skipped} Einträge wurden übersprungen.", + "resultEmpty": "Diese Datei enthielt keine Einnahmen.", + "skipGroupOne": "{count} Eintrag übersprungen: {reason}", + "skipGroupFew": "{count} Einträge übersprungen: {reason}", + "skipGroupOther": "{count} Einträge übersprungen: {reason}", + "reasonDuplicateInFile": "ein anderer Eintrag derselben Datei hat dasselbe Datum und dieselbe Uhrzeit", + "reasonAlreadyRecorded": "für dieses Datum und diese Uhrzeit ist bereits eine Einnahme erfasst", + "reasonUnknown": "Grund nicht erfasst" } }, "charts": { @@ -4661,7 +4670,10 @@ "description": "Rufe Wetter für einen vergangenen Zeitraum ab und speichere es, damit ältere Tage in die Korrelationen einfließen. Der Start steht standardmäßig auf dem Gültigkeitsdatum deines Wohnorts; um weiter zurückzureichen, füge zuerst einen Ortszeitraum für diesen Bereich hinzu.", "run": "Nacherfassen" }, - "noResults": "Keine passenden Orte gefunden." + "noResults": "Keine passenden Orte gefunden.", + "fetchFailedOne": "Die letzte Wetteraktualisierung im Hintergrund ist fehlgeschlagen ({when}).", + "fetchFailedFew": "Die letzten {count} Wetteraktualisierungen im Hintergrund sind fehlgeschlagen, zuletzt {when}.", + "fetchFailedOther": "Die letzten {count} Wetteraktualisierungen im Hintergrund sind fehlgeschlagen, zuletzt {when}." } }, "profile": "Profil", @@ -6289,6 +6301,16 @@ "connected": "Gemeinsame KI-Verbindung verknüpft", "disconnected": "Gemeinsame KI-Verbindung entfernt", "error": "Etwas ist schiefgelaufen — bitte erneut versuchen" + }, + "failingJobs": { + "title": "Hintergrundjobs", + "description": "Queues, deren Jobs nach allen Wiederholungen aufgegeben haben.", + "none": "In den letzten {hours} Stunden ist kein Job fehlgeschlagen.", + "unavailable": "Keine Job-Queue lesbar — dieses Deployment läuft ohne Hintergrund-Worker.", + "countOne": "{count} Fehlschlag", + "countFew": "{count} Fehlschläge", + "countOther": "{count} Fehlschläge", + "lastFailed": "Letzter Fehlschlag {when}" } }, "achievements": { diff --git a/messages/en.json b/messages/en.json index 7edc56190..48bfe097b 100644 --- a/messages/en.json +++ b/messages/en.json @@ -911,9 +911,6 @@ "importFileLoaded": "File \"{name}\" loaded", "importInvalidJson": "File does not contain valid JSON", "importNoArray": "No array found", - "importResult": "{imported} intakes imported", - "importDuplicatesSkipped": "{count} duplicates skipped", - "importInvalidSkipped": "{count} invalid entries skipped", "importFailed": "Import failed", "importInvalidFormat": "Invalid JSON format", "requestExample": "Request example", @@ -1909,6 +1906,18 @@ "scaleAbsolute": "Absolute", "scalePercentSinceStart": "% since start", "scaleToggleLabel": "Chart scale" + }, + "import": { + "resultSuccess": "{imported} intakes imported.", + "resultPartial": "{imported} imported, {skipped} skipped.", + "resultNothing": "Nothing was imported: all {skipped} entries were skipped.", + "resultEmpty": "That file carried no intakes.", + "skipGroupOne": "{count} entry skipped: {reason}", + "skipGroupFew": "{count} entries skipped: {reason}", + "skipGroupOther": "{count} entries skipped: {reason}", + "reasonDuplicateInFile": "another entry in the same file has the same date and time", + "reasonAlreadyRecorded": "an intake is already recorded for that date and time", + "reasonUnknown": "reason not recorded" } }, "charts": { @@ -4661,7 +4670,10 @@ "description": "Fetch and store weather for a past date range so older days join the correlations. The start defaults to your home's effective date; to reach further back, add a location period for that range first.", "run": "Backfill" }, - "noResults": "No matching places found." + "noResults": "No matching places found.", + "fetchFailedOne": "The last background weather update failed ({when}).", + "fetchFailedFew": "The last {count} background weather updates failed, most recently {when}.", + "fetchFailedOther": "The last {count} background weather updates failed, most recently {when}." } }, "profile": "Profile", @@ -6289,6 +6301,16 @@ "connected": "Shared AI connection linked", "disconnected": "Shared AI connection removed", "error": "Something went wrong — please try again" + }, + "failingJobs": { + "title": "Background jobs", + "description": "Queues whose jobs gave up after exhausting their retries.", + "none": "No job has failed in the last {hours} hours.", + "unavailable": "No job queue to read — this deployment runs without the background worker.", + "countOne": "{count} failure", + "countFew": "{count} failures", + "countOther": "{count} failures", + "lastFailed": "Last failure {when}" } }, "achievements": { diff --git a/messages/es.json b/messages/es.json index a4c955d17..fcad5dafa 100644 --- a/messages/es.json +++ b/messages/es.json @@ -911,9 +911,6 @@ "importFileLoaded": "Archivo \"{name}\" cargado", "importInvalidJson": "El archivo no contiene un JSON válido", "importNoArray": "No se encontró ningún array", - "importResult": "{imported} tomas importadas", - "importDuplicatesSkipped": "{count} duplicados omitidos", - "importInvalidSkipped": "{count} entradas no válidas omitidas", "importFailed": "Falló la importación", "importInvalidFormat": "Formato JSON no válido", "requestExample": "Ejemplo de solicitud", @@ -1909,6 +1906,18 @@ "scaleAbsolute": "Absoluto", "scalePercentSinceStart": "% desde el inicio", "scaleToggleLabel": "Escala del gráfico" + }, + "import": { + "resultSuccess": "{imported} tomas importadas.", + "resultPartial": "{imported} importadas, {skipped} omitidas.", + "resultNothing": "No se importó nada: se omitieron las {skipped} entradas.", + "resultEmpty": "Ese archivo no contenía ninguna toma.", + "skipGroupOne": "{count} entrada omitida: {reason}", + "skipGroupFew": "{count} entradas omitidas: {reason}", + "skipGroupOther": "{count} entradas omitidas: {reason}", + "reasonDuplicateInFile": "otra entrada del mismo archivo tiene la misma fecha y hora", + "reasonAlreadyRecorded": "ya hay una toma registrada para esa fecha y hora", + "reasonUnknown": "motivo no registrado" } }, "charts": { @@ -4661,7 +4670,10 @@ "description": "Obtén y guarda el tiempo de un rango de fechas pasado para que los días antiguos entren en las correlaciones. El inicio toma por defecto la fecha de vigencia de tu hogar; para llegar más atrás, añade primero un periodo de ubicación para ese rango.", "run": "Rellenar" }, - "noResults": "No se encontraron lugares coincidentes." + "noResults": "No se encontraron lugares coincidentes.", + "fetchFailedOne": "La última actualización meteorológica en segundo plano falló ({when}).", + "fetchFailedFew": "Las últimas {count} actualizaciones meteorológicas en segundo plano fallaron, la más reciente {when}.", + "fetchFailedOther": "Las últimas {count} actualizaciones meteorológicas en segundo plano fallaron, la más reciente {when}." } }, "profile": "Perfil", @@ -6289,6 +6301,16 @@ "connected": "Conexión de IA compartida vinculada", "disconnected": "Conexión de IA compartida eliminada", "error": "Algo salió mal: inténtalo de nuevo" + }, + "failingJobs": { + "title": "Trabajos en segundo plano", + "description": "Colas cuyos trabajos se rindieron tras agotar sus reintentos.", + "none": "Ningún trabajo ha fallado en las últimas {hours} horas.", + "unavailable": "No hay ninguna cola que consultar: este despliegue funciona sin el worker en segundo plano.", + "countOne": "{count} fallo", + "countFew": "{count} fallos", + "countOther": "{count} fallos", + "lastFailed": "Último fallo {when}" } }, "achievements": { diff --git a/messages/fr.json b/messages/fr.json index cd3cc3217..5d2e86ce7 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -911,9 +911,6 @@ "importFileLoaded": "Fichier « {name} » chargé", "importInvalidJson": "Le fichier ne contient pas de JSON valide", "importNoArray": "Aucun tableau trouvé", - "importResult": "{imported} prises importées", - "importDuplicatesSkipped": "{count} doublons ignorés", - "importInvalidSkipped": "{count} entrées non valides ignorées", "importFailed": "Échec de l’importation", "importInvalidFormat": "Format JSON non valide", "requestExample": "Exemple de requête", @@ -1909,6 +1906,18 @@ "scaleAbsolute": "Absolu", "scalePercentSinceStart": "% depuis le début", "scaleToggleLabel": "Échelle du graphique" + }, + "import": { + "resultSuccess": "{imported} prises importées.", + "resultPartial": "{imported} importées, {skipped} ignorées.", + "resultNothing": "Rien n’a été importé : les {skipped} entrées ont été ignorées.", + "resultEmpty": "Ce fichier ne contenait aucune prise.", + "skipGroupOne": "{count} entrée ignorée : {reason}", + "skipGroupFew": "{count} entrées ignorées : {reason}", + "skipGroupOther": "{count} entrées ignorées : {reason}", + "reasonDuplicateInFile": "une autre entrée du même fichier porte les mêmes date et heure", + "reasonAlreadyRecorded": "une prise est déjà enregistrée pour cette date et cette heure", + "reasonUnknown": "motif non enregistré" } }, "charts": { @@ -4661,7 +4670,10 @@ "description": "Récupérez et enregistrez la météo d'une plage de dates passée pour que les jours anciens rejoignent les corrélations. Le début reprend par défaut la date d'effet de votre domicile ; pour remonter plus loin, ajoutez d'abord une période de localisation pour cette plage.", "run": "Récupérer" }, - "noResults": "Aucun lieu correspondant trouvé." + "noResults": "Aucun lieu correspondant trouvé.", + "fetchFailedOne": "La dernière mise à jour météo en arrière-plan a échoué ({when}).", + "fetchFailedFew": "Les {count} dernières mises à jour météo en arrière-plan ont échoué, la plus récente {when}.", + "fetchFailedOther": "Les {count} dernières mises à jour météo en arrière-plan ont échoué, la plus récente {when}." } }, "profile": "Profil", @@ -6289,6 +6301,16 @@ "connected": "Connexion IA partagée reliée", "disconnected": "Connexion IA partagée supprimée", "error": "Une erreur s'est produite — veuillez réessayer" + }, + "failingJobs": { + "title": "Tâches en arrière-plan", + "description": "Files dont les tâches ont abandonné après avoir épuisé leurs tentatives.", + "none": "Aucune tâche n’a échoué au cours des {hours} dernières heures.", + "unavailable": "Aucune file à consulter : ce déploiement fonctionne sans le worker en arrière-plan.", + "countOne": "{count} échec", + "countFew": "{count} échecs", + "countOther": "{count} échecs", + "lastFailed": "Dernier échec {when}" } }, "achievements": { diff --git a/messages/it.json b/messages/it.json index 858cab7f8..d838baa38 100644 --- a/messages/it.json +++ b/messages/it.json @@ -911,9 +911,6 @@ "importFileLoaded": "File \"{name}\" caricato", "importInvalidJson": "Il file non contiene un JSON valido", "importNoArray": "Nessun array trovato", - "importResult": "{imported} assunzioni importate", - "importDuplicatesSkipped": "{count} duplicati ignorati", - "importInvalidSkipped": "{count} voci non valide ignorate", "importFailed": "Importazione non riuscita", "importInvalidFormat": "Formato JSON non valido", "requestExample": "Esempio di richiesta", @@ -1909,6 +1906,18 @@ "scaleAbsolute": "Assoluto", "scalePercentSinceStart": "% dall'inizio", "scaleToggleLabel": "Scala del grafico" + }, + "import": { + "resultSuccess": "{imported} assunzioni importate.", + "resultPartial": "{imported} importate, {skipped} ignorate.", + "resultNothing": "Non è stato importato nulla: tutte le {skipped} voci sono state ignorate.", + "resultEmpty": "Questo file non conteneva assunzioni.", + "skipGroupOne": "{count} voce ignorata: {reason}", + "skipGroupFew": "{count} voci ignorate: {reason}", + "skipGroupOther": "{count} voci ignorate: {reason}", + "reasonDuplicateInFile": "un’altra voce dello stesso file ha la stessa data e la stessa ora", + "reasonAlreadyRecorded": "per quella data e ora è già registrata un’assunzione", + "reasonUnknown": "motivo non registrato" } }, "charts": { @@ -4661,7 +4670,10 @@ "description": "Recupera e salva il meteo di un intervallo di date passato così i giorni più vecchi entrano nelle correlazioni. L'inizio è impostato in modo predefinito sulla data di validità della tua residenza; per andare più indietro, aggiungi prima un periodo di posizione per quell'intervallo.", "run": "Recupera" }, - "noResults": "Nessun luogo corrispondente trovato." + "noResults": "Nessun luogo corrispondente trovato.", + "fetchFailedOne": "L’ultimo aggiornamento meteo in background non è riuscito ({when}).", + "fetchFailedFew": "Gli ultimi {count} aggiornamenti meteo in background non sono riusciti, il più recente {when}.", + "fetchFailedOther": "Gli ultimi {count} aggiornamenti meteo in background non sono riusciti, il più recente {when}." } }, "profile": "Profilo", @@ -6289,6 +6301,16 @@ "connected": "Connessione IA condivisa collegata", "disconnected": "Connessione IA condivisa rimossa", "error": "Qualcosa è andato storto — riprova" + }, + "failingJobs": { + "title": "Processi in background", + "description": "Code i cui processi hanno rinunciato dopo aver esaurito i tentativi.", + "none": "Nessun processo è fallito nelle ultime {hours} ore.", + "unavailable": "Nessuna coda da leggere: questo deployment funziona senza il worker in background.", + "countOne": "{count} errore", + "countFew": "{count} errori", + "countOther": "{count} errori", + "lastFailed": "Ultimo errore {when}" } }, "achievements": { diff --git a/messages/pl.json b/messages/pl.json index 4a8f61594..2fd455604 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -911,9 +911,6 @@ "importFileLoaded": "Plik „{name}” wczytany", "importInvalidJson": "Plik nie zawiera prawidłowego JSON", "importNoArray": "Nie znaleziono tablicy", - "importResult": "{imported} przyjęć zaimportowano", - "importDuplicatesSkipped": "{count} duplikatów pominięto", - "importInvalidSkipped": "{count} nieprawidłowych wpisów pominięto", "importFailed": "Import nie powiódł się", "importInvalidFormat": "Nieprawidłowy format JSON", "requestExample": "Przykład żądania", @@ -1909,6 +1906,18 @@ "scaleAbsolute": "Bezwzględne", "scalePercentSinceStart": "% od początku", "scaleToggleLabel": "Skala wykresu" + }, + "import": { + "resultSuccess": "Zaimportowano przyjęcia: {imported}.", + "resultPartial": "Zaimportowano {imported}, pominięto {skipped}.", + "resultNothing": "Nic nie zaimportowano: pominięto wszystkie wpisy ({skipped}).", + "resultEmpty": "Ten plik nie zawierał żadnych przyjęć.", + "skipGroupOne": "Pominięto {count} wpis: {reason}", + "skipGroupFew": "Pominięto {count} wpisy: {reason}", + "skipGroupOther": "Pominięto {count} wpisów: {reason}", + "reasonDuplicateInFile": "inny wpis w tym samym pliku ma tę samą datę i godzinę", + "reasonAlreadyRecorded": "przyjęcie o tej dacie i godzinie jest już zapisane", + "reasonUnknown": "powód nieodnotowany" } }, "charts": { @@ -4661,7 +4670,10 @@ "description": "Pobierz i zapisz pogodę dla minionego zakresu dat, aby starsze dni weszły do korelacji. Początek domyślnie przyjmuje datę obowiązywania Twojego domu; aby sięgnąć dalej wstecz, najpierw dodaj okres lokalizacji dla tego zakresu.", "run": "Uzupełnij" }, - "noResults": "Nie znaleziono pasujących miejsc." + "noResults": "Nie znaleziono pasujących miejsc.", + "fetchFailedOne": "Ostatnia aktualizacja pogody w tle nie powiodła się ({when}).", + "fetchFailedFew": "Ostatnie {count} aktualizacje pogody w tle nie powiodły się, najnowsza {when}.", + "fetchFailedOther": "Ostatnich {count} aktualizacji pogody w tle nie powiodło się, najnowsza {when}." } }, "profile": "Profil", @@ -6289,6 +6301,16 @@ "connected": "Wspólne połączenie AI podłączone", "disconnected": "Wspólne połączenie AI usunięte", "error": "Coś poszło nie tak — spróbuj ponownie" + }, + "failingJobs": { + "title": "Zadania w tle", + "description": "Kolejki, których zadania poddały się po wyczerpaniu ponowień.", + "none": "W ciągu ostatnich {hours} godzin żadne zadanie nie zawiodło.", + "unavailable": "Brak kolejki zadań do odczytu — to wdrożenie działa bez workera w tle.", + "countOne": "{count} niepowodzenie", + "countFew": "{count} niepowodzenia", + "countOther": "{count} niepowodzeń", + "lastFailed": "Ostatnie niepowodzenie {when}" } }, "achievements": { diff --git a/src/app/api/admin/status/route.ts b/src/app/api/admin/status/route.ts index 15ae3331b..0633e4262 100644 --- a/src/app/api/admin/status/route.ts +++ b/src/app/api/admin/status/route.ts @@ -3,6 +3,10 @@ import { apiHandler, requireAdmin } from "@/lib/api-handler"; import { apiSuccess } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; import { getWorkerStatus } from "@/lib/jobs/worker-status"; +import { + JOB_FAILURE_WINDOW_HOURS, + readFailingQueues, +} from "@/lib/jobs/job-failures"; import { readFileSync } from "node:fs"; // Build timestamp (set at build time via next.config) @@ -57,6 +61,12 @@ export const GET = apiHandler(async () => { appSettings?.webPushVapidSubject, ); const workerStatus = getWorkerStatus(); + // `worker.errors` is one unnamed counter, which is how a queue that failed on + // every run for months could sit behind a number nobody could act on. The + // queue itself knows which job broke and why, so that is what the status + // reports. `null` = no queue schema to ask (web-only deployment); an empty + // array = asked, nothing is failing. + const failingJobs = await readFailingQueues(); return apiSuccess({ version: process.env.npm_package_version ?? "0.1.0", @@ -66,6 +76,10 @@ export const GET = apiHandler(async () => { startTime: START_TIME, database: "connected", worker: workerStatus, + failingJobs: + failingJobs === null + ? null + : { windowHours: JOB_FAILURE_WINDOW_HOURS, queues: failingJobs }, counts: { users: userCount, measurements: measurementCount, diff --git a/src/app/api/environment/route.ts b/src/app/api/environment/route.ts index 2d6e1ceb2..bb447e24b 100644 --- a/src/app/api/environment/route.ts +++ b/src/app/api/environment/route.ts @@ -12,6 +12,8 @@ import { annotate } from "@/lib/logging/context"; import { requireModuleEnabled } from "@/lib/modules/gate"; import { prisma } from "@/lib/db"; import { OPEN_METEO_ATTRIBUTION } from "@/lib/environment/open-meteo"; +import { ENVIRONMENT_FETCH_QUEUE } from "@/lib/jobs/environment-fetch"; +import { readQueueFailureForUser } from "@/lib/jobs/job-failures"; export const dynamic = "force-dynamic"; @@ -21,37 +23,43 @@ export const GET = apiHandler(async () => { const gate = await requireModuleEnabled(user.id, "environment"); if (!gate.enabled) return gate.response; - const [profile, travel, contextCount, latest] = await Promise.all([ - prisma.user.findUnique({ - where: { id: user.id }, - select: { - homeLat: true, - homeLon: true, - homeLabel: true, - homeTimezone: true, - homeSince: true, - timezone: true, - }, - }), - prisma.environmentTravelLocation.findMany({ - where: { userId: user.id }, - orderBy: { startDate: "desc" }, - select: { - id: true, - startDate: true, - endDate: true, - lat: true, - lon: true, - label: true, - }, - }), - prisma.environmentContext.count({ where: { userId: user.id } }), - prisma.environmentContext.findFirst({ - where: { userId: user.id }, - orderBy: { date: "desc" }, - select: { date: true, fetchedAt: true }, - }), - ]); + const [profile, travel, contextCount, latest, lastFetchFailure] = + await Promise.all([ + prisma.user.findUnique({ + where: { id: user.id }, + select: { + homeLat: true, + homeLon: true, + homeLabel: true, + homeTimezone: true, + homeSince: true, + timezone: true, + }, + }), + prisma.environmentTravelLocation.findMany({ + where: { userId: user.id }, + orderBy: { startDate: "desc" }, + select: { + id: true, + startDate: true, + endDate: true, + lat: true, + lon: true, + label: true, + }, + }), + prisma.environmentContext.count({ where: { userId: user.id } }), + prisma.environmentContext.findFirst({ + where: { userId: user.id }, + orderBy: { date: "desc" }, + select: { date: true, fetchedAt: true }, + }), + // A day count on its own cannot tell an empty module from a broken one — + // which is exactly how a job that failed on every run since the module + // shipped could render as "0 days recorded" and nothing else. The queue + // knows; the overview asks it. + readQueueFailureForUser(ENVIRONMENT_FETCH_QUEUE, user.id), + ]); const home = profile?.homeLat != null && profile?.homeLon != null @@ -72,6 +80,7 @@ export const GET = apiHandler(async () => { has_home: home !== null, travel_count: travel.length, context_days: contextCount, + last_fetch_failed: lastFetchFailure !== null, }, }); @@ -83,6 +92,10 @@ export const GET = apiHandler(async () => { latestDate: latest?.date ?? null, latestFetchedAt: latest?.fetchedAt?.toISOString() ?? null, }, + // null = the last background runs did not fail (or there is no queue to + // ask). Non-null names the failure so a zero day count is never left to + // stand alone. No error text: that message is written for an operator. + lastFetchFailure, attribution: OPEN_METEO_ATTRIBUTION, }); }); diff --git a/src/app/api/medications/[id]/intake/import/__tests__/fixtures/reported-intake-export.json b/src/app/api/medications/[id]/intake/import/__tests__/fixtures/reported-intake-export.json new file mode 100644 index 000000000..14463721c --- /dev/null +++ b/src/app/api/medications/[id]/intake/import/__tests__/fixtures/reported-intake-export.json @@ -0,0 +1,30 @@ +[ + { "datum": "2026-05-31", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-01", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-02", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-03", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-04", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-05", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-06", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-07", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-08", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-09", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-10", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-11", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-12", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-13", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-14", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-15", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-16", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-17", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-18", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-19", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-20", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-21", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-22", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-23", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-24", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-25", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-26", "uhrzeit": "08:30:00", "zaehler": 1.0 }, + { "datum": "2026-06-27", "uhrzeit": "08:30:00", "zaehler": 1.0 } +] diff --git a/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts b/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts index 92d3e92f6..a11c60a78 100644 --- a/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts +++ b/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts @@ -4,6 +4,9 @@ * The request process validates and persists the complete payload, then * returns a polling handle without performing intake writes itself. */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; @@ -233,14 +236,17 @@ describe("POST /api/medications/[id]/intake/import — durable kickoff", () => { userId: "user-1", medicationId: "m1", status: "queued", + // A legacy `zaehler` on the first row rides along and decides nothing: + // both keys come from the instant, so two different times can never + // collapse onto one key again. payload: { entries: [ { - idempotencyKey: "import-m1-41", + idempotencyKey: `import-m1-${Date.parse("2026-01-01T07:00:00")}`, takenAt: expect.any(String), }, { - idempotencyKey: expect.stringMatching(/^import-m1-\d+$/), + idempotencyKey: `import-m1-${Date.parse("2026-01-01T19:00:00")}`, takenAt: expect.any(String), }, ], @@ -248,7 +254,7 @@ describe("POST /api/medications/[id]/intake/import — durable kickoff", () => { progress: expect.objectContaining({ processed: 0, imported: 0, - skippedDuplicates: 0, + skippedByReason: {}, total: 2, }), }), @@ -412,3 +418,69 @@ describe("POST /api/medications/[id]/intake/import — future bound and limiter" ); }); }); + +describe("POST /api/medications/[id]/intake/import — the reported export (#650)", () => { + // The file attached to the report: 28 doses, one per day, all at 08:30, and + // `"zaehler": 1.0` on every single row — a dose quantity, not a row counter. + // The old key derivation used that field verbatim, so all 28 entries queued + // under `import-m1-1`, one row was written and 27 were reported back as + // duplicates of each other. + const REPORTED_EXPORT = JSON.parse( + readFileSync( + join(__dirname, "fixtures", "reported-intake-export.json"), + "utf8", + ), + ) as Array<{ datum: string; uhrzeit: string; zaehler: number }>; + + it("is the reported shape (a fixture that drifted would prove nothing)", () => { + expect(REPORTED_EXPORT).toHaveLength(28); + expect(new Set(REPORTED_EXPORT.map((row) => row.zaehler)).size).toBe(1); + expect( + new Set(REPORTED_EXPORT.map((row) => `${row.datum}T${row.uhrzeit}`)).size, + ).toBe(28); + }); + + it("queues 28 distinct doses instead of collapsing them onto one key", async () => { + const res = await POST(postReq(REPORTED_EXPORT), ROUTE_CTX); + + expect(res.status).toBe(202); + const created = vi.mocked(prisma.medicationIntakeImportJob.create).mock + .calls[0][0] as unknown as { + data: { + payload: { + entries: Array<{ takenAt: string; idempotencyKey: string }>; + }; + }; + }; + const entries = created.data.payload.entries; + expect(entries).toHaveLength(28); + expect(new Set(entries.map((entry) => entry.idempotencyKey)).size).toBe(28); + expect(new Set(entries.map((entry) => entry.takenAt)).size).toBe(28); + // The key is a function of the instant and nothing else. + for (const [index, entry] of entries.entries()) { + const row = REPORTED_EXPORT[index]; + const instant = Date.parse(`${row.datum}T${row.uhrzeit}`); + expect(entry.idempotencyKey).toBe(`import-m1-${instant}`); + expect(entry.takenAt).toBe(new Date(instant).toISOString()); + } + }); + + it("keys two rows at the same instant identically, whatever else they carry", async () => { + await POST( + postReq([ + { datum: "2026-03-01", uhrzeit: "08:00:00", zaehler: 7 }, + { datum: "2026-03-01", uhrzeit: "08:00:00", zaehler: 9 }, + ]), + ROUTE_CTX, + ); + + const created = vi.mocked(prisma.medicationIntakeImportJob.create).mock + .calls[0][0] as unknown as { + data: { payload: { entries: Array<{ idempotencyKey: string }> } }; + }; + const keys = created.data.payload.entries.map( + (entry) => entry.idempotencyKey, + ); + expect(keys[0]).toBe(keys[1]); + }); +}); diff --git a/src/app/api/medications/[id]/intake/import/route.ts b/src/app/api/medications/[id]/intake/import/route.ts index 71ad1f28e..12020d103 100644 --- a/src/app/api/medications/[id]/intake/import/route.ts +++ b/src/app/api/medications/[id]/intake/import/route.ts @@ -21,14 +21,31 @@ import { type MedicationImportProgress, } from "@/lib/jobs/medication-intake-import"; import { annotate } from "@/lib/logging/context"; +import { buildMedicationImportEntries } from "@/lib/medications/intake-import-payload"; import { assertMedicationOwnership } from "@/lib/medications/route-guards"; import { checkRateLimit } from "@/lib/rate-limit"; import { isPlausibleEntryInstant } from "@/lib/validations/entry-instant"; +// The entry shape carried a third field, `zaehler`, and the replay key was +// built from it whenever it was present. That reads as a per-row counter, +// which is what the documented example showed — but the field was optional, +// its uniqueness was never stated and never checked, and an export that puts +// the dose quantity there (1 tablet, on every row) collapsed a month of +// history onto one key: the first row imported, every later one lost the +// `(userId, idempotencyKey)` unique and came back reported as a duplicate of +// a dose it had nothing to do with. +// +// `zaehler` is gone from the contract rather than accepted-and-ignored. A +// payload that still carries it imports unchanged — `z.object` drops +// unrecognised keys — it simply no longer decides anything. The key now comes +// from the instant (`buildMedicationImportEntries`), which is also the grain +// the database already enforces: the live-row unique on +// `(user_id, medication_id, scheduled_for, source)` makes an imported dose +// unique per instant whatever the key says, so a re-import of a file first +// imported under the old key still dedups. const importEntrySchema = z.object({ datum: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), uhrzeit: z.string().regex(/^\d{2}:\d{2}:\d{2}$/), - zaehler: z.union([z.number().int(), z.string()]).optional(), }); function isValidImportTimestamp(datum: string, uhrzeit: string): boolean { @@ -149,19 +166,13 @@ export const POST = apiHandler( const entries = parsed.data; const normalized: MedicationImportPayload = { - entries: entries.map((entry) => { - const takenAt = new Date(`${entry.datum}T${entry.uhrzeit}`); - const idempotencyKey = entry.zaehler - ? `import-${id}-${String(entry.zaehler)}` - : `import-${id}-${takenAt.getTime()}`; - return { takenAt: takenAt.toISOString(), idempotencyKey }; - }), + entries: buildMedicationImportEntries(id, entries), }; const progress: MedicationImportProgress = { processed: 0, total: normalized.entries.length, imported: 0, - skippedDuplicates: 0, + skippedByReason: {}, touchedDays: [], rollupProcessed: 0, }; diff --git a/src/components/admin/_shared.tsx b/src/components/admin/_shared.tsx index e7d143c75..8ff11b29b 100644 --- a/src/components/admin/_shared.tsx +++ b/src/components/admin/_shared.tsx @@ -45,6 +45,14 @@ export interface WorkerStatus { errors: number; } +/** One queue with exhausted-retry failures, as `/api/admin/status` reports it. */ +export interface FailingQueue { + queue: string; + failures: number; + lastFailedAt: string; + lastError: string; +} + export interface SystemStatus { version: string; nodeVersion: string; @@ -53,6 +61,13 @@ export interface SystemStatus { startTime: string; database: string; worker: WorkerStatus; + /** + * Queues whose jobs exhausted their retries recently. `null` means the + * deployment has no pg-boss schema to ask (web-only); an empty `queues` + * array means it was asked and nothing is failing. The two are rendered + * differently on purpose — absence is not health. + */ + failingJobs: { windowHours: number; queues: FailingQueue[] } | null; counts: { users: number; measurements: number; diff --git a/src/components/admin/system-status-section.tsx b/src/components/admin/system-status-section.tsx index 053789cc1..74a67d0d8 100644 --- a/src/components/admin/system-status-section.tsx +++ b/src/components/admin/system-status-section.tsx @@ -23,7 +23,12 @@ import { formatDateTime } from "@/lib/format"; import { SettingsCard } from "@/components/settings/settings-card"; import { SettingsCardHeader } from "@/components/settings/_card-header"; import { useFormatters, useTranslations } from "@/lib/i18n/context"; -import { StatusItem, usePublicVersion, useSystemStatus } from "./_shared"; +import { + StatusItem, + usePublicVersion, + useSystemStatus, + type FailingQueue, +} from "./_shared"; // v1.4.16 phase B3: Recharts is ~108 KiB Brotli — defer-load the host // metrics chart so the system-status page only ships it when an admin @@ -43,6 +48,83 @@ const HostMetricsChart = dynamic( }, ); +/** + * The queues whose jobs gave up, named. + * + * Before this, a background job that threw on every single run showed up as one + * digit in an unnamed `errors` counter, and the module it fed rendered an empty + * screen as if empty were the answer. A broken queue has to say its own name, + * so this card is rendered whenever there is anything to report and stays + * silent otherwise — an operator should be able to trust that a quiet card + * means quiet queues, which is why the "cannot ask" case renders its own line + * rather than looking identical to health. + */ +function FailingJobsCard({ + failingJobs, +}: { + failingJobs: { windowHours: number; queues: FailingQueue[] } | null; +}) { + const { t, tCount } = useTranslations(); + + if (failingJobs === null) { + return ( + + +

+ {t("admin.failingJobs.unavailable")} +

+ + ); + } + + return ( + + + {failingJobs.queues.length === 0 ? ( +

+ {t("admin.failingJobs.none", { hours: failingJobs.windowHours })} +

+ ) : ( +
    + {failingJobs.queues.map((queue) => ( +
  • +

    + {queue.queue}{" "} + + {tCount("admin.failingJobs.count", queue.failures, { + count: queue.failures, + })} + +

    +

    + {t("admin.failingJobs.lastFailed", { + when: formatDateTime(queue.lastFailedAt), + })} +

    + {queue.lastError.length > 0 && ( +

    + {queue.lastError} +

    + )} +
    • + ))} +
+ )} + + ); +} + export function SystemStatusSection() { const { t } = useTranslations(); const fmt = useFormatters(); @@ -216,6 +298,7 @@ export function SystemStatusSection() { )} + {status && } ); } diff --git a/src/components/medications/__tests__/intake-import-result.test.tsx b/src/components/medications/__tests__/intake-import-result.test.tsx new file mode 100644 index 000000000..640a7369c --- /dev/null +++ b/src/components/medications/__tests__/intake-import-result.test.tsx @@ -0,0 +1,145 @@ +/** + * #650 — the intake import says what actually happened. + * + * The report was a file of 28 doses where 27 were "skipped" and the dialog + * answered in success tone with a duplicate count, so nothing on screen said + * the history had not landed. The cause is fixed elsewhere; this pins the + * telling, which is what turned a recoverable problem into a silent one. + * + * Project convention is SSR-only component tests (vitest runs `node`; + * `@testing-library/react` is not installed), so the view takes its state as a + * prop and each case is rendered directly. + */ +import { describe, expect, it } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { I18nProvider } from "@/lib/i18n/context"; +import { + IntakeImportResultView, + type IntakeImportResultState, +} from "../intake-import-result"; + +/** `
  • ` elements only — the lucide icon markup carries `` tags. */ +function countListItems(html: string): number { + return html.split("
  • ").length - 1; +} + +function render(state: IntakeImportResultState): string { + return renderToStaticMarkup( + + + , + ); +} + +/** What the reported run would have produced before the key was fixed. */ +const ALL_SKIPPED: IntakeImportResultState = { + kind: "outcome", + imported: 0, + skipped: 27, + skipReasons: [{ reason: "duplicate_in_file", count: 27 }], +}; + +describe(" — an import that wrote nothing", () => { + it("renders as a failure, never a success", () => { + const html = render(ALL_SKIPPED); + expect(html).toContain('data-outcome="failed"'); + expect(html).not.toContain('data-outcome="success"'); + expect(html).toContain("Nothing was imported"); + expect(html).toContain("27"); + }); + + it("names the skip reason once with its count, not once per entry", () => { + const html = render(ALL_SKIPPED); + expect(html).toContain("27 entries skipped"); + expect(html).toContain("same date and time"); + // One list item for one reason — 27 entries must not become 27 lines. + expect(countListItems(html)).toBe(1); + }); + + it("separates the two skip reasons instead of calling both duplicates", () => { + const html = render({ + kind: "outcome", + imported: 4, + skipped: 5, + skipReasons: [ + { reason: "already_recorded", count: 3 }, + { reason: "duplicate_in_file", count: 2 }, + ], + }); + expect(html).toContain("3 entries skipped"); + expect(html).toContain("already recorded"); + expect(html).toContain("2 entries skipped"); + expect(html).toContain("another entry in the same file"); + expect(countListItems(html)).toBe(2); + }); +}); + +describe(" — the other outcomes", () => { + it("renders a mixed run as a warning, not a tick", () => { + const html = render({ + kind: "outcome", + imported: 26, + skipped: 2, + skipReasons: [{ reason: "already_recorded", count: 2 }], + }); + expect(html).toContain('data-outcome="partial"'); + expect(html).toContain("26 imported, 2 skipped"); + }); + + it("renders a clean run as a success", () => { + const html = render({ + kind: "outcome", + imported: 28, + skipped: 0, + skipReasons: [], + }); + expect(html).toContain('data-outcome="success"'); + expect(html).toContain("28 intakes imported"); + expect(countListItems(html)).toBe(0); + }); + + it("gives a file with no intakes its own message rather than either", () => { + const html = render({ + kind: "outcome", + imported: 0, + skipped: 0, + skipReasons: [], + }); + expect(html).toContain('data-outcome="empty"'); + expect(html).not.toContain('data-outcome="success"'); + expect(html).toContain("no intakes"); + }); + + it("falls back to a stated absence for an unrecognised reason", () => { + const html = render({ + kind: "outcome", + imported: 0, + skipped: 1, + skipReasons: [{ reason: "something_new", count: 1 }], + }); + expect(html).toContain("reason not recorded"); + }); +}); + +describe(" — notices are not outcomes", () => { + it("reports a loaded file in neutral tone, not as an import success", () => { + const html = render({ + kind: "notice", + tone: "info", + text: 'File "intakes.json" loaded', + }); + expect(html).toContain('data-tone="info"'); + expect(html).not.toContain("text-success"); + expect(html).not.toContain("data-outcome"); + }); + + it("reports a refusal in destructive tone", () => { + const html = render({ + kind: "notice", + tone: "error", + text: "Import failed", + }); + expect(html).toContain("text-destructive"); + }); +}); diff --git a/src/components/medications/intake-import-dialog.tsx b/src/components/medications/intake-import-dialog.tsx index 6a201bb2b..b9f2c9c57 100644 --- a/src/components/medications/intake-import-dialog.tsx +++ b/src/components/medications/intake-import-dialog.tsx @@ -21,6 +21,11 @@ import { useState } from "react"; import { useQueryClient } from "@tanstack/react-query"; import { Loader2, MoreHorizontal, RotateCcw, Upload } from "lucide-react"; +import { + IntakeImportResultView, + type IntakeImportResultState, +} from "@/components/medications/intake-import-result"; + import { Button } from "@/components/ui/button"; import { ResponsiveSheet } from "@/components/ui/responsive-sheet"; import { @@ -51,10 +56,7 @@ export function IntakeImportDialog({ const { t } = useTranslations(); const [jsonText, setJsonText] = useState(""); const [importing, setImporting] = useState(false); - const [result, setResult] = useState(null); - const [resultType, setResultType] = useState<"success" | "error" | null>( - null, - ); + const [result, setResult] = useState(null); const [selectedFileName, setSelectedFileName] = useState(null); const [fileInputKey, setFileInputKey] = useState(0); const queryClient = useQueryClient(); @@ -62,7 +64,6 @@ export function IntakeImportDialog({ function resetImportForm() { setJsonText(""); setResult(null); - setResultType(null); setSelectedFileName(null); setFileInputKey((prev) => prev + 1); } @@ -77,18 +78,25 @@ export function IntakeImportDialog({ if (!file) return; setResult(null); - setResultType(null); try { const content = await file.text(); JSON.parse(content); setJsonText(content); setSelectedFileName(file.name); - setResult(t("medications.importFileLoaded", { name: file.name })); - setResultType("success"); + // A readable file is a step, not an import. It says so in neutral tone — + // the success affordance belongs to a run that actually wrote rows. + setResult({ + kind: "notice", + tone: "info", + text: t("medications.importFileLoaded", { name: file.name }), + }); } catch { - setResult(t("medications.importInvalidJson")); - setResultType("error"); + setResult({ + kind: "notice", + tone: "error", + text: t("medications.importInvalidJson"), + }); } } @@ -96,7 +104,6 @@ export function IntakeImportDialog({ if (!medicationId || !jsonText.trim()) return; setImporting(true); setResult(null); - setResultType(null); try { let data = JSON.parse(jsonText.trim()); @@ -114,37 +121,41 @@ export function IntakeImportDialog({ statusUrl: string; }>(`/api/medications/${medicationId}/intake/import`, data); const d = await waitForMedicationIntakeImport(kickoff.statusUrl); - setResult( - t("medications.importResult", { imported: d.imported }) + - (d.skippedDuplicates > 0 - ? `, ${t("medications.importDuplicatesSkipped", { count: d.skippedDuplicates })}` - : "") + - (d.skippedInvalid > 0 - ? `, ${t("medications.importInvalidSkipped", { count: d.skippedInvalid })}` - : ""), - ); - setResultType("success"); + setResult({ + kind: "outcome", + imported: d.imported, + skipped: d.skipped, + skipReasons: d.skipReasons, + }); // An import can land today-dated doses, so clear the Today hero + // dashboard through the blessed helper (they are unmounted behind the // import dialog on the detail page). void invalidateMedicationReads(queryClient); } catch (err) { if (err instanceof MedicationIntakeImportError) { - setResult(t("medications.importFailed")); - setResultType("error"); + setResult({ + kind: "notice", + tone: "error", + text: t("medications.importFailed"), + }); return; } if (!(err instanceof ApiError)) throw err; - setResult(err.message || t("medications.importFailed")); - setResultType("error"); + setResult({ + kind: "notice", + tone: "error", + text: err.message || t("medications.importFailed"), + }); } } catch (err) { - setResult( - err instanceof SyntaxError - ? t("medications.importInvalidFormat") - : (err as Error).message || t("medications.importFailed"), - ); - setResultType("error"); + setResult({ + kind: "notice", + tone: "error", + text: + err instanceof SyntaxError + ? t("medications.importInvalidFormat") + : (err as Error).message || t("medications.importFailed"), + }); } finally { setImporting(false); } @@ -226,8 +237,8 @@ export function IntakeImportDialog({ 
               {`[
-  {"datum": "2026-02-14", "uhrzeit": "10:27:43", "zaehler": 523},
-  {"datum": "2026-02-14", "uhrzeit": "23:33:42", "zaehler": 524}
+  {"datum": "2026-02-14", "uhrzeit": "10:27:43"},
+  {"datum": "2026-02-14", "uhrzeit": "23:33:42"}
 ]`}