diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ceb9557f5..a4add73d3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,21 +10,21 @@ permissions: contents: read jobs: - # The suite outgrew a single runner: 128 files took ~13 minutes when the - # 25-minute guard was written, and the v1.37 sharing work took it to 173 - # files and past 25 minutes of wall clock, so every run died at the timeout - # with every test to that point green. The guard's own rule applies — split, - # don't raise — so the files run as two vitest shards. Each shard boots its - # own Postgres testcontainer; the timeout stays a hang detector, sized so a - # healthy half-suite clears it on a slow runner. + # The suite outgrew a single runner, and then a half-suite outgrew a slow + # one: two shards cleared in ~14 minutes on a quiet day, but with six pull + # requests running side by side the larger shard hit the same 25-minute + # guard the split was meant to escape. Three shards keep the biggest slice + # around 60 files, which clears the guard with real headroom even on the + # slowest runner observed. Each shard boots its own Postgres testcontainer; + # the timeout stays a hang detector, not a performance budget. shard: runs-on: ubuntu-latest timeout-minutes: 25 strategy: fail-fast: false matrix: - shard: [1, 2] - name: integration shard ${{ matrix.shard }}/2 + shard: [1, 2, 3] + name: integration shard ${{ matrix.shard }}/3 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -44,7 +44,7 @@ jobs: # vitest a stray positional argument and the shard flag never applies — # both "shards" then run the full suite. Proven by a paired local run # before this workflow shipped. - - run: pnpm exec vitest run --config vitest.integration.config.mts --shard=${{ matrix.shard }}/2 + - run: pnpm exec vitest run --config vitest.integration.config.mts --shard=${{ matrix.shard }}/3 env: # The integration suite boots a Postgres testcontainer and points # Prisma at it via DATABASE_URL set inside `tests/integration/setup.ts`. diff --git a/CHANGELOG.md b/CHANGELOG.md index f94ce3040..540c25071 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,19 @@ reliably the one that ran. It is selected by name now, and a copy that never arrived fails the build instead of quietly leaving another one in its place. +### Fixed + +- Mood entries now survive a restore whole. The disaster-recovery backup was + carrying only the rated factors of an entry, so every tag ticked present or + absent was missing from the file, and it was dropping the note, the entry's + own timezone and the sync counter as well. A restored entry came back without + its text, read its day boundaries under the old Berlin assumption whatever + zone it had been logged in, and could lose the next sync round to a paired + phone still holding a higher counter. The backup carries every column now, + both halves of the tag taxonomy come back, and a structural check fails the + build if a future column goes missing the same way. Backup files written + before this restore exactly as they did. + ## [1.37.0] — 2026-08-07 ### Added diff --git a/messages/de.json b/messages/de.json index 32695fc29..ee68573fc 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6703,6 +6703,7 @@ "restoreSkippedCycleSymptom": "Zyklussymptom", "restoreSkippedIllnessSymptom": "Krankheitssymptom", "restoreSkippedMoodFactor": "Stimmungsfaktor", + "restoreSkippedMoodTag": "Stimmungs-Tag", "docsLink": "Dokumentation" }, "danger-zone": { diff --git a/messages/en.json b/messages/en.json index a06f738cf..d1d6cba42 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6703,6 +6703,7 @@ "restoreSkippedCycleSymptom": "Cycle symptom", "restoreSkippedIllnessSymptom": "Illness symptom", "restoreSkippedMoodFactor": "Mood factor", + "restoreSkippedMoodTag": "Mood tag", "docsLink": "Documentation" }, "danger-zone": { diff --git a/messages/es.json b/messages/es.json index 0f6924882..53b2af25c 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6703,6 +6703,7 @@ "restoreSkippedCycleSymptom": "Síntoma del ciclo", "restoreSkippedIllnessSymptom": "Síntoma de enfermedad", "restoreSkippedMoodFactor": "Factor de ánimo", + "restoreSkippedMoodTag": "Etiqueta de ánimo", "docsLink": "Documentación" }, "danger-zone": { diff --git a/messages/fr.json b/messages/fr.json index adf2c25bf..0bfb8ebff 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6703,6 +6703,7 @@ "restoreSkippedCycleSymptom": "Symptôme de cycle", "restoreSkippedIllnessSymptom": "Symptôme de maladie", "restoreSkippedMoodFactor": "Facteur d'humeur", + "restoreSkippedMoodTag": "Étiquette d'humeur", "docsLink": "Documentation" }, "danger-zone": { diff --git a/messages/it.json b/messages/it.json index c53f19f4b..cffdc04ea 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6703,6 +6703,7 @@ "restoreSkippedCycleSymptom": "Sintomo del ciclo", "restoreSkippedIllnessSymptom": "Sintomo di malattia", "restoreSkippedMoodFactor": "Fattore dell'umore", + "restoreSkippedMoodTag": "Etichetta dell'umore", "docsLink": "Documentazione" }, "danger-zone": { diff --git a/messages/pl.json b/messages/pl.json index e1968c87a..f6fb02f75 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6703,6 +6703,7 @@ "restoreSkippedCycleSymptom": "Objaw cyklu", "restoreSkippedIllnessSymptom": "Objaw choroby", "restoreSkippedMoodFactor": "Czynnik nastroju", + "restoreSkippedMoodTag": "Tag nastroju", "docsLink": "Dokumentacja" }, "danger-zone": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 92dc14bec..4f5efaea5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,7 @@ settings: overrides: dompurify@<=3.4.10: ^3.4.11 postcss@<8.5.10: ^8.5.10 + nanoid@<3.3.17: ^3.3.17 '@hono/node-server@<1.19.13': ^1.19.13 hono@<4.12.25: ^4.12.25 picomatch@<4.0.4: ^4.0.4 @@ -5556,13 +5557,8 @@ packages: nan@2.27.0: resolution: {integrity: sha512-hC+0LidcL3XE4rp1C4H54KujgXKzbfyTngZTwBByQxsOxCEKZT0MPQ4hOKUH2jU1OYstqdDH4onyHPDzcV0XdQ==} - nanoid@3.3.12: - resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} - engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true - - nanoid@3.3.16: - resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + nanoid@3.3.17: + resolution: {integrity: sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -12636,9 +12632,7 @@ snapshots: nan@2.27.0: optional: true - nanoid@3.3.12: {} - - nanoid@3.3.16: {} + nanoid@3.3.17: {} napi-postinstall@0.3.4: {} @@ -13045,19 +13039,19 @@ snapshots: postcss@8.5.14: dependencies: - nanoid: 3.3.12 + nanoid: 3.3.17 picocolors: 1.1.1 source-map-js: 1.2.1 postcss@8.5.20: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.17 picocolors: 1.1.1 source-map-js: 1.2.1 postcss@8.5.25: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.17 picocolors: 1.1.1 source-map-js: 1.2.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c7fabca7c..3fa8ec2d3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,7 @@ overrides: "dompurify@<=3.4.10": "^3.4.11" "postcss@<8.5.10": "^8.5.10" + "nanoid@<3.3.17": "^3.3.17" "@hono/node-server@<1.19.13": "^1.19.13" "hono@<4.12.25": "^4.12.25" "picomatch@<4.0.4": "^4.0.4" diff --git a/scripts/diagnose-implausible-measurements.ts b/scripts/diagnose-implausible-measurements.ts new file mode 100644 index 000000000..0be518a26 --- /dev/null +++ b/scripts/diagnose-implausible-measurements.ts @@ -0,0 +1,150 @@ +/** + * Read-only diagnostic for stored measurement rows outside the app's own + * declared plausibility domain. + * + * `VALUE_RANGES` in `@/lib/validations/measurement` is the band the + * application says a metric can occupy. Every path a person can drive has + * always enforced it; the provider sync writers did not, so rows the + * application itself calls impossible reached the table — and every mean, + * median and personal band built from them inherited the impossibility. The + * writers enforce it now, which stops new rows landing. It says nothing about + * the rows already there. + * + * This reports them and repairs nothing. What to do with a stored impossible + * row is a judgement the data cannot make: a pulse of 55 million is plainly a + * decode slip and a candidate for deletion, while a weight of 0.9 kg might be + * a neonatal record on a shared scale, or a unit mix-up, or a real reading the + * band is simply too narrow for. Deleting somebody's health record on a + * heuristic is not a thing a script gets to decide. + * + * Output is aggregate: one line per (metric, source) bucket with the row + * count, the number of distinct records affected, the observed extremes, and + * the window they fall in. Individual readings are not printed — the bucket + * plus the metric's own declared bound is what a decision needs, and a log + * file full of somebody's vitals is not. + * + * Run (inside the app container, which has DATABASE_URL): + * pnpm dlx tsx scripts/diagnose-implausible-measurements.ts + */ +import "dotenv/config"; + +import { prisma } from "@/lib/db"; +import { VALUE_RANGES } from "@/lib/validations/measurement"; + +interface BucketRow { + type: string; + source: string; + row_count: bigint; + record_count: bigint; + min_value: number; + max_value: number; + first_at: Date; + last_at: Date; + tombstoned: bigint; +} + +function ymd(date: Date): string { + return date.toISOString().slice(0, 10); +} + +async function main(): Promise { + const types = Object.keys(VALUE_RANGES).sort(); + const mins = types.map((t) => VALUE_RANGES[t]!.min); + const maxes = types.map((t) => VALUE_RANGES[t]!.max); + + // The band table rides in as three parallel bound arrays rather than a + // spliced VALUES list, so the query carries no interpolated text at all. + // `value <> value` is the NaN probe: a double-precision column can hold one, + // and it compares false against every bound including itself. + const buckets = await prisma.$queryRaw` + WITH bands(type, min_value, max_value) AS ( + SELECT * FROM unnest( + ${types}::text[], + ${mins}::double precision[], + ${maxes}::double precision[] + ) + ) + SELECT + m."type"::text AS type, + m."source"::text AS source, + COUNT(*)::bigint AS row_count, + COUNT(DISTINCT m."user_id")::bigint AS record_count, + MIN(m."value") AS min_value, + MAX(m."value") AS max_value, + MIN(m."measured_at") AS first_at, + MAX(m."measured_at") AS last_at, + COUNT(*) FILTER (WHERE m."deleted_at" IS NOT NULL)::bigint AS tombstoned + FROM "measurements" m + JOIN bands b ON b."type" = m."type"::text + WHERE m."value" < b.min_value + OR m."value" > b.max_value + OR m."value" <> m."value" + GROUP BY m."type", m."source" + ORDER BY COUNT(*) DESC, m."type"::text ASC, m."source"::text ASC + `; + + if (buckets.length === 0) { + console.log( + `Every stored measurement sits inside its metric's declared range ` + + `(${types.length} metric bands checked). Nothing to review.`, + ); + return; + } + + const totalRows = buckets.reduce((sum, b) => sum + Number(b.row_count), 0); + const affectedTypes = new Set(buckets.map((b) => b.type)); + const affectedSources = new Set(buckets.map((b) => b.source)); + + console.log( + `${totalRows} stored row(s) sit outside the declared plausibility band, ` + + `across ${affectedTypes.size} metric(s) and ${affectedSources.size} source(s).\n` + + `Nothing below is modified. Each line is a (metric, source) bucket; the ` + + `band is the metric's own declared domain.\n`, + ); + + // Group by metric so the declared band is stated once per heading, and rank + // the metrics by how much of the problem each one is. The query's own + // ordering already ranks the sources inside a metric. + const byType = new Map(); + for (const bucket of buckets) { + const bucketsForType = byType.get(bucket.type); + if (bucketsForType) bucketsForType.push(bucket); + else byType.set(bucket.type, [bucket]); + } + const ranked = [...byType.entries()].sort( + (a, b) => + b[1].reduce((sum, x) => sum + Number(x.row_count), 0) - + a[1].reduce((sum, x) => sum + Number(x.row_count), 0), + ); + + for (const [type, rows] of ranked) { + const band = VALUE_RANGES[type]!; + console.log(`${type} — declared band ${band.min}…${band.max}`); + for (const bucket of rows) { + const live = Number(bucket.row_count) - Number(bucket.tombstoned); + console.log( + ` ${bucket.source.padEnd(14)} ` + + `${String(bucket.row_count).padStart(7)} row(s) ` + + `(${live} live, ${bucket.tombstoned} tombstoned) ` + + `across ${bucket.record_count} record(s) — ` + + `observed ${bucket.min_value}…${bucket.max_value}, ` + + `${ymd(bucket.first_at)} → ${ymd(bucket.last_at)}`, + ); + } + } + + console.log( + `\nThe sync writers refuse these values now, so the population above is ` + + `closed unless a writer without the gate is still in play. Removing a ` + + `row is a maintainer decision, per bucket.`, + ); +} + +main() + .catch((err) => { + console.error(err); + process.exitCode = 1; + }) + .finally(() => { + void prisma.$disconnect(); + }); diff --git a/src/__tests__/measurement-write-plausibility-guard.test.ts b/src/__tests__/measurement-write-plausibility-guard.test.ts new file mode 100644 index 000000000..88fbafee2 --- /dev/null +++ b/src/__tests__/measurement-write-plausibility-guard.test.ts @@ -0,0 +1,200 @@ +/** + * Structural guard on the measurement ingest boundary. + * + * `VALUE_RANGES` is the band the application declares a metric can occupy. + * Every path a person can drive enforced it from the start; the provider sync + * writers never did, and nothing noticed, because each writer is correct on + * its own terms — it wrote what the remote sent. The gap was only visible from + * above, by listing every writer at once and asking which of them had ever + * been told about the band. + * + * So the list lives here. Every file that writes a `Measurement` row is + * enumerated with the reason it is allowed to, and the guard fails when the + * discovered set stops matching. A provider added tomorrow lands a file that + * is not in the map and has to answer the question before it can ship. + * + * This is a tripwire, not a proof. It cannot show a disposition is the right + * one — only that nobody added a writer without writing down which gate covers + * it. A reviewer who waves through a bogus `derived` label defeats it, and no + * test substitutes for that review. + */ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +/** The gate module every provider-fed writer has to reach. */ +const GATE_MODULE = "@/lib/measurements/plausibility-gate"; +/** The predicate every person-driven write path has always used. */ +const VALIDATOR = "validateMeasurementRange"; + +/** + * How a writer satisfies the plausibility domain. + * + * - `gate` — provider-fed; must reach {@link GATE_MODULE} itself. + * - `validator` — person-driven; must call {@link VALIDATOR} itself. + * - `upstream` — validated in a named module before the row is built; the + * named module is checked, so the label cannot be a shrug. + * - `derived` — the value is arithmetic over rows already in the table, not a + * number from outside. Gating here would drop a consolidation of data that + * IS present, which is a different failure from admitting one that is not. + * - `literal` — the written value is a constant in the file, so no external + * number can reach the column. + * - `restore` — replays the record's own backup verbatim. Fidelity is the + * contract; a restore that silently dropped rows would be the worse bug, + * and the diagnostic script is how a stored impossible row gets found. + */ +type Disposition = + | { kind: "gate" } + | { kind: "validator" } + | { kind: "upstream"; validatedIn: string } + | { kind: "derived"; why: string } + | { kind: "literal"; why: string } + | { kind: "restore"; why: string }; + +const WRITERS: Record = { + // ── Provider sync writers, gated ────────────────────────────────────── + "lib/withings/sync.ts": { kind: "gate" }, + "lib/withings/sync-activity.ts": { kind: "gate" }, + "lib/fitbit/sync-core.ts": { kind: "gate" }, + "lib/google-health/sync-core.ts": { kind: "gate" }, + "lib/nightscout/sync.ts": { kind: "gate" }, + // The shared identity seam. Polar, Oura, WHOOP, the Withings sleep leg, + // the Apple export importer and the batch endpoint all write through it, + // so the gate here is what makes a new provider fail closed. + "lib/measurements/reconcile-external-measurement.ts": { kind: "gate" }, + + // ── Person-driven write paths, validated ────────────────────────────── + "app/api/measurements/route.ts": { + kind: "upstream", + validatedIn: "lib/validations/measurement.ts", + }, + "app/api/import/route.ts": { kind: "validator" }, + "lib/mcp/writes.ts": { kind: "validator" }, + "lib/measurements/create-from-telegram.ts": { kind: "validator" }, + "lib/measurements/import-apple-health-export.ts": { kind: "validator" }, + "app/api/import/csv/route.ts": { + kind: "upstream", + validatedIn: "lib/import/csv-measurements.ts", + }, + + // ── Fixed values ────────────────────────────────────────────────────── + "lib/withings/sync-ecg.ts": { + kind: "literal", + why: "an ECG row carries value 1 — the file's own constant, never a provider number", + }, + "app/api/mental-health/assessments/route.ts": { + kind: "literal", + why: "the questionnaire score is summed from Zod-bounded item answers", + }, + "lib/insights/score-row.ts": { + kind: "literal", + why: "the health score is computed in-app on a closed 0-100 scale", + }, + + // ── Derived from rows already stored ────────────────────────────────── + "lib/measurements/consolidate-daily-mean.ts": { + kind: "derived", + why: "writes the mean of the day's own stored rows", + }, + "lib/measurements/consolidate-legacy-steps.ts": { + kind: "derived", + why: "folds stored per-sample step rows into the day's canonical row", + }, + "lib/measurements/dense-intraday-retention.ts": { + kind: "derived", + why: "rebuilds hourly buckets from stored per-sample rows", + }, + "lib/measurements/drain-per-sample-cumulative.ts": { + kind: "derived", + why: "reduces stored cumulative per-sample rows to one daily total", + }, + + // ── Restore ─────────────────────────────────────────────────────────── + "app/api/admin/backups/[id]/restore/route.ts": { + kind: "restore", + why: "replays the record's own export; dropping rows would break the restore contract", + }, +}; + +/** + * Any Prisma client handle followed by a measurement write. Whitespace- and + * handle-tolerant on purpose: the earlier generation of this repo's guards + * demanded a literal `prisma.x.findUnique(` and went green because it matched + * nothing, not because nothing was there. The transaction handles in use are + * `tx` and `pc`; a future one is caught by the `[A-Za-z_$][\w$]*` head. + */ +const MEASUREMENT_WRITE = + /[A-Za-z_$][\w$]*\s*\.\s*measurement\s*\.\s*(create|createMany|createManyAndReturn|upsert)\s*\(/; + +function sourceFiles(): string[] { + return walkSourceFiles(SRC, { floor: 3000 }) + .filter((p) => !p.startsWith("generated/")) + .filter((p) => !p.includes("__tests__")) + .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx")) + .sort(); +} + +function read(rel: string): string { + return readFileSync(join(SRC, rel), "utf8"); +} + +function discoveredWriters(): string[] { + return sourceFiles().filter((rel) => MEASUREMENT_WRITE.test(read(rel))); +} + +describe("every measurement writer declares how it meets the plausibility domain", () => { + it("finds the writers at all — an empty sweep is a broken guard, not a clean repo", () => { + // The map is the claim; this asserts the matcher can still see the + // population it describes. A regex that silently stops matching would + // otherwise turn every check below into a tautology. + expect(discoveredWriters().length).toBeGreaterThanOrEqual(15); + }); + + it("the discovered writer set matches the declared map exactly", () => { + expect(discoveredWriters()).toEqual(Object.keys(WRITERS).sort()); + }); + + it("every provider-fed writer reaches the gate module", () => { + const gated = Object.entries(WRITERS) + .filter(([, d]) => d.kind === "gate") + .map(([rel]) => rel); + expect(gated.length).toBeGreaterThanOrEqual(6); + for (const rel of gated) { + expect(read(rel), `${rel} must import ${GATE_MODULE}`).toContain( + GATE_MODULE, + ); + } + }); + + it("every person-driven writer calls the range validator", () => { + for (const [rel, disposition] of Object.entries(WRITERS)) { + if (disposition.kind === "validator") { + expect(read(rel), `${rel} must call ${VALIDATOR}`).toContain(VALIDATOR); + } + if (disposition.kind === "upstream") { + expect( + read(disposition.validatedIn), + `${disposition.validatedIn} must call ${VALIDATOR} on behalf of ${rel}`, + ).toContain(VALIDATOR); + } + } + }); + + it("the exempt writers carry a written reason", () => { + for (const [rel, disposition] of Object.entries(WRITERS)) { + if ( + disposition.kind === "derived" || + disposition.kind === "literal" || + disposition.kind === "restore" + ) { + expect(disposition.why.length, `${rel} needs a reason`).toBeGreaterThan( + 20, + ); + } + } + }); +}); diff --git a/src/__tests__/mood-backup-completeness.test.ts b/src/__tests__/mood-backup-completeness.test.ts new file mode 100644 index 000000000..ef483325d --- /dev/null +++ b/src/__tests__/mood-backup-completeness.test.ts @@ -0,0 +1,294 @@ +/** + * Pair guard binding `prisma/schema.prisma`'s `MoodEntry` and + * `MoodEntryTagLink` models to the disaster-recovery backup selects. + * + * The sibling of `measurement-backup-completeness.test.ts`, written as its own + * file rather than as a second describe inside it, because that guard's own + * note asks for model-by-model growth and one file per model keeps every + * failure message about one model. + * + * The failure it exists to catch, stated by the defect that prompted it: the + * mood payload carried `id, date, mood, score, tags, source, loggedAt, + * externalId, deletedAt, factors` and nothing else. The free-text note, the + * row's timezone and the last-writer-wins counter were selected by the query + * and written to no payload, so a restored entry came back without its text, + * read its day boundaries under the legacy Europe/Berlin assumption, and lost + * the next sync round to a paired device still holding a higher counter. The + * backup validation schema is `.passthrough()`, so nothing complained at parse + * time either. Nothing here was a wrong value; every one of them was silence. + * + * Both ends of the comparison are literals — a Prisma model block and a named + * select object — so the matcher is exact. Every matcher also asserts a + * non-zero match count: an empty match set fails rather than passing quietly, + * which is how the bearer-scope guard stayed green for months while matching + * nothing at all. + * + * Mutation check: comment out `tz: true` in `MOOD_ENTRY_BACKUP_SELECT` and + * this goes red naming that column; add a scalar to the model without adding + * it to the select and it goes red naming the new one; rename the mapper's + * `moodEntry` parameter and the serialization assertion goes red on all of + * them at once. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, it, expect } from "vitest"; + +const ROOT = process.cwd(); +const SCHEMA = join(ROOT, "prisma", "schema.prisma"); +const BACKUP_BUILDER = join( + ROOT, + "src", + "lib", + "export", + "full-backup-payload.ts", +); + +/** + * Columns the backup is correct to omit, each with the reason. The reasons are + * length-checked below: an exclusion is a claim, and a claim needs saying. + */ +const EXCLUDED_MOOD_ENTRY_COLUMNS: Record = { + userId: "user-scoped backup; owner is re-derived from the restoring account", +}; + +const EXCLUDED_TAG_LINK_COLUMNS: Record = { + moodEntryId: "re-bound to the restored parent row's id, never carried", + moodTagId: + "the link travels by the tag's key; a source-instance id need not exist in the target catalogue", + createdAt: + "the link's own row stamp; the entry carries the moment the person logged it, and a key-only BINARY tag has nowhere to put a second one", +}; + +function modelBlock(schema: string, model: string): string { + const start = schema.indexOf(`model ${model} {`); + expect(start, `model ${model} not found`).toBeGreaterThan(-1); + const end = schema.indexOf("\n}", start); + expect(end).toBeGreaterThan(start); + return schema.slice(start, end); +} + +/** Names of every `model` block, so relation fields can be told apart. */ +function declaredModels(schema: string): Set { + const out = new Set(); + for (const m of schema.matchAll(/^model\s+([A-Za-z_]\w*)\s*\{/gm)) { + out.add(m[1]); + } + return out; +} + +/** + * Scalar (column-backed) fields of a model block: everything that is not a + * relation to another model. Enums and native scalars both map to columns and + * therefore both belong in a backup. + */ +function scalarFields(block: string, models: Set): string[] { + const fields: string[] = []; + for (const rawLine of block.split("\n").slice(1)) { + const line = rawLine.trim(); + if (!line || line.startsWith("//") || line.startsWith("@@")) continue; + const m = /^([a-zA-Z_]\w*)\s+([A-Za-z_]\w*)(\[\])?(\?)?/.exec(line); + if (!m) continue; + const [, name, type, isList] = m; + if (isList) continue; + if (models.has(type)) continue; + if (line.includes("@relation")) continue; + fields.push(name); + } + return fields; +} + +/** Keys of a named `const = {` select literal. */ +function selectFields(source: string, constant: string): string[] { + const start = source.indexOf(`const ${constant} = {`); + expect(start, `${constant} not found`).toBeGreaterThan(-1); + const end = source.indexOf("\n}", start); + expect(end).toBeGreaterThan(start); + const body = source.slice(start, end); + return [...body.matchAll(/^ {2}([a-zA-Z_$][\w$]*): true,$/gm)].map( + (m) => m[1], + ); +} + +/** + * The builder with its comment lines removed. + * + * Every matcher below runs against this rather than against the raw file, + * because prose describing the defect reads exactly like the defect: the first + * draft of this guard went red on its own explanation of the filter it exists + * to forbid. Whole comment lines only — a trailing comment after code is left + * alone, which is safe here because the only risk it carries is a matcher + * accepting a column named in prose, and the select comparison beside it is + * literal either way. + */ +function stripCommentLines(source: string): string { + return source + .split("\n") + .filter((line) => { + const trimmed = line.trim(); + return !( + trimmed.startsWith("//") || + trimmed.startsWith("/*") || + trimmed.startsWith("*") + ); + }) + .join("\n"); +} + +/** + * Columns either serialization arm actually reads off the mapped row. + * + * The capture is the RIGHT-hand side — the column — not the payload key, which + * matters here in a way it did not for `Measurement`: the mood payload renames + * `moodLoggedAt` to `loggedAt` on the wire, so a left-hand matcher would report + * the column as dropped while the value rides perfectly well. + * + * The two arms share one restorable wire format, so this is a union and + * therefore a weaker claim than the select comparison: it catches a column that + * is selected and then written to neither payload, not one carried by only a + * single arm. + */ +function serialisedFields(source: string, variable: string): Set { + const matches = [ + ...source.matchAll( + new RegExp(`[:(,]\\s*${variable}\\.([a-zA-Z_$][\\w$]*)`, "g"), + ), + ]; + expect( + matches.length, + `no \`${variable}.\` reads found — the mapper parameter was renamed and this matcher now proves nothing`, + ).toBeGreaterThan(0); + return new Set(matches.map((m) => m[1])); +} + +describe("mood backup completeness", () => { + const schema = readFileSync(SCHEMA, "utf8"); + const source = stripCommentLines(readFileSync(BACKUP_BUILDER, "utf8")); + const models = declaredModels(schema); + + const entryColumns = scalarFields(modelBlock(schema, "MoodEntry"), models); + const entrySelected = selectFields(source, "MOOD_ENTRY_BACKUP_SELECT"); + const linkColumns = scalarFields( + modelBlock(schema, "MoodEntryTagLink"), + models, + ); + const linkSelected = selectFields(source, "MOOD_ENTRY_TAG_LINK_SELECT"); + + it("parses both ends into a plausible shape", () => { + expect(models.size).toBeGreaterThan(50); + expect(entryColumns.length).toBeGreaterThan(10); + expect(entryColumns).toContain("noteEncrypted"); + expect(entryColumns).toContain("syncVersion"); + expect(entryColumns).not.toContain("user"); + expect(entryColumns).not.toContain("tagLinks"); + expect(entrySelected.length).toBeGreaterThan(10); + + expect(linkColumns).toEqual( + expect.arrayContaining([ + "moodEntryId", + "moodTagId", + "createdAt", + "rating", + ]), + ); + expect(linkColumns).not.toContain("moodEntry"); + expect(linkColumns).not.toContain("moodTag"); + expect(linkSelected.length).toBeGreaterThan(0); + }); + + it("the backup select covers every MoodEntry column", () => { + const missing = entryColumns.filter( + (c) => !entrySelected.includes(c) && !(c in EXCLUDED_MOOD_ENTRY_COLUMNS), + ); + + if (missing.length > 0) { + const report = missing.map((c) => ` ❌ ${c}`).join("\n"); + throw new Error( + `MOOD_ENTRY_BACKUP_SELECT omits ${missing.length} column(s) of the MoodEntry model:\n${report}\n\n` + + "A disaster-recovery restore brings these back at their column " + + "default. Add each to the select and to both serialization arms, " + + "extend `moodEntrySchema` and the restore write — or add it to " + + "EXCLUDED_MOOD_ENTRY_COLUMNS with the reason the backup is correct " + + "to drop it.", + ); + } + }); + + it("every selected MoodEntry column reaches a serialized payload", () => { + const emitted = serialisedFields(source, "moodEntry"); + const dropped = entrySelected.filter((c) => !emitted.has(c)); + + if (dropped.length > 0) { + const report = dropped.map((c) => ` ❌ ${c}`).join("\n"); + throw new Error( + `${dropped.length} selected MoodEntry column(s) reach neither serialization arm:\n${report}\n\n` + + "The query pays for them and no payload carries them, so a restore " + + "cannot put them back. Emit each from the disaster-recovery arm " + + "(and, where it is readable and not a tombstone, from the portable " + + "arm too).", + ); + } + }); + + it("the tag-link select covers every MoodEntryTagLink column", () => { + const missing = linkColumns.filter( + (c) => !linkSelected.includes(c) && !(c in EXCLUDED_TAG_LINK_COLUMNS), + ); + expect( + missing, + `MOOD_ENTRY_TAG_LINK_SELECT omits MoodEntryTagLink column(s): ${missing.join(", ")}`, + ).toEqual([]); + }); + + it("every selected tag-link column reaches a serialized payload", () => { + const emitted = serialisedFields(source, "tagLink"); + const dropped = linkSelected.filter((c) => !emitted.has(c)); + expect( + dropped, + `selected but serialized by neither arm: ${dropped.join(", ")}`, + ).toEqual([]); + }); + + it("both structured-tag arms are carried, not just the rated one", () => { + // The defect this file was written for: the tag-link read carried a + // filter, and a BINARY link's rating is NULL by definition, so every + // present/absent tag — the kind most people pick — was excluded from the + // file and could not come back from it. The claim is therefore about the + // read itself: it takes the links whole, unfiltered. + // + // Asserted on booleans rather than on the file contents so a failure reads + // as one line instead of printing the whole builder back. + const tagLinkRead = /tagLinks:\s*\{([\s\S]*?)\}/.exec(source); + expect( + tagLinkRead, + "no `tagLinks:` read found in the backup builder — this matcher proves nothing", + ).not.toBeNull(); + expect( + /where/.test(tagLinkRead![1]), + "the tag-link read filters again; a filter here is what dropped every BINARY link from the backup", + ).toBe(false); + expect( + source.includes("structuredTags"), + "no `structuredTags` key in the payload — the BINARY links have no carrier", + ).toBe(true); + }); + + it("the exclusion lists carry no stale entry", () => { + for (const [column, reason] of Object.entries( + EXCLUDED_MOOD_ENTRY_COLUMNS, + )) { + expect( + entryColumns, + `${column} is excluded but is no longer a MoodEntry column`, + ).toContain(column); + expect(reason.length).toBeGreaterThan(20); + } + for (const [column, reason] of Object.entries(EXCLUDED_TAG_LINK_COLUMNS)) { + expect( + linkColumns, + `${column} is excluded but is no longer a MoodEntryTagLink column`, + ).toContain(column); + expect(reason.length).toBeGreaterThan(20); + } + }); +}); diff --git a/src/app/api/admin/backups/[id]/restore/route.ts b/src/app/api/admin/backups/[id]/restore/route.ts index d57c7fc0f..4f3597843 100644 --- a/src/app/api/admin/backups/[id]/restore/route.ts +++ b/src/app/api/admin/backups/[id]/restore/route.ts @@ -684,6 +684,10 @@ const handler = apiHandler( entry.factors.map((factor) => factor.key), ); const factorKeys = [...new Set(referencedFactorKeys)]; + const referencedTagKeys = payload.moodEntries.flatMap( + (entry) => entry.structuredTags, + ); + const tagKeys = [...new Set(referencedTagKeys)]; // Re-create the account's own tag definitions first. The lookup // below used to ask only for the seeded catalogue (`userId: null`), // so a single custom rated tag made the whole restore throw @@ -740,6 +744,30 @@ const handler = apiHandler( referencedFactorKeys, ); + // The BINARY half of the same taxonomy, looked up the same way and + // reported the same way. It gets its own family because an + // operator reading the report needs to know whether a rated + // factor's score or a ticked tag went missing; they are different + // losses on the same entry. + const tagRows = + tagKeys.length === 0 + ? [] + : await tx.moodTag.findMany({ + where: { + key: { in: tagKeys }, + kind: "BINARY", + OR: [{ userId: null }, { userId: ownerId }], + }, + select: { id: true, key: true }, + }); + const tagByKey = new Map(tagRows.map((tag) => [tag.key, tag.id])); + recordUnknownKeys( + skips, + "moodTag", + tagKeys.filter((key) => !tagByKey.has(key)), + referencedTagKeys, + ); + for (const entry of payload.moodEntries) { const moodLoggedAt = new Date(entry.loggedAt); const restoredData = { @@ -747,10 +775,44 @@ const handler = apiHandler( mood: entry.mood, score: entry.score, tags: entry.tags ?? null, + // The note comes back the way every other dual-column note + // does: ciphertext verbatim when the file carries it, and a + // portable file's plaintext re-encrypted on the way in, with + // the legacy column left null either way. + note: null, + noteEncrypted: + entry.noteEncrypted == null + ? encryptNote(entry.note ?? null) + : decodeEncryptedBytes(entry.noteEncrypted), source: entry.source ?? "MOODLOG", externalId: entry.externalId ?? null, moodLoggedAt, + // The zone the `date` string is anchored to. A file written + // before this rode the payload carries nothing, and NULL is + // what such a row meant anyway: the legacy Europe/Berlin + // reading. So an old file keeps its old day boundaries and a + // new one keeps the zone the person actually logged in. + tz: entry.tz ?? null, + // Mood reconciles last-writer-wins by `syncVersion`. Restoring + // at the schema default would hand the next sync round to any + // paired device still holding a higher number, and the device + // would overwrite the recovered row with what it had — a + // restore that loses to the client is worse than one that + // fails. An old file carries nothing here and keeps the + // default, which is the value those rows were written with. + ...(entry.syncVersion === undefined + ? {} + : { syncVersion: entry.syncVersion }), deletedAt: entry.deletedAt ? new Date(entry.deletedAt) : null, + ...(entry.syncedAt + ? { syncedAt: new Date(entry.syncedAt) } + : {}), + ...(entry.createdAt + ? { createdAt: new Date(entry.createdAt) } + : {}), + ...(entry.updatedAt + ? { updatedAt: new Date(entry.updatedAt) } + : {}), }; const createData = { userId: ownerId, ...restoredData }; const restored = entry.id @@ -798,8 +860,19 @@ const handler = apiHandler( ] : []; }); - if (factorLinks.length > 0) { - await tx.moodEntryTagLink.createMany({ data: factorLinks }); + // The other arm. A BINARY link carries no score — the presence + // of the row is the signal — so `rating` is NULL by definition, + // which is exactly what the backup's old tag-link filter took to + // mean "nothing here". + const tagLinks = entry.structuredTags.flatMap((key) => { + const moodTagId = tagByKey.get(key); + return moodTagId + ? [{ moodEntryId: restored.id, moodTagId, rating: null }] + : []; + }); + const links = [...factorLinks, ...tagLinks]; + if (links.length > 0) { + await tx.moodEntryTagLink.createMany({ data: links }); } } } diff --git a/src/app/api/measurements/batch/route.ts b/src/app/api/measurements/batch/route.ts index df8ffcd59..1199e2354 100644 --- a/src/app/api/measurements/batch/route.ts +++ b/src/app/api/measurements/batch/route.ts @@ -620,6 +620,17 @@ async function postBatch(request: NextRequest): Promise { }; failedCount++; continue; + case "rejected_range": + // The reconciler carries the same plausibility gate this route + // already applied above, so reaching here means the two disagreed — + // report it exactly as the earlier guard would have, and let the + // wide-event tally record which end refused. + results[p.index] = { + index: p.index, + status: "skipped", + reason: "value_out_of_range", + }; + continue; } if (isMergeableSource(p.row.source as string)) { diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index 2df37779f..76c3acede 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -194,6 +194,9 @@ function catalogueLabel( if (catalogue === "illnessSymptom") { return t("admin.section.backups.restoreSkippedIllnessSymptom"); } + if (catalogue === "moodTag") { + return t("admin.section.backups.restoreSkippedMoodTag"); + } return t("admin.section.backups.restoreSkippedMoodFactor"); } diff --git a/src/lib/export/__tests__/full-backup-payload.test.ts b/src/lib/export/__tests__/full-backup-payload.test.ts index 4d76bc57e..e55bf7b69 100644 --- a/src/lib/export/__tests__/full-backup-payload.test.ts +++ b/src/lib/export/__tests__/full-backup-payload.test.ts @@ -38,6 +38,8 @@ import { restoreProfileData } from "../profile-backup"; const deletedAt = new Date("2026-07-19T12:00:00.000Z"); const measurementNote = encryptToBytes("canonical measurement note"); const sideEffectNote = encryptToBytes("nausea after the evening dose"); +const moodNotePlaintext = "quiet evening, slept badly"; +const moodNote = encryptToBytes(moodNotePlaintext); const appSettings = { id: "singleton", @@ -193,11 +195,27 @@ function makePrisma() { mood: "OKAY", score: 3, tags: null, + note: null, + noteEncrypted: moodNote, source: "MOODLOG", externalId: "moodlog-42", moodLoggedAt: new Date("2026-07-19T08:00:00.000Z"), + tz: "America/New_York", + syncedAt: new Date("2026-07-19T08:00:01.000Z"), + syncVersion: 3, deletedAt, - tagLinks: [{ rating: 4, moodTag: { key: "sleep_quality" } }], + createdAt: new Date("2026-07-19T08:00:00.000Z"), + updatedAt: new Date("2026-07-19T09:00:00.000Z"), + tagLinks: [ + { + rating: 4, + moodTag: { key: "sleep_quality", kind: "RATED" }, + }, + { + rating: null, + moodTag: { key: "headache", kind: "BINARY" }, + }, + ], }, ]), }, @@ -409,7 +427,13 @@ describe("buildFullBackupPayload disaster-recovery mode", () => { expect(prisma.moodEntry.findMany).toHaveBeenCalledWith( expect.objectContaining({ where: { userId: "user-1" }, - include: expect.any(Object), + select: expect.objectContaining({ + id: true, + noteEncrypted: true, + tz: true, + syncVersion: true, + deletedAt: true, + }), }), ); expect(mocks.buildRecordsBackupSection).toHaveBeenCalledWith( @@ -485,7 +509,16 @@ describe("buildFullBackupPayload disaster-recovery mode", () => { id: "mood-tombstone", externalId: "moodlog-42", deletedAt: deletedAt.toISOString(), + tz: "America/New_York", + syncedAt: "2026-07-19T08:00:01.000Z", + syncVersion: 3, + createdAt: "2026-07-19T08:00:00.000Z", + updatedAt: "2026-07-19T09:00:00.000Z", + note: null, + noteEncrypted: Buffer.from(moodNote).toString("base64"), factors: [{ key: "sleep_quality", rating: 4 }], + // The BINARY link the old `rating: { not: null }` filter excluded. + structuredTags: ["headache"], }, ], nutrientDays: [ @@ -1148,3 +1181,79 @@ describe("buildFullBackupPayload — profile and custom metrics", () => { expect(counts.intradayProfiles).toBe(1); }); }); + +describe("buildFullBackupPayload — mood arms", () => { + it("carries the ciphertext and the whole row on the disaster-recovery arm", async () => { + installSectionMocks(); + const prisma = makePrisma(); + + const { payload } = await buildFullBackupPayload( + prisma as never, + "user-1", + { + purpose: "disaster-recovery", + exportedAt: new Date("2026-07-20T00:00:00.000Z"), + }, + ); + + const [entry] = (payload as { moodEntries: Array> }) + .moodEntries; + expect(entry).toEqual( + expect.objectContaining({ + id: "mood-tombstone", + note: null, + noteEncrypted: Buffer.from(moodNote).toString("base64"), + tz: "America/New_York", + syncedAt: "2026-07-19T08:00:01.000Z", + syncVersion: 3, + createdAt: "2026-07-19T08:00:00.000Z", + updatedAt: "2026-07-19T09:00:00.000Z", + deletedAt: deletedAt.toISOString(), + factors: [{ key: "sleep_quality", rating: 4 }], + structuredTags: ["headache"], + }), + ); + }); + + it("carries the readable note and no ciphertext on the portable arm", async () => { + installSectionMocks(); + const prisma = makePrisma(); + + const { payload } = await buildFullBackupPayload( + prisma as never, + "user-1", + { + purpose: "portable-export", + exportedAt: new Date("2026-07-20T00:00:00.000Z"), + }, + ); + + const [entry] = (payload as { moodEntries: Array> }) + .moodEntries; + // Same split the measurement arm makes: the readable artefact carries the + // decrypted text and nothing an operator could not read, and the counters + // that only mean something to this instance stay behind. + expect(entry).toEqual( + expect.objectContaining({ + note: moodNotePlaintext, + tz: "America/New_York", + structuredTags: ["headache"], + }), + ); + expect(entry).not.toHaveProperty("noteEncrypted"); + expect(entry).not.toHaveProperty("syncVersion"); + expect(entry).not.toHaveProperty("syncedAt"); + expect(JSON.stringify(entry)).not.toContain( + Buffer.from(moodNote).toString("base64"), + ); + // The tag-link read is unfiltered on both arms; a BINARY link is the one + // most people have and it was the one the file never carried. + expect(prisma.moodEntry.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + select: expect.objectContaining({ + tagLinks: { select: expect.any(Object) }, + }), + }), + ); + }); +}); diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index 57ef86451..9f4506de5 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -103,6 +103,46 @@ const MEASUREMENT_BACKUP_SELECT = { updatedAt: true, } as const satisfies Prisma.MeasurementSelect; +/** + * Every restorable `MoodEntry` scalar. `userId` is absent for the same reason + * it is absent above: the backup is user-scoped and the owner is re-derived + * from the restoring account. + * + * Named rather than inlined so `mood-backup-completeness.test.ts` can compare + * it field-by-field against the model in `prisma/schema.prisma`, exactly as the + * measurement guard does for the select above. + */ +const MOOD_ENTRY_BACKUP_SELECT = { + id: true, + date: true, + mood: true, + score: true, + tags: true, + note: true, + noteEncrypted: true, + source: true, + externalId: true, + moodLoggedAt: true, + tz: true, + syncedAt: true, + syncVersion: true, + deletedAt: true, + createdAt: true, + updatedAt: true, +} as const satisfies Prisma.MoodEntrySelect; + +/** + * A structured-tag link, as it rides the wire. The link is carried by the + * tag's KEY, not by its id: a restore resolves the key against the target + * instance's catalogue, where the source instance's row id may not exist at + * all. `kind` comes along so the mapper can split RATED factors from BINARY + * tags without a second query. + */ +const MOOD_ENTRY_TAG_LINK_SELECT = { + rating: true, + moodTag: { select: { key: true, kind: true } }, +} as const satisfies Prisma.MoodEntryTagLinkSelect; + /** * Build the canonical full-backup payload for `userId`. Portable exports omit * tombstones and document ciphertext; disaster-recovery payloads preserve @@ -207,14 +247,13 @@ export async function buildFullBackupPayload( prisma.moodEntry.findMany({ where: disasterRecovery ? { userId } : { userId, deletedAt: null }, orderBy: { moodLoggedAt: "desc" }, - include: { - tagLinks: { - where: { rating: { not: null } }, - select: { - rating: true, - moodTag: { select: { key: true } }, - }, - }, + select: { + ...MOOD_ENTRY_BACKUP_SELECT, + // Every link, not only the rated ones. This read used to filter + // `rating: { not: null }`, and a BINARY link carries a NULL rating by + // definition — so the present/absent tags, the kind most people pick, + // were absent from the file and could not come back from it. + tagLinks: { select: MOOD_ENTRY_TAG_LINK_SELECT }, }, }), // The account's OWN mood tags. The seeded catalogue is reference data every @@ -394,21 +433,66 @@ export async function buildFullBackupPayload( skipped: e.skipped, source: e.source, })), - moodEntries: moodEntries.map((e) => ({ - id: e.id, - date: e.date, - mood: e.mood, - score: e.score, - tags: e.tags, - source: e.source, - loggedAt: e.moodLoggedAt.toISOString(), - externalId: e.externalId, - deletedAt: e.deletedAt?.toISOString() ?? null, - factors: e.tagLinks.map((link) => ({ - key: link.moodTag.key, - rating: link.rating!, - })), - })), + // The parameter is spelled out rather than the `e` every other mapper here + // uses, and deliberately: `mood-backup-completeness.test.ts` proves a + // selected column reaches a payload by matching `: moodEntry.`. A matcher + // on `: e.` would also match the intake-event mapper above it and could + // therefore never fail. + moodEntries: moodEntries.map((moodEntry) => { + // The two link arms partition the entry's links exhaustively: a link is + // a rated factor when its tag is RATED and it actually carries a score, + // and a structured tag otherwise. Total on purpose — a malformed link + // (a RATED tag whose rating went missing) rides as a tag key and comes + // back NAMED as unresolvable on restore, rather than falling between the + // two arms and disappearing the way every BINARY link used to. + const isRatedFactor = (link: (typeof moodEntry.tagLinks)[number]) => + link.moodTag.kind === "RATED" && link.rating !== null; + + const factors = moodEntry.tagLinks + .filter(isRatedFactor) + .map((tagLink) => ({ + key: tagLink.moodTag.key, + rating: tagLink.rating as number, + })); + // BINARY link keys. Their own array rather than a nullable rating inside + // `factors`: the restore resolves `factors` against `kind: "RATED"` and + // reports what it cannot resolve, so a rating-less entry in that array + // would collide with both. A separate key leaves every backup file + // written before this parsing and restoring exactly as it did. + const structuredTags = moodEntry.tagLinks + .filter((link) => !isRatedFactor(link)) + .map((tagLink) => tagLink.moodTag.key); + + return { + ...(disasterRecovery + ? { + note: moodEntry.note, + noteEncrypted: moodEntry.noteEncrypted + ? Buffer.from(moodEntry.noteEncrypted).toString("base64") + : null, + syncedAt: moodEntry.syncedAt.toISOString(), + syncVersion: moodEntry.syncVersion, + createdAt: moodEntry.createdAt.toISOString(), + updatedAt: moodEntry.updatedAt.toISOString(), + } + : { note: readNote(moodEntry.noteEncrypted, moodEntry.note) }), + id: moodEntry.id, + date: moodEntry.date, + mood: moodEntry.mood, + score: moodEntry.score, + tags: moodEntry.tags, + source: moodEntry.source, + loggedAt: moodEntry.moodLoggedAt.toISOString(), + externalId: moodEntry.externalId, + // The zone the `date` string is anchored to. Without it a restored row + // falls back to the legacy Europe/Berlin reading and an account that + // logged elsewhere gets its day boundaries quietly moved. + tz: moodEntry.tz, + deletedAt: moodEntry.deletedAt?.toISOString() ?? null, + factors, + structuredTags, + }; + }), customMoodTags: customMoodTags.map((tag) => ({ ...(disasterRecovery ? { id: tag.id } : {}), key: tag.key, diff --git a/src/lib/export/restore-skips.ts b/src/lib/export/restore-skips.ts index e57756edd..326ea093b 100644 --- a/src/lib/export/restore-skips.ts +++ b/src/lib/export/restore-skips.ts @@ -1,8 +1,9 @@ /** * What a restore could not put back, and how much of it there was. * - * Three of the restore's lookups resolve a key against a SEEDED CATALOGUE — - * cycle symptoms, illness symptoms, mood factors. The catalogue is reference + * Four of the restore's lookups resolve a key against a SEEDED CATALOGUE — + * cycle symptoms, illness symptoms, rated mood factors and the mood tags a + * person ticks present or absent. The catalogue is reference * data the instance owns, not data the file carries, so it legitimately drifts * between the day a backup is written and the day it is read: a key gets * renamed, a symptom is retired, the file is a year old, the instance is three @@ -28,7 +29,8 @@ */ /** Which seeded catalogue a key failed to resolve against. */ -export type SkippedCatalogue = "cycleSymptom" | "illnessSymptom" | "moodFactor"; +export type SkippedCatalogue = + "cycleSymptom" | "illnessSymptom" | "moodFactor" | "moodTag"; /** One key this instance does not know, and the links it cost. */ export interface SkippedCatalogueKey { diff --git a/src/lib/fitbit/__tests__/sync-plausibility.test.ts b/src/lib/fitbit/__tests__/sync-plausibility.test.ts new file mode 100644 index 000000000..7efe59f99 --- /dev/null +++ b/src/lib/fitbit/__tests__/sync-plausibility.test.ts @@ -0,0 +1,179 @@ +/** + * `upsertFitbitMeasurements` is the single write seam for every Fitbit + * resource leg — activity, sleep, daily metrics — and it holds Prisma + * directly, so it does not inherit the shared reconciler's gate. These cases + * pin that the provider's numbers clear the app's own declared plausibility + * band before any of them can become a row, and that a refusal is reported. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({ + findManyMock: vi.fn(async () => [] as unknown[]), + createManyMock: vi.fn< + () => Promise> + >(async () => []), + updateMock: vi.fn(async () => ({})), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { + findMany: findManyMock, + createManyAndReturn: createManyMock, + update: updateMock, + }, + }, +})); +vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() })); +vi.mock("@/lib/integrations/status", () => ({ + isReauthRequired: vi.fn(async () => false), + recordSyncFailure: vi.fn(async () => {}), + recordSyncSuccess: vi.fn(async () => {}), +})); +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + collapseToTypeDayKeys: vi.fn(() => []), + recomputeBucketsForMeasurement: vi.fn(async () => {}), + recomputeUserRollups: vi.fn(async () => {}), +})); +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: vi.fn(async () => {}), +})); +vi.mock("@/lib/integrations/oauth-refresh", () => ({ + persistRotatedToken: vi.fn(async () => {}), +})); +vi.mock("../credentials", () => ({ + getUserFitbitCredentials: vi.fn(async () => null), +})); +vi.mock("../client", () => ({ refreshAccessToken: vi.fn() })); + +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate"; +import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate"; + +import { upsertFitbitMeasurements } from "../sync-core"; + +/** RESTING_HEART_RATE is declared 25–220. */ +const REAL_READING = { + type: "RESTING_HEART_RATE", + value: 58, + unit: "bpm", + measuredAt: new Date("2026-07-08T00:00:00.000Z"), + externalId: "rhr:2026-07-08", +}; + +const IMPOSSIBLE_READING = { + type: "RESTING_HEART_RATE", + value: 4_915_200, + unit: "bpm", + measuredAt: new Date("2026-07-09T00:00:00.000Z"), + externalId: "rhr:2026-07-09", +}; + +let event: WideEventBuilder; + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +beforeEach(() => { + findManyMock.mockReset().mockResolvedValue([]); + createManyMock.mockReset().mockResolvedValue([]); + updateMock.mockReset().mockResolvedValue({}); + event = new WideEventBuilder("background"); +}); + +describe("upsertFitbitMeasurements — plausibility gate", () => { + it("writes a reading inside the metric's declared band", async () => { + createManyMock.mockResolvedValue([ + { + id: "row-1", + type: "RESTING_HEART_RATE", + measuredAt: REAL_READING.measuredAt, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertFitbitMeasurements("user-1", [REAL_READING], { deferRollup: true }), + ); + + expect(result.imported).toBe(1); + const planned = (createManyMock.mock.calls[0] as unknown[])[0] as { + data: Array<{ value: number }>; + }; + expect(planned.data).toHaveLength(1); + expect(planned.data[0]!.value).toBe(58); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band reading and annotates the drop", async () => { + createManyMock.mockResolvedValue([ + { + id: "row-1", + type: "RESTING_HEART_RATE", + measuredAt: REAL_READING.measuredAt, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertFitbitMeasurements("user-1", [REAL_READING, IMPOSSIBLE_READING], { + deferRollup: true, + }), + ); + + expect(result.imported).toBe(1); + const planned = (createManyMock.mock.calls[0] as unknown[])[0] as { + data: Array<{ value: number; externalId: string }>; + }; + expect(planned.data).toHaveLength(1); + expect(planned.data[0]!.externalId).toBe("rhr:2026-07-08"); + // The refused reading never even reached the existence probe. + expect( + ( + (findManyMock.mock.calls[0] as unknown[])[0] as { + where: { externalId: { in: string[] } }; + } + ).where.externalId.in, + ).toEqual(["rhr:2026-07-08"]); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { + source: "fitbit", + type: "RESTING_HEART_RATE", + direction: "above_max", + count: 1, + }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); + + it("does not overwrite a stored reading with an out-of-band one", async () => { + // The key already exists; ungated, the update branch rewrote its value. + findManyMock.mockResolvedValue([ + { + id: "existing", + type: "RESTING_HEART_RATE", + externalId: "rhr:2026-07-09", + value: 58, + unit: "bpm", + measuredAt: IMPOSSIBLE_READING.measuredAt, + sleepStage: null, + deletedAt: null, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertFitbitMeasurements("user-1", [IMPOSSIBLE_READING], { + deferRollup: true, + }), + ); + + expect(result.imported).toBe(0); + expect(updateMock).not.toHaveBeenCalled(); + expect(createManyMock).not.toHaveBeenCalled(); + expect(tally()?.total).toBe(1); + }); +}); diff --git a/src/lib/fitbit/sync-core.ts b/src/lib/fitbit/sync-core.ts index 74637e01b..f67f4422b 100644 --- a/src/lib/fitbit/sync-core.ts +++ b/src/lib/fitbit/sync-core.ts @@ -15,6 +15,7 @@ import { persistRotatedToken } from "@/lib/integrations/oauth-refresh"; import { recordSyncFailure } from "@/lib/integrations/status"; import type { FailureKind } from "@/lib/integrations/status"; import { getEvent } from "@/lib/logging/context"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { collapseToTypeDayKeys, recomputeBucketsForMeasurement, @@ -530,7 +531,7 @@ export async function replaceStaleFitbitSleep( */ export async function upsertFitbitMeasurements( userId: string, - readings: FitbitMeasurementUpsert[], + incoming: FitbitMeasurementUpsert[], opts: { deferRollup?: boolean } = {}, ): Promise<{ imported: number; @@ -541,6 +542,15 @@ export async function upsertFitbitMeasurements( measuredAt: Date; }>; }> { + // Every resource leg funnels through here, so this is the one place the + // provider's numbers have to clear the app's declared plausibility band + // before they can become rows. A refused reading is dropped and tallied on + // the ambient event, never clamped and never written for a later reader to + // discover. + const readings = dropImplausibleMeasurements("fitbit", incoming, (r) => ({ + type: r.type, + value: r.value, + })); if (readings.length === 0) return { imported: 0, touched: [], inserted: [] }; // Probe EVERY existing row (live AND tombstoned) for the batch's externalIds diff --git a/src/lib/google-health/__tests__/sync-plausibility.test.ts b/src/lib/google-health/__tests__/sync-plausibility.test.ts new file mode 100644 index 000000000..2fb5b3404 --- /dev/null +++ b/src/lib/google-health/__tests__/sync-plausibility.test.ts @@ -0,0 +1,180 @@ +/** + * `upsertGoogleHealthMeasurements` is the single write seam for every Google + * Health resource leg and it holds Prisma directly, so it does not inherit the + * shared reconciler's gate. These cases pin that the provider's numbers clear + * the app's own declared plausibility band before any of them can become a + * row, and that a refusal is reported rather than swallowed. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({ + findManyMock: vi.fn(async () => [] as unknown[]), + createManyMock: vi.fn< + () => Promise> + >(async () => []), + updateMock: vi.fn(async () => ({})), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { + findMany: findManyMock, + createManyAndReturn: createManyMock, + update: updateMock, + }, + }, +})); +vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() })); +vi.mock("@/lib/integrations/status", () => ({ + isReauthRequired: vi.fn(async () => false), + recordSyncFailure: vi.fn(async () => {}), + recordSyncSuccess: vi.fn(async () => {}), +})); +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + collapseToTypeDayKeys: vi.fn(() => []), + recomputeBucketsForMeasurement: vi.fn(async () => {}), + recomputeUserRollups: vi.fn(async () => {}), +})); +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: vi.fn(async () => {}), +})); +vi.mock("../credentials", () => ({ + getUserGoogleHealthCredentials: vi.fn(async () => null), +})); +vi.mock("../client", async () => { + const actual = await vi.importActual("../client"); + return { + ...actual, + refreshAccessToken: vi.fn(), + noteGoogleHealthMapped: vi.fn(), + noteGoogleHealthWritten: vi.fn(), + noteGoogleHealthOutcomeFailure: vi.fn(), + }; +}); + +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate"; +import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate"; + +import { upsertGoogleHealthMeasurements } from "../sync-core"; + +/** OXYGEN_SATURATION is declared 50–100. */ +const REAL_READING = { + type: "OXYGEN_SATURATION", + value: 97, + unit: "%", + measuredAt: new Date("2026-07-08T00:00:00.000Z"), + externalId: "spo2:2026-07-08", +}; + +const IMPOSSIBLE_READING = { + type: "OXYGEN_SATURATION", + value: 9_700, + unit: "%", + measuredAt: new Date("2026-07-09T00:00:00.000Z"), + externalId: "spo2:2026-07-09", +}; + +let event: WideEventBuilder; + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +beforeEach(() => { + findManyMock.mockReset().mockResolvedValue([]); + createManyMock.mockReset().mockResolvedValue([]); + updateMock.mockReset().mockResolvedValue({}); + event = new WideEventBuilder("background"); +}); + +describe("upsertGoogleHealthMeasurements — plausibility gate", () => { + it("writes a reading inside the metric's declared band", async () => { + createManyMock.mockResolvedValue([ + { + id: "row-1", + type: "OXYGEN_SATURATION", + measuredAt: REAL_READING.measuredAt, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertGoogleHealthMeasurements("user-1", [REAL_READING], { + deferRollup: true, + }), + ); + + expect(result.imported).toBe(1); + const planned = (createManyMock.mock.calls[0] as unknown[])[0] as { + data: Array<{ value: number }>; + }; + expect(planned.data).toHaveLength(1); + expect(planned.data[0]!.value).toBe(97); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band reading and annotates the drop", async () => { + createManyMock.mockResolvedValue([ + { + id: "row-1", + type: "OXYGEN_SATURATION", + measuredAt: REAL_READING.measuredAt, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertGoogleHealthMeasurements( + "user-1", + [REAL_READING, IMPOSSIBLE_READING], + { deferRollup: true }, + ), + ); + + expect(result.imported).toBe(1); + const planned = (createManyMock.mock.calls[0] as unknown[])[0] as { + data: Array<{ externalId: string }>; + }; + expect(planned.data).toHaveLength(1); + expect(planned.data[0]!.externalId).toBe("spo2:2026-07-08"); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { + source: "googleHealth", + type: "OXYGEN_SATURATION", + direction: "above_max", + count: 1, + }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); + + it("does not overwrite a stored reading with an out-of-band one", async () => { + findManyMock.mockResolvedValue([ + { + id: "existing", + type: "OXYGEN_SATURATION", + externalId: "spo2:2026-07-09", + value: 97, + unit: "%", + measuredAt: IMPOSSIBLE_READING.measuredAt, + sleepStage: null, + deletedAt: null, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertGoogleHealthMeasurements("user-1", [IMPOSSIBLE_READING], { + deferRollup: true, + }), + ); + + expect(result.imported).toBe(0); + expect(updateMock).not.toHaveBeenCalled(); + expect(createManyMock).not.toHaveBeenCalled(); + expect(tally()?.total).toBe(1); + }); +}); diff --git a/src/lib/google-health/sync-core.ts b/src/lib/google-health/sync-core.ts index 27ccaa139..d0545ee0d 100644 --- a/src/lib/google-health/sync-core.ts +++ b/src/lib/google-health/sync-core.ts @@ -20,6 +20,7 @@ import { prisma } from "@/lib/db"; import type { MeasurementType } from "@/generated/prisma/client"; import { encrypt, decrypt } from "@/lib/crypto"; import { getEvent } from "@/lib/logging/context"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { recordSyncFailure, type FailureKind } from "@/lib/integrations/status"; import { @@ -487,7 +488,7 @@ export async function replaceStaleGoogleHealthSleep( */ export async function upsertGoogleHealthMeasurements( userId: string, - readings: GoogleHealthMeasurementUpsert[], + incoming: GoogleHealthMeasurementUpsert[], opts: { deferRollup?: boolean } = {}, ): Promise<{ imported: number; @@ -498,7 +499,17 @@ export async function upsertGoogleHealthMeasurements( measuredAt: Date; }>; }> { - noteGoogleHealthMapped(readings.length); + noteGoogleHealthMapped(incoming.length); + // Every resource leg funnels through here, so this is the one place the + // provider's numbers have to clear the app's declared plausibility band + // before they can become rows. A refused reading is dropped and tallied on + // the ambient event, never clamped and never written for a later reader to + // discover. + const readings = dropImplausibleMeasurements( + "googleHealth", + incoming, + (r) => ({ type: r.type, value: r.value }), + ); if (readings.length === 0) return { imported: 0, touched: [], inserted: [] }; // Probe EVERY existing row (live AND tombstoned) for the batch's externalIds diff --git a/src/lib/measurements/__tests__/plausibility-gate.test.ts b/src/lib/measurements/__tests__/plausibility-gate.test.ts new file mode 100644 index 000000000..074d32600 --- /dev/null +++ b/src/lib/measurements/__tests__/plausibility-gate.test.ts @@ -0,0 +1,222 @@ +/** + * The ingest-side plausibility gate. + * + * `VALUE_RANGES` is the band the application declares a metric can occupy, and + * every path a person can drive has always enforced it. The provider sync + * writers did not, which is how a row the application itself calls impossible + * reached the table and poisoned every mean and personal band derived from it. + * These cases pin the two halves of the fix: the refusal, and the fact that a + * refusal is never silent. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { eventStorage } from "@/lib/logging/context"; +import { + RANGE_REJECTED_META_KEY, + classifyRangeRejection, + dropImplausibleMeasurements, + recordRangeRejections, + type RangeRejectionTally, +} from "../plausibility-gate"; + +let event: WideEventBuilder; + +function withEvent(run: () => T): T { + return eventStorage.run(event, run); +} + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +beforeEach(() => { + event = new WideEventBuilder("background"); +}); + +describe("classifyRangeRejection", () => { + it("passes a value inside the metric's declared band", () => { + // PULSE is declared 20–300; both edges are inclusive. + expect(classifyRangeRejection("PULSE", 60)).toBeNull(); + expect(classifyRangeRejection("PULSE", 20)).toBeNull(); + expect(classifyRangeRejection("PULSE", 300)).toBeNull(); + }); + + it("names the edge an out-of-band value broke", () => { + expect(classifyRangeRejection("PULSE", 55643821.505)).toEqual({ + type: "PULSE", + direction: "above_max", + }); + expect(classifyRangeRejection("PULSE", 19)).toEqual({ + type: "PULSE", + direction: "below_min", + }); + }); + + it("refuses a non-finite value whatever the metric", () => { + expect(classifyRangeRejection("PULSE", Number.NaN)).toEqual({ + type: "PULSE", + direction: "not_finite", + }); + // No declared band, so the value itself is the only thing to judge. + expect( + classifyRangeRejection("NOT_A_METRIC", Number.POSITIVE_INFINITY), + ).toEqual({ type: "NOT_A_METRIC", direction: "not_finite" }); + }); + + it("passes a metric with no declared band — an absent range is an absent fact", () => { + expect(classifyRangeRejection("NOT_A_METRIC", 9_999_999)).toBeNull(); + }); +}); + +describe("dropImplausibleMeasurements", () => { + it("keeps the in-band rows and returns them untouched", () => { + const rows = [ + { type: "PULSE", value: 60 }, + { type: "WEIGHT", value: 82.4 }, + ]; + const kept = withEvent(() => + dropImplausibleMeasurements("withings", rows, (r) => r), + ); + + expect(kept).toEqual(rows); + expect(kept[0]).toBe(rows[0]); + // Nothing was refused, so the event carries no tally at all — absence + // reads as absence. + expect(tally()).toBeUndefined(); + expect(event.toJSON().warnings).toBeUndefined(); + }); + + it("drops the out-of-band rows and tallies them on the wide event", () => { + const kept = withEvent(() => + dropImplausibleMeasurements( + "withings", + [ + { type: "PULSE", value: 60 }, + { type: "PULSE", value: 55643821.505 }, + { type: "PULSE", value: 900 }, + { type: "WEIGHT", value: 0.2 }, + ], + (r) => r, + ), + ); + + expect(kept).toEqual([{ type: "PULSE", value: 60 }]); + + const recorded = tally(); + expect(recorded).toEqual({ + total: 3, + buckets: [ + { + source: "withings", + type: "PULSE", + direction: "above_max", + count: 2, + }, + { + source: "withings", + type: "WEIGHT", + direction: "below_min", + count: 1, + }, + ], + }); + }); + + it("keeps the refused readings out of the log line", () => { + withEvent(() => + dropImplausibleMeasurements( + "withings", + [{ type: "PULSE", value: 55643821.505 }], + (r) => r, + ), + ); + + // The value is a health reading. The tally says which metric, which + // provider and which edge — never the number itself. + const serialised = JSON.stringify(event.toJSON()); + expect(serialised).not.toContain("55643821"); + }); + + it("lifts the event to warn once per provider", () => { + withEvent(() => { + dropImplausibleMeasurements( + "withings", + [{ type: "PULSE", value: 900 }], + (r) => r, + ); + dropImplausibleMeasurements( + "withings", + [{ type: "PULSE", value: 901 }], + (r) => r, + ); + dropImplausibleMeasurements( + "fitbit", + [{ type: "PULSE", value: 902 }], + (r) => r, + ); + }); + + expect(event.getLevel()).toBe("warn"); + expect(event.toJSON().warnings).toHaveLength(2); + expect(event.toJSON().warnings?.[0]).toContain("withings"); + expect(event.toJSON().warnings?.[1]).toContain("fitbit"); + }); + + it("accumulates across calls instead of overwriting the previous tally", () => { + // Fitbit gates once per resource leg, and the reconciler gates once per + // row: a tally that replaced rather than merged would report only the + // last batch, and a run that dropped hundreds would read as dropping one. + withEvent(() => { + dropImplausibleMeasurements( + "fitbit", + [{ type: "PULSE", value: 900 }], + (r) => r, + ); + dropImplausibleMeasurements( + "fitbit", + [ + { type: "PULSE", value: 901 }, + { type: "OXYGEN_SATURATION", value: 4 }, + ], + (r) => r, + ); + }); + + expect(tally()).toEqual({ + total: 3, + buckets: [ + { + source: "fitbit", + type: "OXYGEN_SATURATION", + direction: "below_min", + count: 1, + }, + { source: "fitbit", type: "PULSE", direction: "above_max", count: 2 }, + ], + }); + }); + + it("still drops outside a wide-event context", () => { + // A sync running without an ambient event has nowhere to report. The row + // is refused anyway: observability is what is lost, never the guarantee. + const kept = dropImplausibleMeasurements( + "nightscout", + [ + { type: "BLOOD_GLUCOSE", value: 110 }, + { type: "BLOOD_GLUCOSE", value: 9_000 }, + ], + (r) => r, + ); + expect(kept).toEqual([{ type: "BLOOD_GLUCOSE", value: 110 }]); + }); +}); + +describe("recordRangeRejections", () => { + it("writes nothing when there is nothing to report", () => { + withEvent(() => recordRangeRejections("withings", [])); + expect(tally()).toBeUndefined(); + expect(event.getLevel()).toBe("info"); + }); +}); diff --git a/src/lib/measurements/__tests__/reconcile-external-measurement.test.ts b/src/lib/measurements/__tests__/reconcile-external-measurement.test.ts index 5a2d012fd..9fa5c3cf7 100644 --- a/src/lib/measurements/__tests__/reconcile-external-measurement.test.ts +++ b/src/lib/measurements/__tests__/reconcile-external-measurement.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it, vi } from "vitest"; import type { Prisma } from "@/generated/prisma/client"; +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "../plausibility-gate"; import { reconcileExternalMeasurement, type ExternalMeasurementWrite, @@ -383,3 +386,104 @@ describe("reconcileExternalMeasurement", () => { ); }); }); + +/** + * The reconciler is the shared write seam for every provider that does not + * hold Prisma itself — Polar, Oura, WHOOP, the Withings sleep leg, the Apple + * export importer, the batch endpoint. Gating here rather than at each caller + * is what makes a provider added tomorrow inherit the refusal instead of + * having to remember a check it never knew about. + */ +describe("reconcileExternalMeasurement — plausibility gate", () => { + it("reconciles a value inside the metric's declared band", async () => { + const { tx, measurement } = transaction([]); + measurement.create.mockResolvedValue(row({ id: "fresh" })); + + // PULSE is declared 20–300. + const verdict = await reconcileExternalMeasurement( + tx, + desired({ type: "PULSE", value: 112, unit: "bpm", sleepStage: null }), + ); + + expect(verdict.status).toBe("inserted"); + expect(measurement.create).toHaveBeenCalledTimes(1); + }); + + it("refuses an out-of-band value without touching the transaction", async () => { + const { tx, measurement } = transaction([]); + + const verdict = await reconcileExternalMeasurement( + tx, + desired({ + type: "PULSE", + value: 55643821.505, + unit: "bpm", + sleepStage: null, + }), + ); + + expect(verdict).toEqual({ + status: "rejected_range", + rejection: { type: "PULSE", direction: "above_max" }, + }); + // No row written, no row updated, and no savepoint opened — the refusal + // happens before the reconciler does any work at all. + expect(measurement.create).not.toHaveBeenCalled(); + expect(measurement.update).not.toHaveBeenCalled(); + expect(tx.$executeRawUnsafe).not.toHaveBeenCalled(); + }); + + it("refuses rather than overwriting a good stored row", async () => { + // The dangerous shape: a live row already holds a real reading, and the + // provider re-sends the same identity carrying an impossible value. The + // update branch would have rewritten a good value with a bad one. + const existing = row({ + id: "exact-row", + type: "PULSE", + value: 62, + unit: "bpm", + sleepStage: null, + measuredAt: new Date("2026-07-20T06:30:00.000Z"), + }); + const { tx, measurement } = transaction([existing]); + + const verdict = await reconcileExternalMeasurement( + tx, + desired({ + type: "PULSE", + value: 55643821.505, + unit: "bpm", + sleepStage: null, + }), + ); + + expect(verdict.status).toBe("rejected_range"); + expect(measurement.update).not.toHaveBeenCalled(); + }); + + it("tallies the refusal on the ambient wide event under the provider", async () => { + const { tx } = transaction([]); + const event = new WideEventBuilder("background"); + + await eventStorage.run(event, () => + reconcileExternalMeasurement( + tx, + desired({ + type: "PULSE", + value: 55643821.505, + unit: "bpm", + source: "WHOOP", + sleepStage: null, + }), + ), + ); + + expect(event.toJSON().meta?.[RANGE_REJECTED_META_KEY]).toEqual({ + total: 1, + buckets: [ + { source: "whoop", type: "PULSE", direction: "above_max", count: 1 }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); +}); diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index fc1e9f838..31d0b7dc2 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -720,6 +720,16 @@ export async function streamParseExportXml( ), ); + // The reconciler carries the same plausibility gate the block above + // already applied, so this arm means the two disagreed. Tally it under + // the same counter the earlier guard uses and count no row. + if (verdict.status === "rejected_range") { + unknown[`${type}::aggregate_out_of_range`] = + (unknown[`${type}::aggregate_out_of_range`] ?? 0) + 1; + stat.durationMs += Date.now() - rowStart; + continue; + } + cumulativeEstimatedDays.add(dayKey); cumulativeEstimatedRows += 1; if (verdict.status === "inserted") { diff --git a/src/lib/measurements/plausibility-gate.ts b/src/lib/measurements/plausibility-gate.ts new file mode 100644 index 000000000..d079f5391 --- /dev/null +++ b/src/lib/measurements/plausibility-gate.ts @@ -0,0 +1,192 @@ +/** + * The ingest-side half of the app's plausibility domain. + * + * `VALUE_RANGES` in `@/lib/validations/measurement` is the band the + * application declares a metric can occupy — PULSE 20–300, WEIGHT 1–500, + * SLEEP_DURATION 0–1440 minutes. Every path a person can drive enforces it: + * the measurement routes, the batch endpoint, the CSV and Apple-export + * importers, MCP, Telegram. The provider sync writers did not. They took the + * number the remote handed over and wrote it, so a decode slip or a glitched + * device landed a row the application itself calls impossible, and every mean, + * median and personal band downstream digested it as a reading. + * + * `isPlausibleMetricValue` (in `value-domain.ts`) already answers the question + * for the READ side, which is where the damage surfaced first. This module is + * the same question asked one layer earlier, at the point of writing, so the + * row never lands. The two ends share the one predicate on purpose: a band the + * reader hides but the writer accepts is a row that exists and cannot be seen, + * which is the worse of the two failures. + * + * ## Disposition: drop, and say so + * + * An out-of-band incoming value is DROPPED. Not clamped (a clamp invents a + * reading nobody took), not stored-and-flagged (a quarantined row is still a + * row, and every consumer would need to learn the flag), not fatal to the + * batch (one glitched sample must not strand the sync's other readings behind + * a held watermark). Absence reads as absence: the sample is simply not part + * of the record. + * + * A silent drop, though, is indistinguishable from a provider that sent + * nothing — so every drop is tallied onto the ambient wide event under + * {@link RANGE_REJECTED_META_KEY}, with the metric type, the provider, and the + * direction it broke the band in. Counts only. The value itself is a health + * reading and never enters the log line; the direction plus the metric's own + * declared bound is what a diagnosis actually needs. + * + * ## Why meta and not `action` + * + * A wide event holds exactly ONE action, and `setAction` replaces it. A drop + * happens in the middle of a sync whose action is decided at the end, so an + * `action` set here is either clobbered a moment later (the drop vanishes) or, + * on the paths that annotate no terminal action, relabels an otherwise + * successful sync as a rejection. Meta accumulates, survives every later + * `setAction`, and is the shape a dashboard can sum. One warning per provider + * per event rides along so the drop also lifts the event's level to `warn` + * rather than hiding inside a green line. + */ +import { annotate, getEvent } from "@/lib/logging/context"; +import { isPlausibleMetricValue } from "@/lib/measurements/value-domain"; +import { VALUE_RANGES } from "@/lib/validations/measurement"; + +/** + * Stable meta key the tally lands under. Pinned so dashboards can count it + * across providers without knowing which one wrote the event. + */ +export const RANGE_REJECTED_META_KEY = "measurement.range_rejected"; + +/** Which edge of the declared band the value broke, or that it was not a number. */ +export type RangeRejectionDirection = "below_min" | "above_max" | "not_finite"; + +/** One refused reading, reduced to the facts that carry no health value. */ +export interface MeasurementRangeRejection { + /** `MeasurementType` of the refused reading. */ + type: string; + direction: RangeRejectionDirection; +} + +/** One (provider, metric, direction) cell of the event's running tally. */ +export interface RangeRejectionBucket { + source: string; + type: string; + direction: RangeRejectionDirection; + count: number; +} + +/** The pinned meta shape written under {@link RANGE_REJECTED_META_KEY}. */ +export interface RangeRejectionTally { + total: number; + buckets: RangeRejectionBucket[]; +} + +/** + * Classify a candidate write against the metric's declared band. + * + * Returns `null` when the value may be written. A type with no declared range + * passes — an absent range is an absent fact, never a guessed one, and the + * same asymmetry the read side uses (`isPlausibleMetricValue`). A non-finite + * value never passes, whatever the type: NaN is not an input to anything. + */ +export function classifyRangeRejection( + type: string, + value: number, +): MeasurementRangeRejection | null { + if (isPlausibleMetricValue(type, value)) return null; + if (!Number.isFinite(value)) return { type, direction: "not_finite" }; + const range = VALUE_RANGES[type]; + // `isPlausibleMetricValue` only answers false for a finite value when the + // type HAS a range, so the lookup cannot miss here. + return { + type, + direction: value < range!.min ? "below_min" : "above_max", + }; +} + +function readTally(): RangeRejectionTally { + const existing = getEvent()?.toJSON().meta?.[RANGE_REJECTED_META_KEY]; + if ( + existing !== null && + typeof existing === "object" && + Array.isArray((existing as RangeRejectionTally).buckets) + ) { + return existing as RangeRejectionTally; + } + return { total: 0, buckets: [] }; +} + +/** + * Fold refusals onto the ambient wide event. + * + * Merges with whatever the event already carries, so a provider that gates + * several batches (or reconciles row by row) accumulates instead of + * overwriting. Outside a wide-event context this is a no-op — the caller still + * drops the row, it simply has nowhere to report. + */ +export function recordRangeRejections( + source: string, + rejections: readonly MeasurementRangeRejection[], +): void { + if (rejections.length === 0) return; + const evt = getEvent(); + if (!evt) return; + + const tally = readTally(); + const firstForSource = !tally.buckets.some((b) => b.source === source); + + for (const rejection of rejections) { + const bucket = tally.buckets.find( + (b) => + b.source === source && + b.type === rejection.type && + b.direction === rejection.direction, + ); + if (bucket) bucket.count += 1; + else + tally.buckets.push({ + source, + type: rejection.type, + direction: rejection.direction, + count: 1, + }); + tally.total += 1; + } + + tally.buckets.sort( + (a, b) => + a.source.localeCompare(b.source) || + a.type.localeCompare(b.type) || + a.direction.localeCompare(b.direction), + ); + + annotate({ meta: { [RANGE_REJECTED_META_KEY]: tally } }); + if (firstForSource) { + evt.addWarning( + `${source}: refused reading(s) outside the declared plausibility range — ` + + `dropped, see meta.${RANGE_REJECTED_META_KEY}`, + ); + } +} + +/** + * Keep the rows whose value sits inside the metric's declared band; tally and + * drop the rest. + * + * `select` exists because the sync writers carry their own row shapes and none + * of them is worth reshaping at the call site: the gate reads the two fields + * it needs and returns the caller's own objects untouched. + */ +export function dropImplausibleMeasurements( + source: string, + rows: readonly T[], + select: (row: T) => { type: string; value: number }, +): T[] { + const kept: T[] = []; + const rejections: MeasurementRangeRejection[] = []; + for (const row of rows) { + const { type, value } = select(row); + const rejection = classifyRangeRejection(type, value); + if (rejection) rejections.push(rejection); + else kept.push(row); + } + recordRangeRejections(source, rejections); + return kept; +} diff --git a/src/lib/measurements/reconcile-external-measurement.ts b/src/lib/measurements/reconcile-external-measurement.ts index 031dda441..d4fb50784 100644 --- a/src/lib/measurements/reconcile-external-measurement.ts +++ b/src/lib/measurements/reconcile-external-measurement.ts @@ -4,6 +4,11 @@ import { type MeasurementAggregationProvenance, type Prisma as PrismaTypes, } from "@/generated/prisma/client"; +import { + classifyRangeRejection, + recordRangeRejections, + type MeasurementRangeRejection, +} from "@/lib/measurements/plausibility-gate"; const SAVEPOINT = "measurement_identity_reconcile"; @@ -50,6 +55,18 @@ export type MeasurementReconciliationVerdict = | { status: "failed"; error: { message: string; code?: string }; + } + | { + /** + * The incoming value sits outside the metric's declared plausibility + * band, so nothing was written and no existing row was touched. Not a + * failure: the write was refused on purpose and the caller must carry + * on with the rest of its batch. Every caller narrows on this status + * because it carries no `row` — the compiler is what makes a future + * provider handle it rather than dereference a row that does not exist. + */ + status: "rejected_range"; + rejection: MeasurementRangeRejection; }; export type FailedMeasurementReconciliation = Extract< @@ -219,12 +236,26 @@ async function retireCollision( * advisory lock serializes cooperative writers sharing either identity, and a * savepoint turns a non-benign database error into a structured `failed` * verdict without poisoning the caller's surrounding transaction. + * + * This is also the plausibility gate for every provider that reconciles rather + * than writing Prisma directly. Gating HERE rather than at each caller is the + * point: a provider added tomorrow inherits the refusal by construction, and + * cannot land an impossible reading by forgetting a check it never knew about. */ export async function reconcileExternalMeasurement( tx: PrismaTypes.TransactionClient, desired: ExternalMeasurementWrite, options: MeasurementReconciliationOptions = {}, ): Promise { + const rejection = classifyRangeRejection(desired.type, desired.value); + if (rejection) { + recordRangeRejections( + desired.source ? String(desired.source).toLowerCase() : "external", + [rejection], + ); + return { status: "rejected_range", rejection }; + } + await tx.$executeRawUnsafe(`SAVEPOINT ${SAVEPOINT}`); try { diff --git a/src/lib/nightscout/__tests__/sync-plausibility.test.ts b/src/lib/nightscout/__tests__/sync-plausibility.test.ts new file mode 100644 index 000000000..24c023eb7 --- /dev/null +++ b/src/lib/nightscout/__tests__/sync-plausibility.test.ts @@ -0,0 +1,115 @@ +/** + * The Nightscout pull writes BLOOD_GLUCOSE rows directly, so it does not + * inherit the shared reconciler's gate. It is also first-write-wins: a row + * that lands here is immutable, so an impossible sample never corrects itself + * on a later pass. These cases pin the refusal and its annotation. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { + upsertMock, + findManyMock, + emitArrivalsMock, + recomputeMock, + invalidateMock, +} = vi.hoisted(() => ({ + upsertMock: vi.fn(), + findManyMock: vi.fn(), + emitArrivalsMock: vi.fn(), + recomputeMock: vi.fn(), + invalidateMock: vi.fn(), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { measurement: { upsert: upsertMock, findMany: findManyMock } }, +})); + +vi.mock("@/lib/arrivals/measurement-emit", () => ({ + emitInsertedMeasurementArrivals: emitArrivalsMock, +})); + +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + collapseToTypeDayKeys: (rows: Array<{ type: string; measuredAt: Date }>) => + rows.map((r) => ({ type: r.type, measuredAt: r.measuredAt })), + recomputeBucketsForMeasurement: recomputeMock, +})); + +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: invalidateMock, +})); + +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate"; +import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate"; + +import { upsertNightscoutEntries } from "../sync"; + +/** BLOOD_GLUCOSE is declared 20–800 mg/dL. */ +const REAL_ENTRY = { id: "abc", sgv: 112, date: 1718000000000 }; +/** A sensor fault, or an mg/dL field carrying something else entirely. */ +const IMPOSSIBLE_ENTRY = { id: "def", sgv: 41_600, date: 1718000300000 }; + +let event: WideEventBuilder; + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +beforeEach(() => { + vi.clearAllMocks(); + findManyMock.mockResolvedValue([]); + upsertMock.mockResolvedValue({ id: "row-1" }); + emitArrivalsMock.mockResolvedValue(undefined); + recomputeMock.mockResolvedValue(undefined); + invalidateMock.mockResolvedValue(undefined); + event = new WideEventBuilder("background"); +}); + +describe("upsertNightscoutEntries — plausibility gate", () => { + it("writes a reading inside the metric's declared band", async () => { + const result = await eventStorage.run(event, () => + upsertNightscoutEntries("user-1", [REAL_ENTRY]), + ); + + expect(result).toEqual({ inserted: 1, failedRows: 0 }); + expect(upsertMock).toHaveBeenCalledTimes(1); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band reading and annotates the drop", async () => { + const result = await eventStorage.run(event, () => + upsertNightscoutEntries("user-1", [REAL_ENTRY, IMPOSSIBLE_ENTRY]), + ); + + expect(result).toEqual({ inserted: 1, failedRows: 0 }); + expect(upsertMock).toHaveBeenCalledTimes(1); + expect( + (upsertMock.mock.calls[0]![0] as { create: { value: number } }).create + .value, + ).toBe(112); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { + source: "nightscout", + type: "BLOOD_GLUCOSE", + direction: "above_max", + count: 1, + }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); + + it("writes nothing when the whole page is out of band", async () => { + const result = await eventStorage.run(event, () => + upsertNightscoutEntries("user-1", [IMPOSSIBLE_ENTRY]), + ); + + expect(result).toEqual({ inserted: 0, failedRows: 0 }); + expect(upsertMock).not.toHaveBeenCalled(); + expect(tally()?.total).toBe(1); + }); +}); diff --git a/src/lib/nightscout/sync.ts b/src/lib/nightscout/sync.ts index adef2de55..f92104550 100644 --- a/src/lib/nightscout/sync.ts +++ b/src/lib/nightscout/sync.ts @@ -22,6 +22,7 @@ import { prisma } from "@/lib/db"; import type { MeasurementType } from "@/generated/prisma/client"; import { getEvent } from "@/lib/logging/context"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { markSyncFailureRecorded, recordSyncFailure, @@ -221,7 +222,16 @@ export async function upsertNightscoutEntries( if (entries.length === 0) return { inserted: 0, failedRows: 0 }; const type = "BLOOD_GLUCOSE" as MeasurementType; - const mapped = entries.map((entry) => mapSgvEntryToMeasurement(entry)); + // A CGM that reports a glucose the app's own band calls impossible is + // reporting a sensor fault or a unit slip, not a reading. Drop it: the row + // would otherwise be immutable (first-write-wins), so an impossible sample + // that lands here never corrects itself on a later pass. + const mapped = dropImplausibleMeasurements( + "nightscout", + entries.map((entry) => mapSgvEntryToMeasurement(entry)), + (m) => ({ type, value: m.value }), + ); + if (mapped.length === 0) return { inserted: 0, failedRows: 0 }; // Probe which keys already exist so a re-confirmed reading (the upsert's // no-op update branch) is not miscounted as a fresh import and the arrival diff --git a/src/lib/oura/sync.ts b/src/lib/oura/sync.ts index 4dfefaa83..aa11c88f5 100644 --- a/src/lib/oura/sync.ts +++ b/src/lib/oura/sync.ts @@ -575,6 +575,10 @@ export async function upsertOuraMeasurements( for (let index = 0; index < readings.length; index++) { const reading = readings[index]!; const verdict = verdicts[index]!; + // Refused by the plausibility gate: nothing was written and nothing was + // touched, so the reading counts as neither imported nor a rollup dirt + // mark. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); } diff --git a/src/lib/polar/sync.ts b/src/lib/polar/sync.ts index 17b6505c5..8b44200fe 100644 --- a/src/lib/polar/sync.ts +++ b/src/lib/polar/sync.ts @@ -364,6 +364,10 @@ export async function upsertPolarMeasurements( for (let index = 0; index < readings.length; index++) { const reading = readings[index]!; const verdict = verdicts[index]!; + // Refused by the plausibility gate: nothing was written and nothing was + // touched, so the reading counts as neither imported nor a rollup dirt + // mark. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); } diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index 90998406a..10a63024c 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -255,11 +255,29 @@ const moodEntrySchema = z mood: z.string().min(1), score: z.number().int().min(0).max(10), tags: moodEntryTagsSchema, + // The free text, in both storage shapes. A portable export carries the + // decrypted `note`; a disaster-recovery file carries `noteEncrypted` plus + // whatever legacy plaintext the row still holds. Same pair, and same + // handling on restore, as `Measurement.notes` / `notesEncrypted`. + note: z.string().nullable().optional(), + noteEncrypted: base64BytesSchema.nullable().optional(), source: z.string().min(1).optional(), externalId: z.string().nullable().optional(), loggedAt: isoDateTime, + // The IANA zone the `date` string is anchored to. Absent means the legacy + // Europe/Berlin reading, which is what a row written before v1.4.25 means + // by a NULL here — so an old file keeps its old meaning exactly. + tz: z.string().nullable().optional(), + syncedAt: isoDateTime.optional(), + syncVersion: z.number().int().optional(), deletedAt: isoDateTime.nullable().optional(), + createdAt: isoDateTime.optional(), + updatedAt: isoDateTime.optional(), factors: z.array(moodFactorSchema).default([]), + // BINARY structured-tag keys. Separate from `factors`, which keeps its + // RATED-only meaning, so every file written before this one parses + // unchanged and restores identically: absent reads as none. + structuredTags: z.array(z.string().min(1)).default([]), }) .passthrough(); diff --git a/src/lib/whoop/sync-core.ts b/src/lib/whoop/sync-core.ts index 2e2e41212..464ab6718 100644 --- a/src/lib/whoop/sync-core.ts +++ b/src/lib/whoop/sync-core.ts @@ -480,6 +480,10 @@ export async function upsertWhoopMeasurements( for (let index = chunkStart; index < chunkEnd; index++) { const reading = readings[index]!; const verdict = verdicts[index - chunkStart]!; + // Refused by the plausibility gate: nothing was written and nothing was + // touched, so the reading counts as neither imported nor a rollup dirt + // mark. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); touchedTypes.add(dirty.type); diff --git a/src/lib/withings/__tests__/sync-plausibility.test.ts b/src/lib/withings/__tests__/sync-plausibility.test.ts new file mode 100644 index 000000000..8aa7572c7 --- /dev/null +++ b/src/lib/withings/__tests__/sync-plausibility.test.ts @@ -0,0 +1,261 @@ +/** + * Withings writes the measurement table directly — the measures leg row by + * row, the activity leg as a planned batch — so neither inherits the shared + * reconciler's gate. These cases pin that both legs now clear the app's own + * declared plausibility band before anything reaches Prisma, and that a + * refusal lands on the wide event rather than vanishing. + * + * The measures leg matters twice over: it does not only insert, it OVERWRITES + * the stored value of an existing slot, so an ungated impossible reading could + * replace a real one. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { + findFirst: vi.fn(), + findMany: vi.fn(), + create: vi.fn(), + createMany: vi.fn(), + update: vi.fn(), + }, + withingsConnection: { + findUnique: vi.fn(), + update: vi.fn(), + }, + $queryRaw: vi.fn(), + $transaction: vi.fn((ops: unknown) => + Array.isArray(ops) ? Promise.all(ops) : Promise.resolve(), + ), + }, +})); + +vi.mock("@/lib/crypto", () => ({ + decrypt: vi.fn((v: string) => v), + encrypt: vi.fn((v: string) => v), +})); + +vi.mock("../client", async () => { + const actual = await vi.importActual("../client"); + return { + ...actual, + fetchMeasurements: vi.fn(), + refreshAccessToken: vi.fn(), + subscribeWebhook: vi.fn(), + }; +}); + +vi.mock("../credentials", () => ({ + getUserWithingsCredentials: vi.fn(async () => ({ + clientId: "cid", + clientSecret: "secret", + })), +})); + +vi.mock("@/lib/integrations/status", () => ({ + isReauthRequired: vi.fn().mockResolvedValue(false), + parkIntegrationAtReauth: vi.fn(), + recordSyncFailure: vi.fn().mockResolvedValue(undefined), + recordSyncSuccess: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/rollups/measurement-rollups", async () => { + const actual = await vi.importActual< + typeof import("@/lib/rollups/measurement-rollups") + >("@/lib/rollups/measurement-rollups"); + return { + ...actual, + recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + }; +}); + +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserDashboardSnapshot: vi.fn(), +})); + +import { prisma } from "@/lib/db"; +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate"; +import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate"; + +import { fetchMeasurements } from "../client"; +import { syncUserActivity } from "../sync-activity"; +import { syncUserMeasurements } from "../sync"; + +let event: WideEventBuilder; + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +function installActivityFetch( + entries: Array<{ date: string; steps?: number; distance?: number }>, +) { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + status: 200, + headers: new Headers(), + json: async () => ({ + status: 0, + body: { activities: entries, more: false, offset: 0 }, + }), + })), + ); +} + +beforeEach(() => { + vi.clearAllMocks(); + event = new WideEventBuilder("background"); + vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({ + id: "conn-1", + withingsUserId: "wu-1", + accessToken: "enc-access", + refreshToken: "enc-refresh", + scope: "user.metrics,user.activity", + tokenExpiresAt: new Date(Date.now() + 60 * 60 * 1000), + lastSyncedAt: new Date(Date.now() - 60 * 60 * 1000), + } as never); + vi.mocked(prisma.withingsConnection.update).mockResolvedValue({} as never); + vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.measurement.create).mockResolvedValue({ + id: "m-1", + type: "PULSE", + measuredAt: new Date(), + } as never); + vi.mocked(prisma.measurement.createMany).mockResolvedValue({ + count: 1, + } as never); + vi.mocked(prisma.measurement.update).mockResolvedValue({} as never); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("syncUserMeasurements — plausibility gate", () => { + it("writes a reading inside the metric's declared band", async () => { + vi.mocked(fetchMeasurements).mockResolvedValue([ + { + type: "PULSE", + value: 112, + measuredAt: new Date("2026-05-16T08:00:00Z"), + }, + ] as never); + + const result = await eventStorage.run(event, () => + syncUserMeasurements("user-1"), + ); + + expect(result).toEqual({ imported: 1, failed: false }); + expect(prisma.measurement.create).toHaveBeenCalledTimes(1); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band reading and annotates the drop", async () => { + vi.mocked(fetchMeasurements).mockResolvedValue([ + { + type: "PULSE", + value: 112, + measuredAt: new Date("2026-05-16T08:00:00Z"), + }, + { + type: "PULSE", + value: 55643821.505, + measuredAt: new Date("2026-05-16T08:05:00Z"), + }, + ] as never); + + const result = await eventStorage.run(event, () => + syncUserMeasurements("user-1"), + ); + + // Only the real reading landed, and the run is still a clean success: one + // impossible sample must not strand the rest behind a held watermark. + expect(result).toEqual({ imported: 1, failed: false }); + expect(prisma.measurement.create).toHaveBeenCalledTimes(1); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { source: "withings", type: "PULSE", direction: "above_max", count: 1 }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); + + it("does not overwrite a stored reading with an out-of-band one", async () => { + // The slot already holds a real pulse; the provider re-sends the same + // instant carrying a decode slip. Ungated, the update branch rewrote the + // good value with the bad one. + vi.mocked(prisma.measurement.findFirst).mockResolvedValue({ + id: "existing-row", + } as never); + vi.mocked(fetchMeasurements).mockResolvedValue([ + { + type: "PULSE", + value: 55643821.505, + measuredAt: new Date("2026-05-16T08:00:00Z"), + }, + ] as never); + + const result = await eventStorage.run(event, () => + syncUserMeasurements("user-1"), + ); + + expect(result).toEqual({ imported: 0, failed: false }); + expect(prisma.measurement.update).not.toHaveBeenCalled(); + expect(tally()?.total).toBe(1); + }); +}); + +describe("syncUserActivity — plausibility gate", () => { + it("plans a daily total inside the metric's declared band", async () => { + installActivityFetch([{ date: "2026-05-16", steps: 8_400 }]); + + const imported = await eventStorage.run(event, () => + syncUserActivity("user-1"), + ); + + expect(imported).toBe(1); + expect(prisma.measurement.createMany).toHaveBeenCalledTimes(1); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band daily total and annotates the drop", async () => { + // ACTIVITY_STEPS is declared 0–200 000. + installActivityFetch([ + { date: "2026-05-16", steps: 8_400 }, + { date: "2026-05-17", steps: 9_900_000 }, + ]); + + const imported = await eventStorage.run(event, () => + syncUserActivity("user-1"), + ); + + expect(imported).toBe(1); + const created = vi.mocked(prisma.measurement.createMany).mock.calls[0]?.[0] + ?.data as Array<{ value: number }>; + expect(created).toHaveLength(1); + expect(created[0]!.value).toBe(8_400); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { + source: "withings", + type: "ACTIVITY_STEPS", + direction: "above_max", + count: 1, + }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); +}); diff --git a/src/lib/withings/sync-activity.ts b/src/lib/withings/sync-activity.ts index 869fb1d18..a4a0098b2 100644 --- a/src/lib/withings/sync-activity.ts +++ b/src/lib/withings/sync-activity.ts @@ -39,6 +39,7 @@ import { prisma } from "@/lib/db"; import { getEvent } from "@/lib/logging/context"; import { safeFetch } from "@/lib/safe-fetch"; import { getUnitForType } from "@/lib/validations/measurement"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { collapseToTypeDayKeys, recomputeBucketsForMeasurement, @@ -342,7 +343,7 @@ export async function syncUserActivity( value: number; externalId: string; } - const planned: PlannedRow[] = []; + const collected: PlannedRow[] = []; for (const entry of entries) { if (!entry.date) continue; const measuredAt = activityMeasuredAt(entry.date); @@ -352,7 +353,7 @@ export async function syncUserActivity( // valid (a day of rest still records 0 steps). Only skip // undefined / null / NaN. if (raw == null || !Number.isFinite(raw)) continue; - planned.push({ + collected.push({ type, measuredAt, value: raw, @@ -361,6 +362,15 @@ export async function syncUserActivity( } } + // A daily-activity total the app's own band calls impossible (a step count + // past 200 000, a negative distance) is a provider artefact, not a day. Drop + // it before the plan is read against the table, so it can neither insert a + // fresh impossible slot nor overwrite a good stored one. + const planned = dropImplausibleMeasurements("withings", collected, (p) => ({ + type: p.type, + value: p.value, + })); + if (planned.length > 0) { const slotKey = (type: MeasurementType, measuredAt: Date) => `${type}|${measuredAt.getTime()}`; diff --git a/src/lib/withings/sync-sleep.ts b/src/lib/withings/sync-sleep.ts index 840b5d8d3..65099ea7c 100644 --- a/src/lib/withings/sync-sleep.ts +++ b/src/lib/withings/sync-sleep.ts @@ -474,6 +474,9 @@ export async function syncUserSleep( if (verdict.status === "failed") { throw new MeasurementReconciliationError(verdict); } + // Refused by the plausibility gate: nothing was written and nothing + // was touched. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); } @@ -542,6 +545,9 @@ export async function syncUserSleep( if (verdict.status === "failed") { throw new MeasurementReconciliationError(verdict); } + // Refused by the plausibility gate: nothing was written and + // nothing was touched. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); } diff --git a/src/lib/withings/sync.ts b/src/lib/withings/sync.ts index fcee1cc7c..d500a030c 100644 --- a/src/lib/withings/sync.ts +++ b/src/lib/withings/sync.ts @@ -15,6 +15,7 @@ import { getUserWithingsCredentials } from "./credentials"; import { annotate, getEvent } from "@/lib/logging/context"; import { isP2002 } from "@/lib/prisma-errors"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { isReauthRequired, recordSyncFailure, @@ -229,14 +230,24 @@ export async function syncUserMeasurements( ) : new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days back - let measures: Awaited>; + let fetched: Awaited>; try { - measures = await fetchMeasurements(tokenInfo.accessToken, startDate); + fetched = await fetchMeasurements(tokenInfo.accessToken, startDate); } catch (err) { await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_MEASURES); throw err; } + // The app's own plausibility band applies to a provider reading exactly as + // it applies to a typed one. A remote that reports a pulse in the millions + // is reporting a decode slip, not a heartbeat, and the row would poison + // every mean and personal band derived from it. Refuse it here, before the + // update branch below can overwrite a good stored value with it. + const measures = dropImplausibleMeasurements("withings", fetched, (m) => ({ + type: m.type, + value: m.value, + })); + let imported = 0; // v1.4.39.1 — track every (type, measuredAt) we touched so the // persistent rollup tier can be re-folded at the end of the sync. diff --git a/tests/integration/cycle-flow-opens-cycle.test.ts b/tests/integration/cycle-flow-opens-cycle.test.ts index 5e81fda56..7a9fe7d84 100644 --- a/tests/integration/cycle-flow-opens-cycle.test.ts +++ b/tests/integration/cycle-flow-opens-cycle.test.ts @@ -70,6 +70,14 @@ beforeEach(async () => { username: "cycle-flow", email: "cycle-flow@example.test", gender: "FEMALE", + // The suite runs under TZ=UTC and computes its relative day keys with + // local Date arithmetic, i.e. in UTC. The route resolves "today" in the + // USER's zone and falls back to Europe/Berlin when none is set - one + // hour past 22:00 UTC the two clocks disagree about the date and every + // day-of-cycle assertion is off by one. Pinning the fixture zone to UTC + // makes the test's clock and the route's clock the same clock at any + // hour of the day. + timezone: "UTC", }, }); await loginAs(USER_ID); diff --git a/tests/integration/cycle-routes.test.ts b/tests/integration/cycle-routes.test.ts index 8dc05e92b..d75c8822f 100644 --- a/tests/integration/cycle-routes.test.ts +++ b/tests/integration/cycle-routes.test.ts @@ -69,6 +69,14 @@ beforeEach(async () => { username: "cycle-female", email: "cycle-female@example.test", gender: "FEMALE", + // The suite runs under TZ=UTC and computes its relative day keys with + // local Date arithmetic, i.e. in UTC. The route resolves "today" in the + // USER's zone and falls back to Europe/Berlin when none is set - one + // hour past 22:00 UTC the two clocks disagree about the date and every + // day-of-cycle assertion is off by one. Pinning the fixture zone to UTC + // makes the test's clock and the route's clock the same clock at any + // hour of the day. + timezone: "UTC", }, }); await prisma.user.create({ @@ -77,6 +85,7 @@ beforeEach(async () => { username: "cycle-male", email: "cycle-male@example.test", gender: "MALE", + timezone: "UTC", }, }); }); diff --git a/tests/integration/mood-backup-round-trip.test.ts b/tests/integration/mood-backup-round-trip.test.ts new file mode 100644 index 000000000..42c43e088 --- /dev/null +++ b/tests/integration/mood-backup-round-trip.test.ts @@ -0,0 +1,439 @@ +/** + * A mood entry, out through the real backup and back in through the real + * restore, asserted on the rows. + * + * Deliberately not a classification test. Asserting that the payload has a + * `moodEntries` key proves nothing this repository has not already been burned + * by: excise the restore call entirely and a shape check still reports green. + * So this seeds an account, exports through `buildFullBackupPayload`, parses + * through `parseBackupPayload`, restores through the real + * `POST /api/admin/backups/[id]/restore`, and then reads the row back out of + * Postgres and compares values. + * + * The fixture is the entry that used to come back wrong in four ways at once: + * a ticked BINARY tag and a rated factor on the same entry, a note in the + * encrypted column, a timezone that is not the legacy Europe/Berlin default, + * and a sync counter above zero. The old payload carried none of them. + * + * The second case is the compatibility floor: a hand-written payload in the + * shape files were written in BEFORE this change, restored through the same + * route. Every field this adds is optional or defaulted precisely so that file + * keeps working, and the only way to know is to send one. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import type { PrismaClient } from "@/generated/prisma/client"; +import { encrypt } from "@/lib/crypto"; +import { decryptFromBytes } from "@/lib/ai/coach/bytes-codec"; +import { encryptNote } from "@/lib/crypto/note-cipher"; +import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { parseBackupPayload } from "@/lib/validations/backup"; +import { POST } from "@/app/api/admin/backups/[id]/restore/route"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserData: vi.fn(), +})); + +const OWNER_ID = "mood-round-trip-owner"; +const NOTE = "slept badly, long walk in the afternoon"; +const LOGGED_AT = new Date("2026-07-19T23:40:00.000Z"); +const CREATED_AT = new Date("2026-07-19T23:41:00.000Z"); +const UPDATED_AT = new Date("2026-07-20T06:05:00.000Z"); +const DELETED_AT = new Date("2026-07-20T07:00:00.000Z"); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +async function seedAdminSession(prisma: PrismaClient) { + const admin = await prisma.user.create({ + data: { + username: "mood-round-trip-admin", + email: "mood-round-trip-admin@example.test", + role: "ADMIN", + }, + }); + const session = await prisma.session.create({ + data: { userId: admin.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return admin; +} + +async function seedOwner(prisma: PrismaClient) { + await prisma.user.create({ + data: { + id: OWNER_ID, + username: "mood-round-trip-owner", + email: "mood-round-trip-owner@example.test", + }, + }); +} + +/** + * The taxonomy the entry links to: one seeded BINARY tag (`userId: null`, the + * reference data every instance has), one BINARY tag the account made itself, + * and one RATED factor. The account's own tag matters — it travels in the file + * as a `customMoodTags` row, and a restore that could not re-create it would + * lose the link even with everything else correct. + */ +async function seedTaxonomy(prisma: PrismaClient) { + const category = await prisma.moodTagCategory.create({ + data: { + id: "mood-rt-category", + key: "mood_round_trip", + labelKey: "mood.tagCategory.roundTrip", + }, + }); + const seededTag = await prisma.moodTag.create({ + data: { + id: "mood-rt-seeded-tag", + categoryId: category.id, + key: "rt_headache", + labelKey: "mood.tag.rtHeadache", + kind: "BINARY", + }, + }); + const ownTag = await prisma.moodTag.create({ + data: { + id: "mood-rt-own-tag", + userId: OWNER_ID, + categoryId: category.id, + key: "custom:rt_long_walk", + labelKey: "mood.tag.custom", + kind: "BINARY", + }, + }); + const factor = await prisma.moodTag.create({ + data: { + id: "mood-rt-factor", + categoryId: category.id, + key: "rt_sleep_quality", + labelKey: "mood.tag.rtSleepQuality", + kind: "RATED", + scaleMin: 1, + scaleMax: 5, + }, + }); + return { category, seededTag, ownTag, factor }; +} + +function makeRestoreRequest(id: string) { + return new Request(`http://localhost/api/admin/backups/${id}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }); +} + +async function restoreFromPayload( + prisma: PrismaClient, + payload: unknown, + type: string, +) { + const backup = await prisma.dataBackup.create({ + data: { + userId: OWNER_ID, + type, + data: encrypt(JSON.stringify(payload)), + }, + }); + return POST(makeRestoreRequest(backup.id) as never, { + params: Promise.resolve({ id: backup.id }), + }); +} + +describe("mood backup round trip", () => { + it("brings the note, the timezone, the sync counter and both tag arms back", async () => { + const prisma = getPrismaClient(); + await seedAdminSession(prisma); + await seedOwner(prisma); + const { seededTag, ownTag, factor } = await seedTaxonomy(prisma); + + await prisma.moodEntry.create({ + data: { + id: "mood-rt-entry", + userId: OWNER_ID, + date: "2026-07-19", + mood: "OKAY", + score: 3, + tags: JSON.stringify(["free-text-tag"]), + note: null, + noteEncrypted: encryptNote(NOTE), + source: "MANUAL", + externalId: null, + moodLoggedAt: LOGGED_AT, + // Not Europe/Berlin: a restore that drops `tz` moves this entry's day + // boundary by six hours and nothing else in the row shows it. + tz: "America/New_York", + syncVersion: 3, + createdAt: CREATED_AT, + updatedAt: UPDATED_AT, + tagLinks: { + create: [ + { moodTagId: factor.id, rating: 4 }, + { moodTagId: seededTag.id, rating: null }, + { moodTagId: ownTag.id, rating: null }, + ], + }, + }, + }); + await prisma.moodEntry.create({ + data: { + id: "mood-rt-tombstone", + userId: OWNER_ID, + date: "2026-07-18", + mood: "SCHLECHT", + score: 2, + source: "MANUAL", + moodLoggedAt: new Date("2026-07-18T22:00:00.000Z"), + tz: "America/New_York", + syncVersion: 5, + deletedAt: DELETED_AT, + }, + }); + + const { payload } = await buildFullBackupPayload(prisma, OWNER_ID, { + purpose: "disaster-recovery", + }); + const parsed = parseBackupPayload(payload); + + // Wipe what the restore is supposed to bring back. The tag rows stay: the + // instance's catalogue is reference data, and the account's own tag rides + // in the file and is re-created by the restore either way. + await prisma.moodEntry.deleteMany({ where: { userId: OWNER_ID } }); + await prisma.moodTag.deleteMany({ where: { userId: OWNER_ID } }); + expect(await prisma.moodEntry.count({ where: { userId: OWNER_ID } })).toBe( + 0, + ); + + const response = await restoreFromPayload( + prisma, + parsed, + "MOOD_ROUND_TRIP", + ); + expect(response.status).toBe(200); + const body = (await response.json()) as { + data: { skipped: { links: number } }; + }; + // Nothing was dropped. A restore that lost a link and still answered 200 + // is exactly what the skip report exists to make visible. + expect(body.data.skipped.links).toBe(0); + + const restored = await prisma.moodEntry.findUniqueOrThrow({ + where: { id: "mood-rt-entry" }, + }); + // Asserted as "there is a note" before it is decrypted, so a payload that + // stopped carrying the ciphertext fails by name rather than as a TypeError + // three frames inside the cipher. + expect( + restored.noteEncrypted, + "the restored entry has no note ciphertext — the backup dropped it", + ).not.toBeNull(); + expect(decryptFromBytes(restored.noteEncrypted!)).toBe(NOTE); + expect(restored.note).toBeNull(); + expect(restored.tz).toBe("America/New_York"); + expect(restored.syncVersion).toBe(3); + expect(restored.moodLoggedAt).toEqual(LOGGED_AT); + expect(restored.createdAt).toEqual(CREATED_AT); + expect(restored.updatedAt).toEqual(UPDATED_AT); + expect(restored.date).toBe("2026-07-19"); + expect(restored.score).toBe(3); + expect(restored.tags).toBe(JSON.stringify(["free-text-tag"])); + expect(restored.deletedAt).toBeNull(); + + const tombstone = await prisma.moodEntry.findUniqueOrThrow({ + where: { id: "mood-rt-tombstone" }, + }); + expect(tombstone.deletedAt).toEqual(DELETED_AT); + expect(tombstone.syncVersion).toBe(5); + + const links = await prisma.moodEntryTagLink.findMany({ + where: { moodEntryId: "mood-rt-entry" }, + select: { rating: true, moodTag: { select: { key: true, kind: true } } }, + orderBy: { moodTagId: "asc" }, + }); + const byKey = new Map(links.map((l) => [l.moodTag.key, l])); + expect([...byKey.keys()].sort()).toEqual([ + "custom:rt_long_walk", + "rt_headache", + "rt_sleep_quality", + ]); + // The rating is what tells the two arms apart in the row: a ticked tag + // carries NULL, and that NULL is what the old backup filter read as + // "nothing to carry". + expect(byKey.get("rt_headache")!.rating).toBeNull(); + expect(byKey.get("custom:rt_long_walk")!.rating).toBeNull(); + expect(byKey.get("rt_sleep_quality")!.rating).toBe(4); + }); + + it("restores a file written in the shape that predates these fields", async () => { + const prisma = getPrismaClient(); + await seedAdminSession(prisma); + await seedOwner(prisma); + const { factor } = await seedTaxonomy(prisma); + + // Verbatim the shape the builder emitted before this change: no `note`, no + // `noteEncrypted`, no `tz`, no `syncedAt`, no `syncVersion`, no + // `createdAt`/`updatedAt`, no `structuredTags`. Hand-written rather than + // generated, because a fixture built from today's builder cannot describe + // yesterday's file. + const legacyPayload = { + schemaVersion: "2", + exportedAt: "2026-07-20T00:00:00.000Z", + userId: OWNER_ID, + appSettings: null, + measurements: [], + medications: [], + intakeEvents: [], + moodEntries: [ + { + id: "mood-rt-legacy", + date: "2026-07-19", + mood: "GUT", + score: 4, + tags: null, + source: "MOODLOG", + loggedAt: LOGGED_AT.toISOString(), + externalId: "moodlog-legacy", + deletedAt: null, + factors: [{ key: factor.key, rating: 5 }], + }, + ], + customMoodTags: [], + nutrientDays: [], + }; + + const parsed = parseBackupPayload(legacyPayload); + // Defaults, not values: absence has to stay absence through the parse. + expect(parsed.moodEntries[0].structuredTags).toEqual([]); + expect(parsed.moodEntries[0].tz).toBeUndefined(); + expect(parsed.moodEntries[0].syncVersion).toBeUndefined(); + + const response = await restoreFromPayload( + prisma, + parsed, + "MOOD_LEGACY_SHAPE", + ); + expect(response.status).toBe(200); + + const restored = await prisma.moodEntry.findUniqueOrThrow({ + where: { id: "mood-rt-legacy" }, + }); + // Identical to what the old restore wrote for the same file: the note + // columns empty, `tz` NULL — which is what such a row meant, the legacy + // Europe/Berlin reading — and the counter at the schema default. + expect(restored.note).toBeNull(); + expect(restored.noteEncrypted).toBeNull(); + expect(restored.tz).toBeNull(); + expect(restored.syncVersion).toBe(0); + expect(restored.mood).toBe("GUT"); + expect(restored.score).toBe(4); + expect(restored.externalId).toBe("moodlog-legacy"); + expect(restored.deletedAt).toBeNull(); + + const links = await prisma.moodEntryTagLink.findMany({ + where: { moodEntryId: "mood-rt-legacy" }, + select: { rating: true, moodTagId: true }, + }); + expect(links).toEqual([{ moodTagId: factor.id, rating: 5 }]); + }); + + it("names a ticked tag this instance cannot resolve instead of losing it quietly", async () => { + const prisma = getPrismaClient(); + await seedAdminSession(prisma); + await seedOwner(prisma); + await seedTaxonomy(prisma); + + const payload = { + schemaVersion: "2", + exportedAt: "2026-07-20T00:00:00.000Z", + userId: OWNER_ID, + appSettings: null, + measurements: [], + medications: [], + intakeEvents: [], + moodEntries: [ + { + id: "mood-rt-unknown-tag", + date: "2026-07-19", + mood: "OKAY", + score: 3, + tags: null, + source: "MANUAL", + loggedAt: LOGGED_AT.toISOString(), + externalId: null, + deletedAt: null, + factors: [], + structuredTags: ["rt_headache", "rt_retired_tag"], + }, + ], + customMoodTags: [], + nutrientDays: [], + }; + + const response = await restoreFromPayload( + prisma, + parseBackupPayload(payload), + "MOOD_UNKNOWN_TAG", + ); + expect(response.status).toBe(200); + const body = (await response.json()) as { + data: { + skipped: { + links: number; + catalogueKeys: Array<{ + catalogue: string; + key: string; + links: number; + }>; + }; + }; + }; + expect(body.data.skipped.catalogueKeys).toEqual([ + { catalogue: "moodTag", key: "rt_retired_tag", links: 1 }, + ]); + expect(body.data.skipped.links).toBe(1); + + // The entry itself came back whole. One key that has nowhere to attach + // costs the link, never the record. + const restored = await prisma.moodEntry.findUniqueOrThrow({ + where: { id: "mood-rt-unknown-tag" }, + }); + expect(restored.score).toBe(3); + const links = await prisma.moodEntryTagLink.findMany({ + where: { moodEntryId: restored.id }, + select: { moodTag: { select: { key: true } } }, + }); + expect(links).toEqual([{ moodTag: { key: "rt_headache" } }]); + }); +});