From 1bdc5ea4eefdc5d5e85bd2f3b353ba85915cc957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 7 Aug 2026 20:57:58 +0200 Subject: [PATCH 1/9] test(backup): bind MoodEntry to a named backup select MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mood read was an inline `include`, so nothing could compare it to the model the way `measurement-backup-completeness.test.ts` compares the measurement select. Name it — `MOOD_ENTRY_BACKUP_SELECT` and `MOOD_ENTRY_TAG_LINK_SELECT` — and add the sibling guard that binds both to `prisma/schema.prisma`. The mapper's parameter is spelled `moodEntry` rather than `e` because the guard proves a column reaches a payload by matching `: moodEntry.`, and `e` is also the intake-event mapper's parameter one section above: a matcher on `: e.` would match across both and could never fail. The guard is red as it stands, and names the defect: `note`, `noteEncrypted`, `tz`, `syncedAt`, `syncVersion`, `createdAt` and `updatedAt` are selected and written to no payload. The fix follows. --- .../mood-backup-completeness.test.ts | 263 ++++++++++++++++++ .../__tests__/full-backup-payload.test.ts | 8 +- src/lib/export/full-backup-payload.ts | 79 ++++-- 3 files changed, 331 insertions(+), 19 deletions(-) create mode 100644 src/__tests__/mood-backup-completeness.test.ts diff --git a/src/__tests__/mood-backup-completeness.test.ts b/src/__tests__/mood-backup-completeness.test.ts new file mode 100644 index 000000000..ca7eb3b0f --- /dev/null +++ b/src/__tests__/mood-backup-completeness.test.ts @@ -0,0 +1,263 @@ +/** + * Pair guard binding `prisma/schema.prisma`'s `MoodEntry` and + * `MoodEntryTagLink` models to the disaster-recovery backup selects. + * + * The sibling of `measurement-backup-completeness.test.ts`, written as its own + * file rather than as a second describe inside it, because that guard's own + * note asks for model-by-model growth and one file per model keeps every + * failure message about one model. + * + * The failure it exists to catch, stated by the defect that prompted it: the + * mood payload carried `id, date, mood, score, tags, source, loggedAt, + * externalId, deletedAt, factors` and nothing else. The free-text note, the + * row's timezone and the last-writer-wins counter were selected by the query + * and written to no payload, so a restored entry came back without its text, + * read its day boundaries under the legacy Europe/Berlin assumption, and lost + * the next sync round to a paired device still holding a higher counter. The + * backup validation schema is `.passthrough()`, so nothing complained at parse + * time either. Nothing here was a wrong value; every one of them was silence. + * + * Both ends of the comparison are literals — a Prisma model block and a named + * select object — so the matcher is exact. Every matcher also asserts a + * non-zero match count: an empty match set fails rather than passing quietly, + * which is how the bearer-scope guard stayed green for months while matching + * nothing at all. + * + * Mutation check: comment out `tz: true` in `MOOD_ENTRY_BACKUP_SELECT` and + * this goes red naming that column; add a scalar to the model without adding + * it to the select and it goes red naming the new one; rename the mapper's + * `moodEntry` parameter and the serialization assertion goes red on all of + * them at once. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, it, expect } from "vitest"; + +const ROOT = process.cwd(); +const SCHEMA = join(ROOT, "prisma", "schema.prisma"); +const BACKUP_BUILDER = join( + ROOT, + "src", + "lib", + "export", + "full-backup-payload.ts", +); + +/** + * Columns the backup is correct to omit, each with the reason. The reasons are + * length-checked below: an exclusion is a claim, and a claim needs saying. + */ +const EXCLUDED_MOOD_ENTRY_COLUMNS: Record = { + userId: "user-scoped backup; owner is re-derived from the restoring account", +}; + +const EXCLUDED_TAG_LINK_COLUMNS: Record = { + moodEntryId: "re-bound to the restored parent row's id, never carried", + moodTagId: + "the link travels by the tag's key; a source-instance id need not exist in the target catalogue", + createdAt: + "the link's own row stamp; the entry carries the moment the person logged it, and a key-only BINARY tag has nowhere to put a second one", +}; + +function modelBlock(schema: string, model: string): string { + const start = schema.indexOf(`model ${model} {`); + expect(start, `model ${model} not found`).toBeGreaterThan(-1); + const end = schema.indexOf("\n}", start); + expect(end).toBeGreaterThan(start); + return schema.slice(start, end); +} + +/** Names of every `model` block, so relation fields can be told apart. */ +function declaredModels(schema: string): Set { + const out = new Set(); + for (const m of schema.matchAll(/^model\s+([A-Za-z_]\w*)\s*\{/gm)) { + out.add(m[1]); + } + return out; +} + +/** + * Scalar (column-backed) fields of a model block: everything that is not a + * relation to another model. Enums and native scalars both map to columns and + * therefore both belong in a backup. + */ +function scalarFields(block: string, models: Set): string[] { + const fields: string[] = []; + for (const rawLine of block.split("\n").slice(1)) { + const line = rawLine.trim(); + if (!line || line.startsWith("//") || line.startsWith("@@")) continue; + const m = /^([a-zA-Z_]\w*)\s+([A-Za-z_]\w*)(\[\])?(\?)?/.exec(line); + if (!m) continue; + const [, name, type, isList] = m; + if (isList) continue; + if (models.has(type)) continue; + if (line.includes("@relation")) continue; + fields.push(name); + } + return fields; +} + +/** Keys of a named `const = {` select literal. */ +function selectFields(source: string, constant: string): string[] { + const start = source.indexOf(`const ${constant} = {`); + expect(start, `${constant} not found`).toBeGreaterThan(-1); + const end = source.indexOf("\n}", start); + expect(end).toBeGreaterThan(start); + const body = source.slice(start, end); + return [...body.matchAll(/^ {2}([a-zA-Z_$][\w$]*): true,$/gm)].map( + (m) => m[1], + ); +} + +/** + * Columns either serialization arm actually reads off the mapped row. + * + * The capture is the RIGHT-hand side — the column — not the payload key, which + * matters here in a way it did not for `Measurement`: the mood payload renames + * `moodLoggedAt` to `loggedAt` on the wire, so a left-hand matcher would report + * the column as dropped while the value rides perfectly well. + * + * The two arms share one restorable wire format, so this is a union and + * therefore a weaker claim than the select comparison: it catches a column that + * is selected and then written to neither payload, not one carried by only a + * single arm. + */ +function serialisedFields(source: string, variable: string): Set { + const matches = [ + ...source.matchAll( + new RegExp(`[:(,]\\s*${variable}\\.([a-zA-Z_$][\\w$]*)`, "g"), + ), + ]; + expect( + matches.length, + `no \`${variable}.\` reads found — the mapper parameter was renamed and this matcher now proves nothing`, + ).toBeGreaterThan(0); + return new Set(matches.map((m) => m[1])); +} + +describe("mood backup completeness", () => { + const schema = readFileSync(SCHEMA, "utf8"); + const source = readFileSync(BACKUP_BUILDER, "utf8"); + const models = declaredModels(schema); + + const entryColumns = scalarFields(modelBlock(schema, "MoodEntry"), models); + const entrySelected = selectFields(source, "MOOD_ENTRY_BACKUP_SELECT"); + const linkColumns = scalarFields( + modelBlock(schema, "MoodEntryTagLink"), + models, + ); + const linkSelected = selectFields(source, "MOOD_ENTRY_TAG_LINK_SELECT"); + + it("parses both ends into a plausible shape", () => { + expect(models.size).toBeGreaterThan(50); + expect(entryColumns.length).toBeGreaterThan(10); + expect(entryColumns).toContain("noteEncrypted"); + expect(entryColumns).toContain("syncVersion"); + expect(entryColumns).not.toContain("user"); + expect(entryColumns).not.toContain("tagLinks"); + expect(entrySelected.length).toBeGreaterThan(10); + + expect(linkColumns).toEqual( + expect.arrayContaining([ + "moodEntryId", + "moodTagId", + "createdAt", + "rating", + ]), + ); + expect(linkColumns).not.toContain("moodEntry"); + expect(linkColumns).not.toContain("moodTag"); + expect(linkSelected.length).toBeGreaterThan(0); + }); + + it("the backup select covers every MoodEntry column", () => { + const missing = entryColumns.filter( + (c) => !entrySelected.includes(c) && !(c in EXCLUDED_MOOD_ENTRY_COLUMNS), + ); + + if (missing.length > 0) { + const report = missing.map((c) => ` ❌ ${c}`).join("\n"); + throw new Error( + `MOOD_ENTRY_BACKUP_SELECT omits ${missing.length} column(s) of the MoodEntry model:\n${report}\n\n` + + "A disaster-recovery restore brings these back at their column " + + "default. Add each to the select and to both serialization arms, " + + "extend `moodEntrySchema` and the restore write — or add it to " + + "EXCLUDED_MOOD_ENTRY_COLUMNS with the reason the backup is correct " + + "to drop it.", + ); + } + }); + + it("every selected MoodEntry column reaches a serialized payload", () => { + const emitted = serialisedFields(source, "moodEntry"); + const dropped = entrySelected.filter((c) => !emitted.has(c)); + + if (dropped.length > 0) { + const report = dropped.map((c) => ` ❌ ${c}`).join("\n"); + throw new Error( + `${dropped.length} selected MoodEntry column(s) reach neither serialization arm:\n${report}\n\n` + + "The query pays for them and no payload carries them, so a restore " + + "cannot put them back. Emit each from the disaster-recovery arm " + + "(and, where it is readable and not a tombstone, from the portable " + + "arm too).", + ); + } + }); + + it("the tag-link select covers every MoodEntryTagLink column", () => { + const missing = linkColumns.filter( + (c) => !linkSelected.includes(c) && !(c in EXCLUDED_TAG_LINK_COLUMNS), + ); + expect( + missing, + `MOOD_ENTRY_TAG_LINK_SELECT omits MoodEntryTagLink column(s): ${missing.join(", ")}`, + ).toEqual([]); + }); + + it("every selected tag-link column reaches a serialized payload", () => { + const emitted = serialisedFields(source, "tagLink"); + const dropped = linkSelected.filter((c) => !emitted.has(c)); + expect( + dropped, + `selected but serialized by neither arm: ${dropped.join(", ")}`, + ).toEqual([]); + }); + + it("both structured-tag arms are carried, not just the rated one", () => { + // The defect this file was written for: the tag-link read filtered + // `rating: { not: null }`, and a BINARY link carries a NULL rating by + // definition. Every present/absent tag — the kind most people pick — was + // excluded from the file and could not come back. + // + // Asserted on a boolean rather than on the file contents so a failure + // reads as one line instead of printing the whole builder back. + expect( + source.includes("rating: { not: null }"), + "the tag-link read filters out NULL ratings again; BINARY links are being dropped from the backup", + ).toBe(false); + expect( + source.includes("structuredTags"), + "no `structuredTags` key in the payload — the BINARY links have no carrier", + ).toBe(true); + }); + + it("the exclusion lists carry no stale entry", () => { + for (const [column, reason] of Object.entries( + EXCLUDED_MOOD_ENTRY_COLUMNS, + )) { + expect( + entryColumns, + `${column} is excluded but is no longer a MoodEntry column`, + ).toContain(column); + expect(reason.length).toBeGreaterThan(20); + } + for (const [column, reason] of Object.entries(EXCLUDED_TAG_LINK_COLUMNS)) { + expect( + linkColumns, + `${column} is excluded but is no longer a MoodEntryTagLink column`, + ).toContain(column); + expect(reason.length).toBeGreaterThan(20); + } + }); +}); diff --git a/src/lib/export/__tests__/full-backup-payload.test.ts b/src/lib/export/__tests__/full-backup-payload.test.ts index 4d76bc57e..3632d9042 100644 --- a/src/lib/export/__tests__/full-backup-payload.test.ts +++ b/src/lib/export/__tests__/full-backup-payload.test.ts @@ -409,7 +409,13 @@ describe("buildFullBackupPayload disaster-recovery mode", () => { expect(prisma.moodEntry.findMany).toHaveBeenCalledWith( expect.objectContaining({ where: { userId: "user-1" }, - include: expect.any(Object), + select: expect.objectContaining({ + id: true, + noteEncrypted: true, + tz: true, + syncVersion: true, + deletedAt: true, + }), }), ); expect(mocks.buildRecordsBackupSection).toHaveBeenCalledWith( diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index 57ef86451..08298bf65 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -103,6 +103,46 @@ const MEASUREMENT_BACKUP_SELECT = { updatedAt: true, } as const satisfies Prisma.MeasurementSelect; +/** + * Every restorable `MoodEntry` scalar. `userId` is absent for the same reason + * it is absent above: the backup is user-scoped and the owner is re-derived + * from the restoring account. + * + * Named rather than inlined so `mood-backup-completeness.test.ts` can compare + * it field-by-field against the model in `prisma/schema.prisma`, exactly as the + * measurement guard does for the select above. + */ +const MOOD_ENTRY_BACKUP_SELECT = { + id: true, + date: true, + mood: true, + score: true, + tags: true, + note: true, + noteEncrypted: true, + source: true, + externalId: true, + moodLoggedAt: true, + tz: true, + syncedAt: true, + syncVersion: true, + deletedAt: true, + createdAt: true, + updatedAt: true, +} as const satisfies Prisma.MoodEntrySelect; + +/** + * A structured-tag link, as it rides the wire. The link is carried by the + * tag's KEY, not by its id: a restore resolves the key against the target + * instance's catalogue, where the source instance's row id may not exist at + * all. `kind` comes along so the mapper can split RATED factors from BINARY + * tags without a second query. + */ +const MOOD_ENTRY_TAG_LINK_SELECT = { + rating: true, + moodTag: { select: { key: true, kind: true } }, +} as const satisfies Prisma.MoodEntryTagLinkSelect; + /** * Build the canonical full-backup payload for `userId`. Portable exports omit * tombstones and document ciphertext; disaster-recovery payloads preserve @@ -207,13 +247,11 @@ export async function buildFullBackupPayload( prisma.moodEntry.findMany({ where: disasterRecovery ? { userId } : { userId, deletedAt: null }, orderBy: { moodLoggedAt: "desc" }, - include: { + select: { + ...MOOD_ENTRY_BACKUP_SELECT, tagLinks: { where: { rating: { not: null } }, - select: { - rating: true, - moodTag: { select: { key: true } }, - }, + select: MOOD_ENTRY_TAG_LINK_SELECT, }, }, }), @@ -394,19 +432,24 @@ export async function buildFullBackupPayload( skipped: e.skipped, source: e.source, })), - moodEntries: moodEntries.map((e) => ({ - id: e.id, - date: e.date, - mood: e.mood, - score: e.score, - tags: e.tags, - source: e.source, - loggedAt: e.moodLoggedAt.toISOString(), - externalId: e.externalId, - deletedAt: e.deletedAt?.toISOString() ?? null, - factors: e.tagLinks.map((link) => ({ - key: link.moodTag.key, - rating: link.rating!, + // The parameter is spelled out rather than the `e` every other mapper here + // uses, and deliberately: `mood-backup-completeness.test.ts` proves a + // selected column reaches a payload by matching `: moodEntry.`. A matcher + // on `: e.` would also match the intake-event mapper above it and could + // therefore never fail. + moodEntries: moodEntries.map((moodEntry) => ({ + id: moodEntry.id, + date: moodEntry.date, + mood: moodEntry.mood, + score: moodEntry.score, + tags: moodEntry.tags, + source: moodEntry.source, + loggedAt: moodEntry.moodLoggedAt.toISOString(), + externalId: moodEntry.externalId, + deletedAt: moodEntry.deletedAt?.toISOString() ?? null, + factors: moodEntry.tagLinks.map((tagLink) => ({ + key: tagLink.moodTag.key, + rating: tagLink.rating!, })), })), customMoodTags: customMoodTags.map((tag) => ({ From 222047e0d48b355235705c9bb29e6e0ee990ea41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 7 Aug 2026 21:13:42 +0200 Subject: [PATCH 2/9] fix(backup): carry every mood column and both tag arms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The disaster-recovery payload dropped the free-text note, the row's timezone and the last-writer-wins counter, and the tag-link read filtered `rating: { not: null }` — which is the definition of a BINARY link, so every present/absent tag a person picked was missing from the file too. Both arms now mirror the measurement pair: the recovery arm carries the note ciphertext verbatim plus `tz`, `syncedAt`, `syncVersion`, `createdAt` and `updatedAt`; the portable arm carries the decrypted note, `tz` and no ciphertext. BINARY keys ride in their own `structuredTags` array rather than as a nullable rating inside `factors`, because a restore resolves `factors` against RATED tags and names what it cannot resolve — a rating-less entry there would collide with both. Absent means empty, so a file written before this parses and restores exactly as it did. The two link arms partition the links exhaustively: a malformed link comes back named as unresolvable instead of falling between them. --- .../mood-backup-completeness.test.ts | 49 ++++++-- .../__tests__/full-backup-payload.test.ts | 105 +++++++++++++++++- src/lib/export/full-backup-payload.ts | 79 +++++++++---- 3 files changed, 204 insertions(+), 29 deletions(-) diff --git a/src/__tests__/mood-backup-completeness.test.ts b/src/__tests__/mood-backup-completeness.test.ts index ca7eb3b0f..ef483325d 100644 --- a/src/__tests__/mood-backup-completeness.test.ts +++ b/src/__tests__/mood-backup-completeness.test.ts @@ -110,6 +110,31 @@ function selectFields(source: string, constant: string): string[] { ); } +/** + * The builder with its comment lines removed. + * + * Every matcher below runs against this rather than against the raw file, + * because prose describing the defect reads exactly like the defect: the first + * draft of this guard went red on its own explanation of the filter it exists + * to forbid. Whole comment lines only — a trailing comment after code is left + * alone, which is safe here because the only risk it carries is a matcher + * accepting a column named in prose, and the select comparison beside it is + * literal either way. + */ +function stripCommentLines(source: string): string { + return source + .split("\n") + .filter((line) => { + const trimmed = line.trim(); + return !( + trimmed.startsWith("//") || + trimmed.startsWith("/*") || + trimmed.startsWith("*") + ); + }) + .join("\n"); +} + /** * Columns either serialization arm actually reads off the mapped row. * @@ -138,7 +163,7 @@ function serialisedFields(source: string, variable: string): Set { describe("mood backup completeness", () => { const schema = readFileSync(SCHEMA, "utf8"); - const source = readFileSync(BACKUP_BUILDER, "utf8"); + const source = stripCommentLines(readFileSync(BACKUP_BUILDER, "utf8")); const models = declaredModels(schema); const entryColumns = scalarFields(modelBlock(schema, "MoodEntry"), models); @@ -225,16 +250,22 @@ describe("mood backup completeness", () => { }); it("both structured-tag arms are carried, not just the rated one", () => { - // The defect this file was written for: the tag-link read filtered - // `rating: { not: null }`, and a BINARY link carries a NULL rating by - // definition. Every present/absent tag — the kind most people pick — was - // excluded from the file and could not come back. + // The defect this file was written for: the tag-link read carried a + // filter, and a BINARY link's rating is NULL by definition, so every + // present/absent tag — the kind most people pick — was excluded from the + // file and could not come back from it. The claim is therefore about the + // read itself: it takes the links whole, unfiltered. // - // Asserted on a boolean rather than on the file contents so a failure - // reads as one line instead of printing the whole builder back. + // Asserted on booleans rather than on the file contents so a failure reads + // as one line instead of printing the whole builder back. + const tagLinkRead = /tagLinks:\s*\{([\s\S]*?)\}/.exec(source); expect( - source.includes("rating: { not: null }"), - "the tag-link read filters out NULL ratings again; BINARY links are being dropped from the backup", + tagLinkRead, + "no `tagLinks:` read found in the backup builder — this matcher proves nothing", + ).not.toBeNull(); + expect( + /where/.test(tagLinkRead![1]), + "the tag-link read filters again; a filter here is what dropped every BINARY link from the backup", ).toBe(false); expect( source.includes("structuredTags"), diff --git a/src/lib/export/__tests__/full-backup-payload.test.ts b/src/lib/export/__tests__/full-backup-payload.test.ts index 3632d9042..e55bf7b69 100644 --- a/src/lib/export/__tests__/full-backup-payload.test.ts +++ b/src/lib/export/__tests__/full-backup-payload.test.ts @@ -38,6 +38,8 @@ import { restoreProfileData } from "../profile-backup"; const deletedAt = new Date("2026-07-19T12:00:00.000Z"); const measurementNote = encryptToBytes("canonical measurement note"); const sideEffectNote = encryptToBytes("nausea after the evening dose"); +const moodNotePlaintext = "quiet evening, slept badly"; +const moodNote = encryptToBytes(moodNotePlaintext); const appSettings = { id: "singleton", @@ -193,11 +195,27 @@ function makePrisma() { mood: "OKAY", score: 3, tags: null, + note: null, + noteEncrypted: moodNote, source: "MOODLOG", externalId: "moodlog-42", moodLoggedAt: new Date("2026-07-19T08:00:00.000Z"), + tz: "America/New_York", + syncedAt: new Date("2026-07-19T08:00:01.000Z"), + syncVersion: 3, deletedAt, - tagLinks: [{ rating: 4, moodTag: { key: "sleep_quality" } }], + createdAt: new Date("2026-07-19T08:00:00.000Z"), + updatedAt: new Date("2026-07-19T09:00:00.000Z"), + tagLinks: [ + { + rating: 4, + moodTag: { key: "sleep_quality", kind: "RATED" }, + }, + { + rating: null, + moodTag: { key: "headache", kind: "BINARY" }, + }, + ], }, ]), }, @@ -491,7 +509,16 @@ describe("buildFullBackupPayload disaster-recovery mode", () => { id: "mood-tombstone", externalId: "moodlog-42", deletedAt: deletedAt.toISOString(), + tz: "America/New_York", + syncedAt: "2026-07-19T08:00:01.000Z", + syncVersion: 3, + createdAt: "2026-07-19T08:00:00.000Z", + updatedAt: "2026-07-19T09:00:00.000Z", + note: null, + noteEncrypted: Buffer.from(moodNote).toString("base64"), factors: [{ key: "sleep_quality", rating: 4 }], + // The BINARY link the old `rating: { not: null }` filter excluded. + structuredTags: ["headache"], }, ], nutrientDays: [ @@ -1154,3 +1181,79 @@ describe("buildFullBackupPayload — profile and custom metrics", () => { expect(counts.intradayProfiles).toBe(1); }); }); + +describe("buildFullBackupPayload — mood arms", () => { + it("carries the ciphertext and the whole row on the disaster-recovery arm", async () => { + installSectionMocks(); + const prisma = makePrisma(); + + const { payload } = await buildFullBackupPayload( + prisma as never, + "user-1", + { + purpose: "disaster-recovery", + exportedAt: new Date("2026-07-20T00:00:00.000Z"), + }, + ); + + const [entry] = (payload as { moodEntries: Array> }) + .moodEntries; + expect(entry).toEqual( + expect.objectContaining({ + id: "mood-tombstone", + note: null, + noteEncrypted: Buffer.from(moodNote).toString("base64"), + tz: "America/New_York", + syncedAt: "2026-07-19T08:00:01.000Z", + syncVersion: 3, + createdAt: "2026-07-19T08:00:00.000Z", + updatedAt: "2026-07-19T09:00:00.000Z", + deletedAt: deletedAt.toISOString(), + factors: [{ key: "sleep_quality", rating: 4 }], + structuredTags: ["headache"], + }), + ); + }); + + it("carries the readable note and no ciphertext on the portable arm", async () => { + installSectionMocks(); + const prisma = makePrisma(); + + const { payload } = await buildFullBackupPayload( + prisma as never, + "user-1", + { + purpose: "portable-export", + exportedAt: new Date("2026-07-20T00:00:00.000Z"), + }, + ); + + const [entry] = (payload as { moodEntries: Array> }) + .moodEntries; + // Same split the measurement arm makes: the readable artefact carries the + // decrypted text and nothing an operator could not read, and the counters + // that only mean something to this instance stay behind. + expect(entry).toEqual( + expect.objectContaining({ + note: moodNotePlaintext, + tz: "America/New_York", + structuredTags: ["headache"], + }), + ); + expect(entry).not.toHaveProperty("noteEncrypted"); + expect(entry).not.toHaveProperty("syncVersion"); + expect(entry).not.toHaveProperty("syncedAt"); + expect(JSON.stringify(entry)).not.toContain( + Buffer.from(moodNote).toString("base64"), + ); + // The tag-link read is unfiltered on both arms; a BINARY link is the one + // most people have and it was the one the file never carried. + expect(prisma.moodEntry.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + select: expect.objectContaining({ + tagLinks: { select: expect.any(Object) }, + }), + }), + ); + }); +}); diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index 08298bf65..9f4506de5 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -249,10 +249,11 @@ export async function buildFullBackupPayload( orderBy: { moodLoggedAt: "desc" }, select: { ...MOOD_ENTRY_BACKUP_SELECT, - tagLinks: { - where: { rating: { not: null } }, - select: MOOD_ENTRY_TAG_LINK_SELECT, - }, + // Every link, not only the rated ones. This read used to filter + // `rating: { not: null }`, and a BINARY link carries a NULL rating by + // definition — so the present/absent tags, the kind most people pick, + // were absent from the file and could not come back from it. + tagLinks: { select: MOOD_ENTRY_TAG_LINK_SELECT }, }, }), // The account's OWN mood tags. The seeded catalogue is reference data every @@ -437,21 +438,61 @@ export async function buildFullBackupPayload( // selected column reaches a payload by matching `: moodEntry.`. A matcher // on `: e.` would also match the intake-event mapper above it and could // therefore never fail. - moodEntries: moodEntries.map((moodEntry) => ({ - id: moodEntry.id, - date: moodEntry.date, - mood: moodEntry.mood, - score: moodEntry.score, - tags: moodEntry.tags, - source: moodEntry.source, - loggedAt: moodEntry.moodLoggedAt.toISOString(), - externalId: moodEntry.externalId, - deletedAt: moodEntry.deletedAt?.toISOString() ?? null, - factors: moodEntry.tagLinks.map((tagLink) => ({ - key: tagLink.moodTag.key, - rating: tagLink.rating!, - })), - })), + moodEntries: moodEntries.map((moodEntry) => { + // The two link arms partition the entry's links exhaustively: a link is + // a rated factor when its tag is RATED and it actually carries a score, + // and a structured tag otherwise. Total on purpose — a malformed link + // (a RATED tag whose rating went missing) rides as a tag key and comes + // back NAMED as unresolvable on restore, rather than falling between the + // two arms and disappearing the way every BINARY link used to. + const isRatedFactor = (link: (typeof moodEntry.tagLinks)[number]) => + link.moodTag.kind === "RATED" && link.rating !== null; + + const factors = moodEntry.tagLinks + .filter(isRatedFactor) + .map((tagLink) => ({ + key: tagLink.moodTag.key, + rating: tagLink.rating as number, + })); + // BINARY link keys. Their own array rather than a nullable rating inside + // `factors`: the restore resolves `factors` against `kind: "RATED"` and + // reports what it cannot resolve, so a rating-less entry in that array + // would collide with both. A separate key leaves every backup file + // written before this parsing and restoring exactly as it did. + const structuredTags = moodEntry.tagLinks + .filter((link) => !isRatedFactor(link)) + .map((tagLink) => tagLink.moodTag.key); + + return { + ...(disasterRecovery + ? { + note: moodEntry.note, + noteEncrypted: moodEntry.noteEncrypted + ? Buffer.from(moodEntry.noteEncrypted).toString("base64") + : null, + syncedAt: moodEntry.syncedAt.toISOString(), + syncVersion: moodEntry.syncVersion, + createdAt: moodEntry.createdAt.toISOString(), + updatedAt: moodEntry.updatedAt.toISOString(), + } + : { note: readNote(moodEntry.noteEncrypted, moodEntry.note) }), + id: moodEntry.id, + date: moodEntry.date, + mood: moodEntry.mood, + score: moodEntry.score, + tags: moodEntry.tags, + source: moodEntry.source, + loggedAt: moodEntry.moodLoggedAt.toISOString(), + externalId: moodEntry.externalId, + // The zone the `date` string is anchored to. Without it a restored row + // falls back to the legacy Europe/Berlin reading and an account that + // logged elsewhere gets its day boundaries quietly moved. + tz: moodEntry.tz, + deletedAt: moodEntry.deletedAt?.toISOString() ?? null, + factors, + structuredTags, + }; + }), customMoodTags: customMoodTags.map((tag) => ({ ...(disasterRecovery ? { id: tag.id } : {}), key: tag.key, From 18edd34c33dc1aac8127b77e60ec4475a1ddae7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 7 Aug 2026 21:37:06 +0200 Subject: [PATCH 3/9] fix(backup): restore the mood note, zone, counter and binary tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `moodEntrySchema` gains `note`, `noteEncrypted`, `tz`, `syncedAt`, `syncVersion`, `createdAt`, `updatedAt` and `structuredTags`; every one of them optional or defaulted, so a backup file written before this parses unchanged and restores exactly as it did. The restore writes them. The note follows the measurement pair: ciphertext verbatim when the file carries it, a portable file's plaintext re-encrypted on the way in, legacy column null either way. `tz` absent still means the legacy Europe/Berlin reading, which is what those rows meant. `syncVersion` comes back at the value the file recorded — restoring at the default handed the next round to any paired device holding a higher number, and the device would then overwrite the recovered entry with what it had. The link rebuild re-creates both arms: rated factors from `factors`, ticked tags from `structuredTags` with a NULL rating. An unresolvable tag key is reported under its own `moodTag` family, beside the rated one, because an operator needs to know which of the two an entry lost. --- messages/de.json | 1 + messages/en.json | 1 + messages/es.json | 1 + messages/fr.json | 1 + messages/it.json | 1 + messages/pl.json | 1 + .../api/admin/backups/[id]/restore/route.ts | 77 ++++++++++++++++++- src/components/admin/backups-section.tsx | 3 + src/lib/export/restore-skips.ts | 8 +- src/lib/validations/backup.ts | 18 +++++ 10 files changed, 107 insertions(+), 5 deletions(-) diff --git a/messages/de.json b/messages/de.json index 7468d4f41..455bad74f 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6701,6 +6701,7 @@ "restoreSkippedCycleSymptom": "Zyklussymptom", "restoreSkippedIllnessSymptom": "Krankheitssymptom", "restoreSkippedMoodFactor": "Stimmungsfaktor", + "restoreSkippedMoodTag": "Stimmungs-Tag", "docsLink": "Dokumentation" }, "danger-zone": { diff --git a/messages/en.json b/messages/en.json index 1b9ce1268..2041766c0 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6701,6 +6701,7 @@ "restoreSkippedCycleSymptom": "Cycle symptom", "restoreSkippedIllnessSymptom": "Illness symptom", "restoreSkippedMoodFactor": "Mood factor", + "restoreSkippedMoodTag": "Mood tag", "docsLink": "Documentation" }, "danger-zone": { diff --git a/messages/es.json b/messages/es.json index 35193f614..bafa72e6e 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6701,6 +6701,7 @@ "restoreSkippedCycleSymptom": "Síntoma del ciclo", "restoreSkippedIllnessSymptom": "Síntoma de enfermedad", "restoreSkippedMoodFactor": "Factor de ánimo", + "restoreSkippedMoodTag": "Etiqueta de ánimo", "docsLink": "Documentación" }, "danger-zone": { diff --git a/messages/fr.json b/messages/fr.json index b3df467d1..5324f563a 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6701,6 +6701,7 @@ "restoreSkippedCycleSymptom": "Symptôme de cycle", "restoreSkippedIllnessSymptom": "Symptôme de maladie", "restoreSkippedMoodFactor": "Facteur d'humeur", + "restoreSkippedMoodTag": "Étiquette d'humeur", "docsLink": "Documentation" }, "danger-zone": { diff --git a/messages/it.json b/messages/it.json index 7f96ce248..38af4fc16 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6701,6 +6701,7 @@ "restoreSkippedCycleSymptom": "Sintomo del ciclo", "restoreSkippedIllnessSymptom": "Sintomo di malattia", "restoreSkippedMoodFactor": "Fattore dell'umore", + "restoreSkippedMoodTag": "Etichetta dell'umore", "docsLink": "Documentazione" }, "danger-zone": { diff --git a/messages/pl.json b/messages/pl.json index 2da87ee8b..702ad81c5 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6701,6 +6701,7 @@ "restoreSkippedCycleSymptom": "Objaw cyklu", "restoreSkippedIllnessSymptom": "Objaw choroby", "restoreSkippedMoodFactor": "Czynnik nastroju", + "restoreSkippedMoodTag": "Tag nastroju", "docsLink": "Dokumentacja" }, "danger-zone": { diff --git a/src/app/api/admin/backups/[id]/restore/route.ts b/src/app/api/admin/backups/[id]/restore/route.ts index 3c22417f3..8ede74cac 100644 --- a/src/app/api/admin/backups/[id]/restore/route.ts +++ b/src/app/api/admin/backups/[id]/restore/route.ts @@ -684,6 +684,10 @@ const handler = apiHandler( entry.factors.map((factor) => factor.key), ); const factorKeys = [...new Set(referencedFactorKeys)]; + const referencedTagKeys = payload.moodEntries.flatMap( + (entry) => entry.structuredTags, + ); + const tagKeys = [...new Set(referencedTagKeys)]; // Re-create the account's own tag definitions first. The lookup // below used to ask only for the seeded catalogue (`userId: null`), // so a single custom rated tag made the whole restore throw @@ -740,6 +744,30 @@ const handler = apiHandler( referencedFactorKeys, ); + // The BINARY half of the same taxonomy, looked up the same way and + // reported the same way. It gets its own family because an + // operator reading the report needs to know whether a rated + // factor's score or a ticked tag went missing; they are different + // losses on the same entry. + const tagRows = + tagKeys.length === 0 + ? [] + : await tx.moodTag.findMany({ + where: { + key: { in: tagKeys }, + kind: "BINARY", + OR: [{ userId: null }, { userId: ownerId }], + }, + select: { id: true, key: true }, + }); + const tagByKey = new Map(tagRows.map((tag) => [tag.key, tag.id])); + recordUnknownKeys( + skips, + "moodTag", + tagKeys.filter((key) => !tagByKey.has(key)), + referencedTagKeys, + ); + for (const entry of payload.moodEntries) { const moodLoggedAt = new Date(entry.loggedAt); const restoredData = { @@ -747,10 +775,44 @@ const handler = apiHandler( mood: entry.mood, score: entry.score, tags: entry.tags ?? null, + // The note comes back the way every other dual-column note + // does: ciphertext verbatim when the file carries it, and a + // portable file's plaintext re-encrypted on the way in, with + // the legacy column left null either way. + note: null, + noteEncrypted: + entry.noteEncrypted == null + ? encryptNote(entry.note ?? null) + : decodeEncryptedBytes(entry.noteEncrypted), source: entry.source ?? "MOODLOG", externalId: entry.externalId ?? null, moodLoggedAt, + // The zone the `date` string is anchored to. A file written + // before this rode the payload carries nothing, and NULL is + // what such a row meant anyway: the legacy Europe/Berlin + // reading. So an old file keeps its old day boundaries and a + // new one keeps the zone the person actually logged in. + tz: entry.tz ?? null, + // Mood reconciles last-writer-wins by `syncVersion`. Restoring + // at the schema default would hand the next sync round to any + // paired device still holding a higher number, and the device + // would overwrite the recovered row with what it had — a + // restore that loses to the client is worse than one that + // fails. An old file carries nothing here and keeps the + // default, which is the value those rows were written with. + ...(entry.syncVersion === undefined + ? {} + : { syncVersion: entry.syncVersion }), deletedAt: entry.deletedAt ? new Date(entry.deletedAt) : null, + ...(entry.syncedAt + ? { syncedAt: new Date(entry.syncedAt) } + : {}), + ...(entry.createdAt + ? { createdAt: new Date(entry.createdAt) } + : {}), + ...(entry.updatedAt + ? { updatedAt: new Date(entry.updatedAt) } + : {}), }; const createData = { userId: ownerId, ...restoredData }; const restored = entry.id @@ -798,8 +860,19 @@ const handler = apiHandler( ] : []; }); - if (factorLinks.length > 0) { - await tx.moodEntryTagLink.createMany({ data: factorLinks }); + // The other arm. A BINARY link carries no score — the presence + // of the row is the signal — so `rating` is NULL by definition, + // which is exactly what the backup's old tag-link filter took to + // mean "nothing here". + const tagLinks = entry.structuredTags.flatMap((key) => { + const moodTagId = tagByKey.get(key); + return moodTagId + ? [{ moodEntryId: restored.id, moodTagId, rating: null }] + : []; + }); + const links = [...factorLinks, ...tagLinks]; + if (links.length > 0) { + await tx.moodEntryTagLink.createMany({ data: links }); } } } diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index 2df37779f..76c3acede 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -194,6 +194,9 @@ function catalogueLabel( if (catalogue === "illnessSymptom") { return t("admin.section.backups.restoreSkippedIllnessSymptom"); } + if (catalogue === "moodTag") { + return t("admin.section.backups.restoreSkippedMoodTag"); + } return t("admin.section.backups.restoreSkippedMoodFactor"); } diff --git a/src/lib/export/restore-skips.ts b/src/lib/export/restore-skips.ts index e57756edd..326ea093b 100644 --- a/src/lib/export/restore-skips.ts +++ b/src/lib/export/restore-skips.ts @@ -1,8 +1,9 @@ /** * What a restore could not put back, and how much of it there was. * - * Three of the restore's lookups resolve a key against a SEEDED CATALOGUE — - * cycle symptoms, illness symptoms, mood factors. The catalogue is reference + * Four of the restore's lookups resolve a key against a SEEDED CATALOGUE — + * cycle symptoms, illness symptoms, rated mood factors and the mood tags a + * person ticks present or absent. The catalogue is reference * data the instance owns, not data the file carries, so it legitimately drifts * between the day a backup is written and the day it is read: a key gets * renamed, a symptom is retired, the file is a year old, the instance is three @@ -28,7 +29,8 @@ */ /** Which seeded catalogue a key failed to resolve against. */ -export type SkippedCatalogue = "cycleSymptom" | "illnessSymptom" | "moodFactor"; +export type SkippedCatalogue = + "cycleSymptom" | "illnessSymptom" | "moodFactor" | "moodTag"; /** One key this instance does not know, and the links it cost. */ export interface SkippedCatalogueKey { diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index b4afaf650..3f1a981e1 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -255,11 +255,29 @@ const moodEntrySchema = z mood: z.string().min(1), score: z.number().int().min(0).max(10), tags: moodEntryTagsSchema, + // The free text, in both storage shapes. A portable export carries the + // decrypted `note`; a disaster-recovery file carries `noteEncrypted` plus + // whatever legacy plaintext the row still holds. Same pair, and same + // handling on restore, as `Measurement.notes` / `notesEncrypted`. + note: z.string().nullable().optional(), + noteEncrypted: base64BytesSchema.nullable().optional(), source: z.string().min(1).optional(), externalId: z.string().nullable().optional(), loggedAt: isoDateTime, + // The IANA zone the `date` string is anchored to. Absent means the legacy + // Europe/Berlin reading, which is what a row written before v1.4.25 means + // by a NULL here — so an old file keeps its old meaning exactly. + tz: z.string().nullable().optional(), + syncedAt: isoDateTime.optional(), + syncVersion: z.number().int().optional(), deletedAt: isoDateTime.nullable().optional(), + createdAt: isoDateTime.optional(), + updatedAt: isoDateTime.optional(), factors: z.array(moodFactorSchema).default([]), + // BINARY structured-tag keys. Separate from `factors`, which keeps its + // RATED-only meaning, so every file written before this one parses + // unchanged and restores identically: absent reads as none. + structuredTags: z.array(z.string().min(1)).default([]), }) .passthrough(); From 9388f1d032671dc4a06adcefff87fd37536c88ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 7 Aug 2026 21:44:01 +0200 Subject: [PATCH 4/9] test(backup): round-trip a mood entry through the real restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seeds an entry with a ticked tag, a rated factor, an encrypted note, a non-Berlin timezone and a sync counter above zero, exports through the real builder, restores through the real `POST /api/admin/backups/[id]/restore`, and reads the row back out of Postgres. Values, not a payload shape: this repository has already proved that asserting a `moodEntries` key stays green with the restore call excised. Two more cases beside it. A hand-written payload in the shape files were written in before the note, zone, counter and tag keys rode along — the compatibility floor, and the only way to know it holds is to send one. And a ticked tag this instance cannot resolve, which has to come back named in the skip report rather than silently absent. Seen red three times on the way in: re-adding the tag-link filter loses the ticked tags, dropping `tz` from the restore write loses the zone, dropping the note ciphertext from the payload loses the text. Each reverted and the file's hash checked back to its recorded value. --- .../mood-backup-round-trip.test.ts | 439 ++++++++++++++++++ 1 file changed, 439 insertions(+) create mode 100644 tests/integration/mood-backup-round-trip.test.ts diff --git a/tests/integration/mood-backup-round-trip.test.ts b/tests/integration/mood-backup-round-trip.test.ts new file mode 100644 index 000000000..42c43e088 --- /dev/null +++ b/tests/integration/mood-backup-round-trip.test.ts @@ -0,0 +1,439 @@ +/** + * A mood entry, out through the real backup and back in through the real + * restore, asserted on the rows. + * + * Deliberately not a classification test. Asserting that the payload has a + * `moodEntries` key proves nothing this repository has not already been burned + * by: excise the restore call entirely and a shape check still reports green. + * So this seeds an account, exports through `buildFullBackupPayload`, parses + * through `parseBackupPayload`, restores through the real + * `POST /api/admin/backups/[id]/restore`, and then reads the row back out of + * Postgres and compares values. + * + * The fixture is the entry that used to come back wrong in four ways at once: + * a ticked BINARY tag and a rated factor on the same entry, a note in the + * encrypted column, a timezone that is not the legacy Europe/Berlin default, + * and a sync counter above zero. The old payload carried none of them. + * + * The second case is the compatibility floor: a hand-written payload in the + * shape files were written in BEFORE this change, restored through the same + * route. Every field this adds is optional or defaulted precisely so that file + * keeps working, and the only way to know is to send one. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import type { PrismaClient } from "@/generated/prisma/client"; +import { encrypt } from "@/lib/crypto"; +import { decryptFromBytes } from "@/lib/ai/coach/bytes-codec"; +import { encryptNote } from "@/lib/crypto/note-cipher"; +import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { parseBackupPayload } from "@/lib/validations/backup"; +import { POST } from "@/app/api/admin/backups/[id]/restore/route"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserData: vi.fn(), +})); + +const OWNER_ID = "mood-round-trip-owner"; +const NOTE = "slept badly, long walk in the afternoon"; +const LOGGED_AT = new Date("2026-07-19T23:40:00.000Z"); +const CREATED_AT = new Date("2026-07-19T23:41:00.000Z"); +const UPDATED_AT = new Date("2026-07-20T06:05:00.000Z"); +const DELETED_AT = new Date("2026-07-20T07:00:00.000Z"); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +async function seedAdminSession(prisma: PrismaClient) { + const admin = await prisma.user.create({ + data: { + username: "mood-round-trip-admin", + email: "mood-round-trip-admin@example.test", + role: "ADMIN", + }, + }); + const session = await prisma.session.create({ + data: { userId: admin.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return admin; +} + +async function seedOwner(prisma: PrismaClient) { + await prisma.user.create({ + data: { + id: OWNER_ID, + username: "mood-round-trip-owner", + email: "mood-round-trip-owner@example.test", + }, + }); +} + +/** + * The taxonomy the entry links to: one seeded BINARY tag (`userId: null`, the + * reference data every instance has), one BINARY tag the account made itself, + * and one RATED factor. The account's own tag matters — it travels in the file + * as a `customMoodTags` row, and a restore that could not re-create it would + * lose the link even with everything else correct. + */ +async function seedTaxonomy(prisma: PrismaClient) { + const category = await prisma.moodTagCategory.create({ + data: { + id: "mood-rt-category", + key: "mood_round_trip", + labelKey: "mood.tagCategory.roundTrip", + }, + }); + const seededTag = await prisma.moodTag.create({ + data: { + id: "mood-rt-seeded-tag", + categoryId: category.id, + key: "rt_headache", + labelKey: "mood.tag.rtHeadache", + kind: "BINARY", + }, + }); + const ownTag = await prisma.moodTag.create({ + data: { + id: "mood-rt-own-tag", + userId: OWNER_ID, + categoryId: category.id, + key: "custom:rt_long_walk", + labelKey: "mood.tag.custom", + kind: "BINARY", + }, + }); + const factor = await prisma.moodTag.create({ + data: { + id: "mood-rt-factor", + categoryId: category.id, + key: "rt_sleep_quality", + labelKey: "mood.tag.rtSleepQuality", + kind: "RATED", + scaleMin: 1, + scaleMax: 5, + }, + }); + return { category, seededTag, ownTag, factor }; +} + +function makeRestoreRequest(id: string) { + return new Request(`http://localhost/api/admin/backups/${id}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }); +} + +async function restoreFromPayload( + prisma: PrismaClient, + payload: unknown, + type: string, +) { + const backup = await prisma.dataBackup.create({ + data: { + userId: OWNER_ID, + type, + data: encrypt(JSON.stringify(payload)), + }, + }); + return POST(makeRestoreRequest(backup.id) as never, { + params: Promise.resolve({ id: backup.id }), + }); +} + +describe("mood backup round trip", () => { + it("brings the note, the timezone, the sync counter and both tag arms back", async () => { + const prisma = getPrismaClient(); + await seedAdminSession(prisma); + await seedOwner(prisma); + const { seededTag, ownTag, factor } = await seedTaxonomy(prisma); + + await prisma.moodEntry.create({ + data: { + id: "mood-rt-entry", + userId: OWNER_ID, + date: "2026-07-19", + mood: "OKAY", + score: 3, + tags: JSON.stringify(["free-text-tag"]), + note: null, + noteEncrypted: encryptNote(NOTE), + source: "MANUAL", + externalId: null, + moodLoggedAt: LOGGED_AT, + // Not Europe/Berlin: a restore that drops `tz` moves this entry's day + // boundary by six hours and nothing else in the row shows it. + tz: "America/New_York", + syncVersion: 3, + createdAt: CREATED_AT, + updatedAt: UPDATED_AT, + tagLinks: { + create: [ + { moodTagId: factor.id, rating: 4 }, + { moodTagId: seededTag.id, rating: null }, + { moodTagId: ownTag.id, rating: null }, + ], + }, + }, + }); + await prisma.moodEntry.create({ + data: { + id: "mood-rt-tombstone", + userId: OWNER_ID, + date: "2026-07-18", + mood: "SCHLECHT", + score: 2, + source: "MANUAL", + moodLoggedAt: new Date("2026-07-18T22:00:00.000Z"), + tz: "America/New_York", + syncVersion: 5, + deletedAt: DELETED_AT, + }, + }); + + const { payload } = await buildFullBackupPayload(prisma, OWNER_ID, { + purpose: "disaster-recovery", + }); + const parsed = parseBackupPayload(payload); + + // Wipe what the restore is supposed to bring back. The tag rows stay: the + // instance's catalogue is reference data, and the account's own tag rides + // in the file and is re-created by the restore either way. + await prisma.moodEntry.deleteMany({ where: { userId: OWNER_ID } }); + await prisma.moodTag.deleteMany({ where: { userId: OWNER_ID } }); + expect(await prisma.moodEntry.count({ where: { userId: OWNER_ID } })).toBe( + 0, + ); + + const response = await restoreFromPayload( + prisma, + parsed, + "MOOD_ROUND_TRIP", + ); + expect(response.status).toBe(200); + const body = (await response.json()) as { + data: { skipped: { links: number } }; + }; + // Nothing was dropped. A restore that lost a link and still answered 200 + // is exactly what the skip report exists to make visible. + expect(body.data.skipped.links).toBe(0); + + const restored = await prisma.moodEntry.findUniqueOrThrow({ + where: { id: "mood-rt-entry" }, + }); + // Asserted as "there is a note" before it is decrypted, so a payload that + // stopped carrying the ciphertext fails by name rather than as a TypeError + // three frames inside the cipher. + expect( + restored.noteEncrypted, + "the restored entry has no note ciphertext — the backup dropped it", + ).not.toBeNull(); + expect(decryptFromBytes(restored.noteEncrypted!)).toBe(NOTE); + expect(restored.note).toBeNull(); + expect(restored.tz).toBe("America/New_York"); + expect(restored.syncVersion).toBe(3); + expect(restored.moodLoggedAt).toEqual(LOGGED_AT); + expect(restored.createdAt).toEqual(CREATED_AT); + expect(restored.updatedAt).toEqual(UPDATED_AT); + expect(restored.date).toBe("2026-07-19"); + expect(restored.score).toBe(3); + expect(restored.tags).toBe(JSON.stringify(["free-text-tag"])); + expect(restored.deletedAt).toBeNull(); + + const tombstone = await prisma.moodEntry.findUniqueOrThrow({ + where: { id: "mood-rt-tombstone" }, + }); + expect(tombstone.deletedAt).toEqual(DELETED_AT); + expect(tombstone.syncVersion).toBe(5); + + const links = await prisma.moodEntryTagLink.findMany({ + where: { moodEntryId: "mood-rt-entry" }, + select: { rating: true, moodTag: { select: { key: true, kind: true } } }, + orderBy: { moodTagId: "asc" }, + }); + const byKey = new Map(links.map((l) => [l.moodTag.key, l])); + expect([...byKey.keys()].sort()).toEqual([ + "custom:rt_long_walk", + "rt_headache", + "rt_sleep_quality", + ]); + // The rating is what tells the two arms apart in the row: a ticked tag + // carries NULL, and that NULL is what the old backup filter read as + // "nothing to carry". + expect(byKey.get("rt_headache")!.rating).toBeNull(); + expect(byKey.get("custom:rt_long_walk")!.rating).toBeNull(); + expect(byKey.get("rt_sleep_quality")!.rating).toBe(4); + }); + + it("restores a file written in the shape that predates these fields", async () => { + const prisma = getPrismaClient(); + await seedAdminSession(prisma); + await seedOwner(prisma); + const { factor } = await seedTaxonomy(prisma); + + // Verbatim the shape the builder emitted before this change: no `note`, no + // `noteEncrypted`, no `tz`, no `syncedAt`, no `syncVersion`, no + // `createdAt`/`updatedAt`, no `structuredTags`. Hand-written rather than + // generated, because a fixture built from today's builder cannot describe + // yesterday's file. + const legacyPayload = { + schemaVersion: "2", + exportedAt: "2026-07-20T00:00:00.000Z", + userId: OWNER_ID, + appSettings: null, + measurements: [], + medications: [], + intakeEvents: [], + moodEntries: [ + { + id: "mood-rt-legacy", + date: "2026-07-19", + mood: "GUT", + score: 4, + tags: null, + source: "MOODLOG", + loggedAt: LOGGED_AT.toISOString(), + externalId: "moodlog-legacy", + deletedAt: null, + factors: [{ key: factor.key, rating: 5 }], + }, + ], + customMoodTags: [], + nutrientDays: [], + }; + + const parsed = parseBackupPayload(legacyPayload); + // Defaults, not values: absence has to stay absence through the parse. + expect(parsed.moodEntries[0].structuredTags).toEqual([]); + expect(parsed.moodEntries[0].tz).toBeUndefined(); + expect(parsed.moodEntries[0].syncVersion).toBeUndefined(); + + const response = await restoreFromPayload( + prisma, + parsed, + "MOOD_LEGACY_SHAPE", + ); + expect(response.status).toBe(200); + + const restored = await prisma.moodEntry.findUniqueOrThrow({ + where: { id: "mood-rt-legacy" }, + }); + // Identical to what the old restore wrote for the same file: the note + // columns empty, `tz` NULL — which is what such a row meant, the legacy + // Europe/Berlin reading — and the counter at the schema default. + expect(restored.note).toBeNull(); + expect(restored.noteEncrypted).toBeNull(); + expect(restored.tz).toBeNull(); + expect(restored.syncVersion).toBe(0); + expect(restored.mood).toBe("GUT"); + expect(restored.score).toBe(4); + expect(restored.externalId).toBe("moodlog-legacy"); + expect(restored.deletedAt).toBeNull(); + + const links = await prisma.moodEntryTagLink.findMany({ + where: { moodEntryId: "mood-rt-legacy" }, + select: { rating: true, moodTagId: true }, + }); + expect(links).toEqual([{ moodTagId: factor.id, rating: 5 }]); + }); + + it("names a ticked tag this instance cannot resolve instead of losing it quietly", async () => { + const prisma = getPrismaClient(); + await seedAdminSession(prisma); + await seedOwner(prisma); + await seedTaxonomy(prisma); + + const payload = { + schemaVersion: "2", + exportedAt: "2026-07-20T00:00:00.000Z", + userId: OWNER_ID, + appSettings: null, + measurements: [], + medications: [], + intakeEvents: [], + moodEntries: [ + { + id: "mood-rt-unknown-tag", + date: "2026-07-19", + mood: "OKAY", + score: 3, + tags: null, + source: "MANUAL", + loggedAt: LOGGED_AT.toISOString(), + externalId: null, + deletedAt: null, + factors: [], + structuredTags: ["rt_headache", "rt_retired_tag"], + }, + ], + customMoodTags: [], + nutrientDays: [], + }; + + const response = await restoreFromPayload( + prisma, + parseBackupPayload(payload), + "MOOD_UNKNOWN_TAG", + ); + expect(response.status).toBe(200); + const body = (await response.json()) as { + data: { + skipped: { + links: number; + catalogueKeys: Array<{ + catalogue: string; + key: string; + links: number; + }>; + }; + }; + }; + expect(body.data.skipped.catalogueKeys).toEqual([ + { catalogue: "moodTag", key: "rt_retired_tag", links: 1 }, + ]); + expect(body.data.skipped.links).toBe(1); + + // The entry itself came back whole. One key that has nowhere to attach + // costs the link, never the record. + const restored = await prisma.moodEntry.findUniqueOrThrow({ + where: { id: "mood-rt-unknown-tag" }, + }); + expect(restored.score).toBe(3); + const links = await prisma.moodEntryTagLink.findMany({ + where: { moodEntryId: restored.id }, + select: { moodTag: { select: { key: true } } }, + }); + expect(links).toEqual([{ moodTag: { key: "rt_headache" } }]); + }); +}); From 8c5ed74bc0ebdf927bedebeac4968370af4a6369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 7 Aug 2026 21:51:30 +0200 Subject: [PATCH 5/9] docs(changelog): note the mood restore fix --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f94ce3040..540c25071 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,19 @@ reliably the one that ran. It is selected by name now, and a copy that never arrived fails the build instead of quietly leaving another one in its place. +### Fixed + +- Mood entries now survive a restore whole. The disaster-recovery backup was + carrying only the rated factors of an entry, so every tag ticked present or + absent was missing from the file, and it was dropping the note, the entry's + own timezone and the sync counter as well. A restored entry came back without + its text, read its day boundaries under the old Berlin assumption whatever + zone it had been logged in, and could lose the next sync round to a paired + phone still holding a higher counter. The backup carries every column now, + both halves of the tag taxonomy come back, and a structural check fails the + build if a future column goes missing the same way. Backup files written + before this restore exactly as they did. + ## [1.37.0] — 2026-08-07 ### Added From 2941d56238bce1cf97c6df9cc31c5a9b06cd0957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 7 Aug 2026 22:19:43 +0200 Subject: [PATCH 6/9] fix(sync): enforce the plausibility domain on every provider write path Every interactive path, every import, MCP and Telegram checked an incoming measurement against VALUE_RANGES before storing it. The provider sync writers did not, so a decode slip or a glitched sensor landed a row the application itself calls impossible, and every mean, median and personal band derived from it inherited the impossibility. Filtering the read side hid the row. It kept landing. The gate is one module reading the domain the app already declares, applied at two kinds of seam. reconcileExternalMeasurement is the shared identity writer for Polar, Oura, WHOOP, the Withings sleep leg, the Apple export importer and the batch endpoint, so gating there means a provider added later inherits the refusal rather than having to remember a check it never knew about; the new rejected_range verdict carries no row, which is what makes the compiler force every caller to handle it. The five writers that hold Prisma themselves gate at their entry, before the existence probe decides insert versus overwrite, so an impossible reading can no longer replace a good stored value. An out-of-band value is dropped. Not clamped, because a clamp invents a reading nobody took. Not stored behind a flag, because a quarantined row is still a row and every consumer would have to learn the flag. Not fatal to the batch, because one bad sample must not strand the run's other readings behind a held watermark. Absence reads as absence. A silent drop, though, is indistinguishable from a provider that sent nothing, so every drop is tallied on the wide event under measurement.range_rejected with the metric, the provider and the edge it broke. Counts only: the reading itself is health data and never enters the log line, and the direction plus the metric's own declared bound is what a diagnosis actually needs. The tally accumulates rather than replacing, because a provider gates once per resource leg and the reconciler once per row. One warning per provider lifts the event to warn so a drop does not hide inside a green line. A structural guard enumerates all twenty measurement writers with the reason each is allowed to write, so a new one has to answer the question before it ships. scripts/diagnose-implausible-measurements.ts reports the rows already stored, aggregated per metric and source with no value beyond the bucket's own extremes. It repairs nothing: whether a stored impossible row is a decode slip or a band that is too narrow is not a call a script gets to make. --- scripts/diagnose-implausible-measurements.ts | 150 ++++++++++ ...asurement-write-plausibility-guard.test.ts | 200 ++++++++++++++ src/app/api/measurements/batch/route.ts | 11 + .../__tests__/sync-plausibility.test.ts | 179 ++++++++++++ src/lib/fitbit/sync-core.ts | 12 +- .../__tests__/sync-plausibility.test.ts | 180 ++++++++++++ src/lib/google-health/sync-core.ts | 15 +- .../__tests__/plausibility-gate.test.ts | 222 +++++++++++++++ .../reconcile-external-measurement.test.ts | 104 +++++++ .../import-apple-health-export.ts | 10 + src/lib/measurements/plausibility-gate.ts | 192 +++++++++++++ .../reconcile-external-measurement.ts | 31 +++ .../__tests__/sync-plausibility.test.ts | 115 ++++++++ src/lib/nightscout/sync.ts | 12 +- src/lib/oura/sync.ts | 4 + src/lib/polar/sync.ts | 4 + src/lib/whoop/sync-core.ts | 4 + .../__tests__/sync-plausibility.test.ts | 261 ++++++++++++++++++ src/lib/withings/sync-activity.ts | 14 +- src/lib/withings/sync-sleep.ts | 6 + src/lib/withings/sync.ts | 15 +- 21 files changed, 1733 insertions(+), 8 deletions(-) create mode 100644 scripts/diagnose-implausible-measurements.ts create mode 100644 src/__tests__/measurement-write-plausibility-guard.test.ts create mode 100644 src/lib/fitbit/__tests__/sync-plausibility.test.ts create mode 100644 src/lib/google-health/__tests__/sync-plausibility.test.ts create mode 100644 src/lib/measurements/__tests__/plausibility-gate.test.ts create mode 100644 src/lib/measurements/plausibility-gate.ts create mode 100644 src/lib/nightscout/__tests__/sync-plausibility.test.ts create mode 100644 src/lib/withings/__tests__/sync-plausibility.test.ts diff --git a/scripts/diagnose-implausible-measurements.ts b/scripts/diagnose-implausible-measurements.ts new file mode 100644 index 000000000..0be518a26 --- /dev/null +++ b/scripts/diagnose-implausible-measurements.ts @@ -0,0 +1,150 @@ +/** + * Read-only diagnostic for stored measurement rows outside the app's own + * declared plausibility domain. + * + * `VALUE_RANGES` in `@/lib/validations/measurement` is the band the + * application says a metric can occupy. Every path a person can drive has + * always enforced it; the provider sync writers did not, so rows the + * application itself calls impossible reached the table — and every mean, + * median and personal band built from them inherited the impossibility. The + * writers enforce it now, which stops new rows landing. It says nothing about + * the rows already there. + * + * This reports them and repairs nothing. What to do with a stored impossible + * row is a judgement the data cannot make: a pulse of 55 million is plainly a + * decode slip and a candidate for deletion, while a weight of 0.9 kg might be + * a neonatal record on a shared scale, or a unit mix-up, or a real reading the + * band is simply too narrow for. Deleting somebody's health record on a + * heuristic is not a thing a script gets to decide. + * + * Output is aggregate: one line per (metric, source) bucket with the row + * count, the number of distinct records affected, the observed extremes, and + * the window they fall in. Individual readings are not printed — the bucket + * plus the metric's own declared bound is what a decision needs, and a log + * file full of somebody's vitals is not. + * + * Run (inside the app container, which has DATABASE_URL): + * pnpm dlx tsx scripts/diagnose-implausible-measurements.ts + */ +import "dotenv/config"; + +import { prisma } from "@/lib/db"; +import { VALUE_RANGES } from "@/lib/validations/measurement"; + +interface BucketRow { + type: string; + source: string; + row_count: bigint; + record_count: bigint; + min_value: number; + max_value: number; + first_at: Date; + last_at: Date; + tombstoned: bigint; +} + +function ymd(date: Date): string { + return date.toISOString().slice(0, 10); +} + +async function main(): Promise { + const types = Object.keys(VALUE_RANGES).sort(); + const mins = types.map((t) => VALUE_RANGES[t]!.min); + const maxes = types.map((t) => VALUE_RANGES[t]!.max); + + // The band table rides in as three parallel bound arrays rather than a + // spliced VALUES list, so the query carries no interpolated text at all. + // `value <> value` is the NaN probe: a double-precision column can hold one, + // and it compares false against every bound including itself. + const buckets = await prisma.$queryRaw` + WITH bands(type, min_value, max_value) AS ( + SELECT * FROM unnest( + ${types}::text[], + ${mins}::double precision[], + ${maxes}::double precision[] + ) + ) + SELECT + m."type"::text AS type, + m."source"::text AS source, + COUNT(*)::bigint AS row_count, + COUNT(DISTINCT m."user_id")::bigint AS record_count, + MIN(m."value") AS min_value, + MAX(m."value") AS max_value, + MIN(m."measured_at") AS first_at, + MAX(m."measured_at") AS last_at, + COUNT(*) FILTER (WHERE m."deleted_at" IS NOT NULL)::bigint AS tombstoned + FROM "measurements" m + JOIN bands b ON b."type" = m."type"::text + WHERE m."value" < b.min_value + OR m."value" > b.max_value + OR m."value" <> m."value" + GROUP BY m."type", m."source" + ORDER BY COUNT(*) DESC, m."type"::text ASC, m."source"::text ASC + `; + + if (buckets.length === 0) { + console.log( + `Every stored measurement sits inside its metric's declared range ` + + `(${types.length} metric bands checked). Nothing to review.`, + ); + return; + } + + const totalRows = buckets.reduce((sum, b) => sum + Number(b.row_count), 0); + const affectedTypes = new Set(buckets.map((b) => b.type)); + const affectedSources = new Set(buckets.map((b) => b.source)); + + console.log( + `${totalRows} stored row(s) sit outside the declared plausibility band, ` + + `across ${affectedTypes.size} metric(s) and ${affectedSources.size} source(s).\n` + + `Nothing below is modified. Each line is a (metric, source) bucket; the ` + + `band is the metric's own declared domain.\n`, + ); + + // Group by metric so the declared band is stated once per heading, and rank + // the metrics by how much of the problem each one is. The query's own + // ordering already ranks the sources inside a metric. + const byType = new Map(); + for (const bucket of buckets) { + const bucketsForType = byType.get(bucket.type); + if (bucketsForType) bucketsForType.push(bucket); + else byType.set(bucket.type, [bucket]); + } + const ranked = [...byType.entries()].sort( + (a, b) => + b[1].reduce((sum, x) => sum + Number(x.row_count), 0) - + a[1].reduce((sum, x) => sum + Number(x.row_count), 0), + ); + + for (const [type, rows] of ranked) { + const band = VALUE_RANGES[type]!; + console.log(`${type} — declared band ${band.min}…${band.max}`); + for (const bucket of rows) { + const live = Number(bucket.row_count) - Number(bucket.tombstoned); + console.log( + ` ${bucket.source.padEnd(14)} ` + + `${String(bucket.row_count).padStart(7)} row(s) ` + + `(${live} live, ${bucket.tombstoned} tombstoned) ` + + `across ${bucket.record_count} record(s) — ` + + `observed ${bucket.min_value}…${bucket.max_value}, ` + + `${ymd(bucket.first_at)} → ${ymd(bucket.last_at)}`, + ); + } + } + + console.log( + `\nThe sync writers refuse these values now, so the population above is ` + + `closed unless a writer without the gate is still in play. Removing a ` + + `row is a maintainer decision, per bucket.`, + ); +} + +main() + .catch((err) => { + console.error(err); + process.exitCode = 1; + }) + .finally(() => { + void prisma.$disconnect(); + }); diff --git a/src/__tests__/measurement-write-plausibility-guard.test.ts b/src/__tests__/measurement-write-plausibility-guard.test.ts new file mode 100644 index 000000000..88fbafee2 --- /dev/null +++ b/src/__tests__/measurement-write-plausibility-guard.test.ts @@ -0,0 +1,200 @@ +/** + * Structural guard on the measurement ingest boundary. + * + * `VALUE_RANGES` is the band the application declares a metric can occupy. + * Every path a person can drive enforced it from the start; the provider sync + * writers never did, and nothing noticed, because each writer is correct on + * its own terms — it wrote what the remote sent. The gap was only visible from + * above, by listing every writer at once and asking which of them had ever + * been told about the band. + * + * So the list lives here. Every file that writes a `Measurement` row is + * enumerated with the reason it is allowed to, and the guard fails when the + * discovered set stops matching. A provider added tomorrow lands a file that + * is not in the map and has to answer the question before it can ship. + * + * This is a tripwire, not a proof. It cannot show a disposition is the right + * one — only that nobody added a writer without writing down which gate covers + * it. A reviewer who waves through a bogus `derived` label defeats it, and no + * test substitutes for that review. + */ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +/** The gate module every provider-fed writer has to reach. */ +const GATE_MODULE = "@/lib/measurements/plausibility-gate"; +/** The predicate every person-driven write path has always used. */ +const VALIDATOR = "validateMeasurementRange"; + +/** + * How a writer satisfies the plausibility domain. + * + * - `gate` — provider-fed; must reach {@link GATE_MODULE} itself. + * - `validator` — person-driven; must call {@link VALIDATOR} itself. + * - `upstream` — validated in a named module before the row is built; the + * named module is checked, so the label cannot be a shrug. + * - `derived` — the value is arithmetic over rows already in the table, not a + * number from outside. Gating here would drop a consolidation of data that + * IS present, which is a different failure from admitting one that is not. + * - `literal` — the written value is a constant in the file, so no external + * number can reach the column. + * - `restore` — replays the record's own backup verbatim. Fidelity is the + * contract; a restore that silently dropped rows would be the worse bug, + * and the diagnostic script is how a stored impossible row gets found. + */ +type Disposition = + | { kind: "gate" } + | { kind: "validator" } + | { kind: "upstream"; validatedIn: string } + | { kind: "derived"; why: string } + | { kind: "literal"; why: string } + | { kind: "restore"; why: string }; + +const WRITERS: Record = { + // ── Provider sync writers, gated ────────────────────────────────────── + "lib/withings/sync.ts": { kind: "gate" }, + "lib/withings/sync-activity.ts": { kind: "gate" }, + "lib/fitbit/sync-core.ts": { kind: "gate" }, + "lib/google-health/sync-core.ts": { kind: "gate" }, + "lib/nightscout/sync.ts": { kind: "gate" }, + // The shared identity seam. Polar, Oura, WHOOP, the Withings sleep leg, + // the Apple export importer and the batch endpoint all write through it, + // so the gate here is what makes a new provider fail closed. + "lib/measurements/reconcile-external-measurement.ts": { kind: "gate" }, + + // ── Person-driven write paths, validated ────────────────────────────── + "app/api/measurements/route.ts": { + kind: "upstream", + validatedIn: "lib/validations/measurement.ts", + }, + "app/api/import/route.ts": { kind: "validator" }, + "lib/mcp/writes.ts": { kind: "validator" }, + "lib/measurements/create-from-telegram.ts": { kind: "validator" }, + "lib/measurements/import-apple-health-export.ts": { kind: "validator" }, + "app/api/import/csv/route.ts": { + kind: "upstream", + validatedIn: "lib/import/csv-measurements.ts", + }, + + // ── Fixed values ────────────────────────────────────────────────────── + "lib/withings/sync-ecg.ts": { + kind: "literal", + why: "an ECG row carries value 1 — the file's own constant, never a provider number", + }, + "app/api/mental-health/assessments/route.ts": { + kind: "literal", + why: "the questionnaire score is summed from Zod-bounded item answers", + }, + "lib/insights/score-row.ts": { + kind: "literal", + why: "the health score is computed in-app on a closed 0-100 scale", + }, + + // ── Derived from rows already stored ────────────────────────────────── + "lib/measurements/consolidate-daily-mean.ts": { + kind: "derived", + why: "writes the mean of the day's own stored rows", + }, + "lib/measurements/consolidate-legacy-steps.ts": { + kind: "derived", + why: "folds stored per-sample step rows into the day's canonical row", + }, + "lib/measurements/dense-intraday-retention.ts": { + kind: "derived", + why: "rebuilds hourly buckets from stored per-sample rows", + }, + "lib/measurements/drain-per-sample-cumulative.ts": { + kind: "derived", + why: "reduces stored cumulative per-sample rows to one daily total", + }, + + // ── Restore ─────────────────────────────────────────────────────────── + "app/api/admin/backups/[id]/restore/route.ts": { + kind: "restore", + why: "replays the record's own export; dropping rows would break the restore contract", + }, +}; + +/** + * Any Prisma client handle followed by a measurement write. Whitespace- and + * handle-tolerant on purpose: the earlier generation of this repo's guards + * demanded a literal `prisma.x.findUnique(` and went green because it matched + * nothing, not because nothing was there. The transaction handles in use are + * `tx` and `pc`; a future one is caught by the `[A-Za-z_$][\w$]*` head. + */ +const MEASUREMENT_WRITE = + /[A-Za-z_$][\w$]*\s*\.\s*measurement\s*\.\s*(create|createMany|createManyAndReturn|upsert)\s*\(/; + +function sourceFiles(): string[] { + return walkSourceFiles(SRC, { floor: 3000 }) + .filter((p) => !p.startsWith("generated/")) + .filter((p) => !p.includes("__tests__")) + .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx")) + .sort(); +} + +function read(rel: string): string { + return readFileSync(join(SRC, rel), "utf8"); +} + +function discoveredWriters(): string[] { + return sourceFiles().filter((rel) => MEASUREMENT_WRITE.test(read(rel))); +} + +describe("every measurement writer declares how it meets the plausibility domain", () => { + it("finds the writers at all — an empty sweep is a broken guard, not a clean repo", () => { + // The map is the claim; this asserts the matcher can still see the + // population it describes. A regex that silently stops matching would + // otherwise turn every check below into a tautology. + expect(discoveredWriters().length).toBeGreaterThanOrEqual(15); + }); + + it("the discovered writer set matches the declared map exactly", () => { + expect(discoveredWriters()).toEqual(Object.keys(WRITERS).sort()); + }); + + it("every provider-fed writer reaches the gate module", () => { + const gated = Object.entries(WRITERS) + .filter(([, d]) => d.kind === "gate") + .map(([rel]) => rel); + expect(gated.length).toBeGreaterThanOrEqual(6); + for (const rel of gated) { + expect(read(rel), `${rel} must import ${GATE_MODULE}`).toContain( + GATE_MODULE, + ); + } + }); + + it("every person-driven writer calls the range validator", () => { + for (const [rel, disposition] of Object.entries(WRITERS)) { + if (disposition.kind === "validator") { + expect(read(rel), `${rel} must call ${VALIDATOR}`).toContain(VALIDATOR); + } + if (disposition.kind === "upstream") { + expect( + read(disposition.validatedIn), + `${disposition.validatedIn} must call ${VALIDATOR} on behalf of ${rel}`, + ).toContain(VALIDATOR); + } + } + }); + + it("the exempt writers carry a written reason", () => { + for (const [rel, disposition] of Object.entries(WRITERS)) { + if ( + disposition.kind === "derived" || + disposition.kind === "literal" || + disposition.kind === "restore" + ) { + expect(disposition.why.length, `${rel} needs a reason`).toBeGreaterThan( + 20, + ); + } + } + }); +}); diff --git a/src/app/api/measurements/batch/route.ts b/src/app/api/measurements/batch/route.ts index df8ffcd59..1199e2354 100644 --- a/src/app/api/measurements/batch/route.ts +++ b/src/app/api/measurements/batch/route.ts @@ -620,6 +620,17 @@ async function postBatch(request: NextRequest): Promise { }; failedCount++; continue; + case "rejected_range": + // The reconciler carries the same plausibility gate this route + // already applied above, so reaching here means the two disagreed — + // report it exactly as the earlier guard would have, and let the + // wide-event tally record which end refused. + results[p.index] = { + index: p.index, + status: "skipped", + reason: "value_out_of_range", + }; + continue; } if (isMergeableSource(p.row.source as string)) { diff --git a/src/lib/fitbit/__tests__/sync-plausibility.test.ts b/src/lib/fitbit/__tests__/sync-plausibility.test.ts new file mode 100644 index 000000000..7efe59f99 --- /dev/null +++ b/src/lib/fitbit/__tests__/sync-plausibility.test.ts @@ -0,0 +1,179 @@ +/** + * `upsertFitbitMeasurements` is the single write seam for every Fitbit + * resource leg — activity, sleep, daily metrics — and it holds Prisma + * directly, so it does not inherit the shared reconciler's gate. These cases + * pin that the provider's numbers clear the app's own declared plausibility + * band before any of them can become a row, and that a refusal is reported. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({ + findManyMock: vi.fn(async () => [] as unknown[]), + createManyMock: vi.fn< + () => Promise> + >(async () => []), + updateMock: vi.fn(async () => ({})), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { + findMany: findManyMock, + createManyAndReturn: createManyMock, + update: updateMock, + }, + }, +})); +vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() })); +vi.mock("@/lib/integrations/status", () => ({ + isReauthRequired: vi.fn(async () => false), + recordSyncFailure: vi.fn(async () => {}), + recordSyncSuccess: vi.fn(async () => {}), +})); +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + collapseToTypeDayKeys: vi.fn(() => []), + recomputeBucketsForMeasurement: vi.fn(async () => {}), + recomputeUserRollups: vi.fn(async () => {}), +})); +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: vi.fn(async () => {}), +})); +vi.mock("@/lib/integrations/oauth-refresh", () => ({ + persistRotatedToken: vi.fn(async () => {}), +})); +vi.mock("../credentials", () => ({ + getUserFitbitCredentials: vi.fn(async () => null), +})); +vi.mock("../client", () => ({ refreshAccessToken: vi.fn() })); + +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate"; +import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate"; + +import { upsertFitbitMeasurements } from "../sync-core"; + +/** RESTING_HEART_RATE is declared 25–220. */ +const REAL_READING = { + type: "RESTING_HEART_RATE", + value: 58, + unit: "bpm", + measuredAt: new Date("2026-07-08T00:00:00.000Z"), + externalId: "rhr:2026-07-08", +}; + +const IMPOSSIBLE_READING = { + type: "RESTING_HEART_RATE", + value: 4_915_200, + unit: "bpm", + measuredAt: new Date("2026-07-09T00:00:00.000Z"), + externalId: "rhr:2026-07-09", +}; + +let event: WideEventBuilder; + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +beforeEach(() => { + findManyMock.mockReset().mockResolvedValue([]); + createManyMock.mockReset().mockResolvedValue([]); + updateMock.mockReset().mockResolvedValue({}); + event = new WideEventBuilder("background"); +}); + +describe("upsertFitbitMeasurements — plausibility gate", () => { + it("writes a reading inside the metric's declared band", async () => { + createManyMock.mockResolvedValue([ + { + id: "row-1", + type: "RESTING_HEART_RATE", + measuredAt: REAL_READING.measuredAt, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertFitbitMeasurements("user-1", [REAL_READING], { deferRollup: true }), + ); + + expect(result.imported).toBe(1); + const planned = (createManyMock.mock.calls[0] as unknown[])[0] as { + data: Array<{ value: number }>; + }; + expect(planned.data).toHaveLength(1); + expect(planned.data[0]!.value).toBe(58); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band reading and annotates the drop", async () => { + createManyMock.mockResolvedValue([ + { + id: "row-1", + type: "RESTING_HEART_RATE", + measuredAt: REAL_READING.measuredAt, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertFitbitMeasurements("user-1", [REAL_READING, IMPOSSIBLE_READING], { + deferRollup: true, + }), + ); + + expect(result.imported).toBe(1); + const planned = (createManyMock.mock.calls[0] as unknown[])[0] as { + data: Array<{ value: number; externalId: string }>; + }; + expect(planned.data).toHaveLength(1); + expect(planned.data[0]!.externalId).toBe("rhr:2026-07-08"); + // The refused reading never even reached the existence probe. + expect( + ( + (findManyMock.mock.calls[0] as unknown[])[0] as { + where: { externalId: { in: string[] } }; + } + ).where.externalId.in, + ).toEqual(["rhr:2026-07-08"]); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { + source: "fitbit", + type: "RESTING_HEART_RATE", + direction: "above_max", + count: 1, + }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); + + it("does not overwrite a stored reading with an out-of-band one", async () => { + // The key already exists; ungated, the update branch rewrote its value. + findManyMock.mockResolvedValue([ + { + id: "existing", + type: "RESTING_HEART_RATE", + externalId: "rhr:2026-07-09", + value: 58, + unit: "bpm", + measuredAt: IMPOSSIBLE_READING.measuredAt, + sleepStage: null, + deletedAt: null, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertFitbitMeasurements("user-1", [IMPOSSIBLE_READING], { + deferRollup: true, + }), + ); + + expect(result.imported).toBe(0); + expect(updateMock).not.toHaveBeenCalled(); + expect(createManyMock).not.toHaveBeenCalled(); + expect(tally()?.total).toBe(1); + }); +}); diff --git a/src/lib/fitbit/sync-core.ts b/src/lib/fitbit/sync-core.ts index 74637e01b..f67f4422b 100644 --- a/src/lib/fitbit/sync-core.ts +++ b/src/lib/fitbit/sync-core.ts @@ -15,6 +15,7 @@ import { persistRotatedToken } from "@/lib/integrations/oauth-refresh"; import { recordSyncFailure } from "@/lib/integrations/status"; import type { FailureKind } from "@/lib/integrations/status"; import { getEvent } from "@/lib/logging/context"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { collapseToTypeDayKeys, recomputeBucketsForMeasurement, @@ -530,7 +531,7 @@ export async function replaceStaleFitbitSleep( */ export async function upsertFitbitMeasurements( userId: string, - readings: FitbitMeasurementUpsert[], + incoming: FitbitMeasurementUpsert[], opts: { deferRollup?: boolean } = {}, ): Promise<{ imported: number; @@ -541,6 +542,15 @@ export async function upsertFitbitMeasurements( measuredAt: Date; }>; }> { + // Every resource leg funnels through here, so this is the one place the + // provider's numbers have to clear the app's declared plausibility band + // before they can become rows. A refused reading is dropped and tallied on + // the ambient event, never clamped and never written for a later reader to + // discover. + const readings = dropImplausibleMeasurements("fitbit", incoming, (r) => ({ + type: r.type, + value: r.value, + })); if (readings.length === 0) return { imported: 0, touched: [], inserted: [] }; // Probe EVERY existing row (live AND tombstoned) for the batch's externalIds diff --git a/src/lib/google-health/__tests__/sync-plausibility.test.ts b/src/lib/google-health/__tests__/sync-plausibility.test.ts new file mode 100644 index 000000000..2fb5b3404 --- /dev/null +++ b/src/lib/google-health/__tests__/sync-plausibility.test.ts @@ -0,0 +1,180 @@ +/** + * `upsertGoogleHealthMeasurements` is the single write seam for every Google + * Health resource leg and it holds Prisma directly, so it does not inherit the + * shared reconciler's gate. These cases pin that the provider's numbers clear + * the app's own declared plausibility band before any of them can become a + * row, and that a refusal is reported rather than swallowed. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({ + findManyMock: vi.fn(async () => [] as unknown[]), + createManyMock: vi.fn< + () => Promise> + >(async () => []), + updateMock: vi.fn(async () => ({})), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { + findMany: findManyMock, + createManyAndReturn: createManyMock, + update: updateMock, + }, + }, +})); +vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() })); +vi.mock("@/lib/integrations/status", () => ({ + isReauthRequired: vi.fn(async () => false), + recordSyncFailure: vi.fn(async () => {}), + recordSyncSuccess: vi.fn(async () => {}), +})); +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + collapseToTypeDayKeys: vi.fn(() => []), + recomputeBucketsForMeasurement: vi.fn(async () => {}), + recomputeUserRollups: vi.fn(async () => {}), +})); +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: vi.fn(async () => {}), +})); +vi.mock("../credentials", () => ({ + getUserGoogleHealthCredentials: vi.fn(async () => null), +})); +vi.mock("../client", async () => { + const actual = await vi.importActual("../client"); + return { + ...actual, + refreshAccessToken: vi.fn(), + noteGoogleHealthMapped: vi.fn(), + noteGoogleHealthWritten: vi.fn(), + noteGoogleHealthOutcomeFailure: vi.fn(), + }; +}); + +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate"; +import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate"; + +import { upsertGoogleHealthMeasurements } from "../sync-core"; + +/** OXYGEN_SATURATION is declared 50–100. */ +const REAL_READING = { + type: "OXYGEN_SATURATION", + value: 97, + unit: "%", + measuredAt: new Date("2026-07-08T00:00:00.000Z"), + externalId: "spo2:2026-07-08", +}; + +const IMPOSSIBLE_READING = { + type: "OXYGEN_SATURATION", + value: 9_700, + unit: "%", + measuredAt: new Date("2026-07-09T00:00:00.000Z"), + externalId: "spo2:2026-07-09", +}; + +let event: WideEventBuilder; + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +beforeEach(() => { + findManyMock.mockReset().mockResolvedValue([]); + createManyMock.mockReset().mockResolvedValue([]); + updateMock.mockReset().mockResolvedValue({}); + event = new WideEventBuilder("background"); +}); + +describe("upsertGoogleHealthMeasurements — plausibility gate", () => { + it("writes a reading inside the metric's declared band", async () => { + createManyMock.mockResolvedValue([ + { + id: "row-1", + type: "OXYGEN_SATURATION", + measuredAt: REAL_READING.measuredAt, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertGoogleHealthMeasurements("user-1", [REAL_READING], { + deferRollup: true, + }), + ); + + expect(result.imported).toBe(1); + const planned = (createManyMock.mock.calls[0] as unknown[])[0] as { + data: Array<{ value: number }>; + }; + expect(planned.data).toHaveLength(1); + expect(planned.data[0]!.value).toBe(97); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band reading and annotates the drop", async () => { + createManyMock.mockResolvedValue([ + { + id: "row-1", + type: "OXYGEN_SATURATION", + measuredAt: REAL_READING.measuredAt, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertGoogleHealthMeasurements( + "user-1", + [REAL_READING, IMPOSSIBLE_READING], + { deferRollup: true }, + ), + ); + + expect(result.imported).toBe(1); + const planned = (createManyMock.mock.calls[0] as unknown[])[0] as { + data: Array<{ externalId: string }>; + }; + expect(planned.data).toHaveLength(1); + expect(planned.data[0]!.externalId).toBe("spo2:2026-07-08"); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { + source: "googleHealth", + type: "OXYGEN_SATURATION", + direction: "above_max", + count: 1, + }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); + + it("does not overwrite a stored reading with an out-of-band one", async () => { + findManyMock.mockResolvedValue([ + { + id: "existing", + type: "OXYGEN_SATURATION", + externalId: "spo2:2026-07-09", + value: 97, + unit: "%", + measuredAt: IMPOSSIBLE_READING.measuredAt, + sleepStage: null, + deletedAt: null, + }, + ]); + + const result = await eventStorage.run(event, () => + upsertGoogleHealthMeasurements("user-1", [IMPOSSIBLE_READING], { + deferRollup: true, + }), + ); + + expect(result.imported).toBe(0); + expect(updateMock).not.toHaveBeenCalled(); + expect(createManyMock).not.toHaveBeenCalled(); + expect(tally()?.total).toBe(1); + }); +}); diff --git a/src/lib/google-health/sync-core.ts b/src/lib/google-health/sync-core.ts index 27ccaa139..d0545ee0d 100644 --- a/src/lib/google-health/sync-core.ts +++ b/src/lib/google-health/sync-core.ts @@ -20,6 +20,7 @@ import { prisma } from "@/lib/db"; import type { MeasurementType } from "@/generated/prisma/client"; import { encrypt, decrypt } from "@/lib/crypto"; import { getEvent } from "@/lib/logging/context"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { recordSyncFailure, type FailureKind } from "@/lib/integrations/status"; import { @@ -487,7 +488,7 @@ export async function replaceStaleGoogleHealthSleep( */ export async function upsertGoogleHealthMeasurements( userId: string, - readings: GoogleHealthMeasurementUpsert[], + incoming: GoogleHealthMeasurementUpsert[], opts: { deferRollup?: boolean } = {}, ): Promise<{ imported: number; @@ -498,7 +499,17 @@ export async function upsertGoogleHealthMeasurements( measuredAt: Date; }>; }> { - noteGoogleHealthMapped(readings.length); + noteGoogleHealthMapped(incoming.length); + // Every resource leg funnels through here, so this is the one place the + // provider's numbers have to clear the app's declared plausibility band + // before they can become rows. A refused reading is dropped and tallied on + // the ambient event, never clamped and never written for a later reader to + // discover. + const readings = dropImplausibleMeasurements( + "googleHealth", + incoming, + (r) => ({ type: r.type, value: r.value }), + ); if (readings.length === 0) return { imported: 0, touched: [], inserted: [] }; // Probe EVERY existing row (live AND tombstoned) for the batch's externalIds diff --git a/src/lib/measurements/__tests__/plausibility-gate.test.ts b/src/lib/measurements/__tests__/plausibility-gate.test.ts new file mode 100644 index 000000000..074d32600 --- /dev/null +++ b/src/lib/measurements/__tests__/plausibility-gate.test.ts @@ -0,0 +1,222 @@ +/** + * The ingest-side plausibility gate. + * + * `VALUE_RANGES` is the band the application declares a metric can occupy, and + * every path a person can drive has always enforced it. The provider sync + * writers did not, which is how a row the application itself calls impossible + * reached the table and poisoned every mean and personal band derived from it. + * These cases pin the two halves of the fix: the refusal, and the fact that a + * refusal is never silent. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { eventStorage } from "@/lib/logging/context"; +import { + RANGE_REJECTED_META_KEY, + classifyRangeRejection, + dropImplausibleMeasurements, + recordRangeRejections, + type RangeRejectionTally, +} from "../plausibility-gate"; + +let event: WideEventBuilder; + +function withEvent(run: () => T): T { + return eventStorage.run(event, run); +} + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +beforeEach(() => { + event = new WideEventBuilder("background"); +}); + +describe("classifyRangeRejection", () => { + it("passes a value inside the metric's declared band", () => { + // PULSE is declared 20–300; both edges are inclusive. + expect(classifyRangeRejection("PULSE", 60)).toBeNull(); + expect(classifyRangeRejection("PULSE", 20)).toBeNull(); + expect(classifyRangeRejection("PULSE", 300)).toBeNull(); + }); + + it("names the edge an out-of-band value broke", () => { + expect(classifyRangeRejection("PULSE", 55643821.505)).toEqual({ + type: "PULSE", + direction: "above_max", + }); + expect(classifyRangeRejection("PULSE", 19)).toEqual({ + type: "PULSE", + direction: "below_min", + }); + }); + + it("refuses a non-finite value whatever the metric", () => { + expect(classifyRangeRejection("PULSE", Number.NaN)).toEqual({ + type: "PULSE", + direction: "not_finite", + }); + // No declared band, so the value itself is the only thing to judge. + expect( + classifyRangeRejection("NOT_A_METRIC", Number.POSITIVE_INFINITY), + ).toEqual({ type: "NOT_A_METRIC", direction: "not_finite" }); + }); + + it("passes a metric with no declared band — an absent range is an absent fact", () => { + expect(classifyRangeRejection("NOT_A_METRIC", 9_999_999)).toBeNull(); + }); +}); + +describe("dropImplausibleMeasurements", () => { + it("keeps the in-band rows and returns them untouched", () => { + const rows = [ + { type: "PULSE", value: 60 }, + { type: "WEIGHT", value: 82.4 }, + ]; + const kept = withEvent(() => + dropImplausibleMeasurements("withings", rows, (r) => r), + ); + + expect(kept).toEqual(rows); + expect(kept[0]).toBe(rows[0]); + // Nothing was refused, so the event carries no tally at all — absence + // reads as absence. + expect(tally()).toBeUndefined(); + expect(event.toJSON().warnings).toBeUndefined(); + }); + + it("drops the out-of-band rows and tallies them on the wide event", () => { + const kept = withEvent(() => + dropImplausibleMeasurements( + "withings", + [ + { type: "PULSE", value: 60 }, + { type: "PULSE", value: 55643821.505 }, + { type: "PULSE", value: 900 }, + { type: "WEIGHT", value: 0.2 }, + ], + (r) => r, + ), + ); + + expect(kept).toEqual([{ type: "PULSE", value: 60 }]); + + const recorded = tally(); + expect(recorded).toEqual({ + total: 3, + buckets: [ + { + source: "withings", + type: "PULSE", + direction: "above_max", + count: 2, + }, + { + source: "withings", + type: "WEIGHT", + direction: "below_min", + count: 1, + }, + ], + }); + }); + + it("keeps the refused readings out of the log line", () => { + withEvent(() => + dropImplausibleMeasurements( + "withings", + [{ type: "PULSE", value: 55643821.505 }], + (r) => r, + ), + ); + + // The value is a health reading. The tally says which metric, which + // provider and which edge — never the number itself. + const serialised = JSON.stringify(event.toJSON()); + expect(serialised).not.toContain("55643821"); + }); + + it("lifts the event to warn once per provider", () => { + withEvent(() => { + dropImplausibleMeasurements( + "withings", + [{ type: "PULSE", value: 900 }], + (r) => r, + ); + dropImplausibleMeasurements( + "withings", + [{ type: "PULSE", value: 901 }], + (r) => r, + ); + dropImplausibleMeasurements( + "fitbit", + [{ type: "PULSE", value: 902 }], + (r) => r, + ); + }); + + expect(event.getLevel()).toBe("warn"); + expect(event.toJSON().warnings).toHaveLength(2); + expect(event.toJSON().warnings?.[0]).toContain("withings"); + expect(event.toJSON().warnings?.[1]).toContain("fitbit"); + }); + + it("accumulates across calls instead of overwriting the previous tally", () => { + // Fitbit gates once per resource leg, and the reconciler gates once per + // row: a tally that replaced rather than merged would report only the + // last batch, and a run that dropped hundreds would read as dropping one. + withEvent(() => { + dropImplausibleMeasurements( + "fitbit", + [{ type: "PULSE", value: 900 }], + (r) => r, + ); + dropImplausibleMeasurements( + "fitbit", + [ + { type: "PULSE", value: 901 }, + { type: "OXYGEN_SATURATION", value: 4 }, + ], + (r) => r, + ); + }); + + expect(tally()).toEqual({ + total: 3, + buckets: [ + { + source: "fitbit", + type: "OXYGEN_SATURATION", + direction: "below_min", + count: 1, + }, + { source: "fitbit", type: "PULSE", direction: "above_max", count: 2 }, + ], + }); + }); + + it("still drops outside a wide-event context", () => { + // A sync running without an ambient event has nowhere to report. The row + // is refused anyway: observability is what is lost, never the guarantee. + const kept = dropImplausibleMeasurements( + "nightscout", + [ + { type: "BLOOD_GLUCOSE", value: 110 }, + { type: "BLOOD_GLUCOSE", value: 9_000 }, + ], + (r) => r, + ); + expect(kept).toEqual([{ type: "BLOOD_GLUCOSE", value: 110 }]); + }); +}); + +describe("recordRangeRejections", () => { + it("writes nothing when there is nothing to report", () => { + withEvent(() => recordRangeRejections("withings", [])); + expect(tally()).toBeUndefined(); + expect(event.getLevel()).toBe("info"); + }); +}); diff --git a/src/lib/measurements/__tests__/reconcile-external-measurement.test.ts b/src/lib/measurements/__tests__/reconcile-external-measurement.test.ts index 5a2d012fd..9fa5c3cf7 100644 --- a/src/lib/measurements/__tests__/reconcile-external-measurement.test.ts +++ b/src/lib/measurements/__tests__/reconcile-external-measurement.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it, vi } from "vitest"; import type { Prisma } from "@/generated/prisma/client"; +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "../plausibility-gate"; import { reconcileExternalMeasurement, type ExternalMeasurementWrite, @@ -383,3 +386,104 @@ describe("reconcileExternalMeasurement", () => { ); }); }); + +/** + * The reconciler is the shared write seam for every provider that does not + * hold Prisma itself — Polar, Oura, WHOOP, the Withings sleep leg, the Apple + * export importer, the batch endpoint. Gating here rather than at each caller + * is what makes a provider added tomorrow inherit the refusal instead of + * having to remember a check it never knew about. + */ +describe("reconcileExternalMeasurement — plausibility gate", () => { + it("reconciles a value inside the metric's declared band", async () => { + const { tx, measurement } = transaction([]); + measurement.create.mockResolvedValue(row({ id: "fresh" })); + + // PULSE is declared 20–300. + const verdict = await reconcileExternalMeasurement( + tx, + desired({ type: "PULSE", value: 112, unit: "bpm", sleepStage: null }), + ); + + expect(verdict.status).toBe("inserted"); + expect(measurement.create).toHaveBeenCalledTimes(1); + }); + + it("refuses an out-of-band value without touching the transaction", async () => { + const { tx, measurement } = transaction([]); + + const verdict = await reconcileExternalMeasurement( + tx, + desired({ + type: "PULSE", + value: 55643821.505, + unit: "bpm", + sleepStage: null, + }), + ); + + expect(verdict).toEqual({ + status: "rejected_range", + rejection: { type: "PULSE", direction: "above_max" }, + }); + // No row written, no row updated, and no savepoint opened — the refusal + // happens before the reconciler does any work at all. + expect(measurement.create).not.toHaveBeenCalled(); + expect(measurement.update).not.toHaveBeenCalled(); + expect(tx.$executeRawUnsafe).not.toHaveBeenCalled(); + }); + + it("refuses rather than overwriting a good stored row", async () => { + // The dangerous shape: a live row already holds a real reading, and the + // provider re-sends the same identity carrying an impossible value. The + // update branch would have rewritten a good value with a bad one. + const existing = row({ + id: "exact-row", + type: "PULSE", + value: 62, + unit: "bpm", + sleepStage: null, + measuredAt: new Date("2026-07-20T06:30:00.000Z"), + }); + const { tx, measurement } = transaction([existing]); + + const verdict = await reconcileExternalMeasurement( + tx, + desired({ + type: "PULSE", + value: 55643821.505, + unit: "bpm", + sleepStage: null, + }), + ); + + expect(verdict.status).toBe("rejected_range"); + expect(measurement.update).not.toHaveBeenCalled(); + }); + + it("tallies the refusal on the ambient wide event under the provider", async () => { + const { tx } = transaction([]); + const event = new WideEventBuilder("background"); + + await eventStorage.run(event, () => + reconcileExternalMeasurement( + tx, + desired({ + type: "PULSE", + value: 55643821.505, + unit: "bpm", + source: "WHOOP", + sleepStage: null, + }), + ), + ); + + expect(event.toJSON().meta?.[RANGE_REJECTED_META_KEY]).toEqual({ + total: 1, + buckets: [ + { source: "whoop", type: "PULSE", direction: "above_max", count: 1 }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); +}); diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index fc1e9f838..31d0b7dc2 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -720,6 +720,16 @@ export async function streamParseExportXml( ), ); + // The reconciler carries the same plausibility gate the block above + // already applied, so this arm means the two disagreed. Tally it under + // the same counter the earlier guard uses and count no row. + if (verdict.status === "rejected_range") { + unknown[`${type}::aggregate_out_of_range`] = + (unknown[`${type}::aggregate_out_of_range`] ?? 0) + 1; + stat.durationMs += Date.now() - rowStart; + continue; + } + cumulativeEstimatedDays.add(dayKey); cumulativeEstimatedRows += 1; if (verdict.status === "inserted") { diff --git a/src/lib/measurements/plausibility-gate.ts b/src/lib/measurements/plausibility-gate.ts new file mode 100644 index 000000000..d079f5391 --- /dev/null +++ b/src/lib/measurements/plausibility-gate.ts @@ -0,0 +1,192 @@ +/** + * The ingest-side half of the app's plausibility domain. + * + * `VALUE_RANGES` in `@/lib/validations/measurement` is the band the + * application declares a metric can occupy — PULSE 20–300, WEIGHT 1–500, + * SLEEP_DURATION 0–1440 minutes. Every path a person can drive enforces it: + * the measurement routes, the batch endpoint, the CSV and Apple-export + * importers, MCP, Telegram. The provider sync writers did not. They took the + * number the remote handed over and wrote it, so a decode slip or a glitched + * device landed a row the application itself calls impossible, and every mean, + * median and personal band downstream digested it as a reading. + * + * `isPlausibleMetricValue` (in `value-domain.ts`) already answers the question + * for the READ side, which is where the damage surfaced first. This module is + * the same question asked one layer earlier, at the point of writing, so the + * row never lands. The two ends share the one predicate on purpose: a band the + * reader hides but the writer accepts is a row that exists and cannot be seen, + * which is the worse of the two failures. + * + * ## Disposition: drop, and say so + * + * An out-of-band incoming value is DROPPED. Not clamped (a clamp invents a + * reading nobody took), not stored-and-flagged (a quarantined row is still a + * row, and every consumer would need to learn the flag), not fatal to the + * batch (one glitched sample must not strand the sync's other readings behind + * a held watermark). Absence reads as absence: the sample is simply not part + * of the record. + * + * A silent drop, though, is indistinguishable from a provider that sent + * nothing — so every drop is tallied onto the ambient wide event under + * {@link RANGE_REJECTED_META_KEY}, with the metric type, the provider, and the + * direction it broke the band in. Counts only. The value itself is a health + * reading and never enters the log line; the direction plus the metric's own + * declared bound is what a diagnosis actually needs. + * + * ## Why meta and not `action` + * + * A wide event holds exactly ONE action, and `setAction` replaces it. A drop + * happens in the middle of a sync whose action is decided at the end, so an + * `action` set here is either clobbered a moment later (the drop vanishes) or, + * on the paths that annotate no terminal action, relabels an otherwise + * successful sync as a rejection. Meta accumulates, survives every later + * `setAction`, and is the shape a dashboard can sum. One warning per provider + * per event rides along so the drop also lifts the event's level to `warn` + * rather than hiding inside a green line. + */ +import { annotate, getEvent } from "@/lib/logging/context"; +import { isPlausibleMetricValue } from "@/lib/measurements/value-domain"; +import { VALUE_RANGES } from "@/lib/validations/measurement"; + +/** + * Stable meta key the tally lands under. Pinned so dashboards can count it + * across providers without knowing which one wrote the event. + */ +export const RANGE_REJECTED_META_KEY = "measurement.range_rejected"; + +/** Which edge of the declared band the value broke, or that it was not a number. */ +export type RangeRejectionDirection = "below_min" | "above_max" | "not_finite"; + +/** One refused reading, reduced to the facts that carry no health value. */ +export interface MeasurementRangeRejection { + /** `MeasurementType` of the refused reading. */ + type: string; + direction: RangeRejectionDirection; +} + +/** One (provider, metric, direction) cell of the event's running tally. */ +export interface RangeRejectionBucket { + source: string; + type: string; + direction: RangeRejectionDirection; + count: number; +} + +/** The pinned meta shape written under {@link RANGE_REJECTED_META_KEY}. */ +export interface RangeRejectionTally { + total: number; + buckets: RangeRejectionBucket[]; +} + +/** + * Classify a candidate write against the metric's declared band. + * + * Returns `null` when the value may be written. A type with no declared range + * passes — an absent range is an absent fact, never a guessed one, and the + * same asymmetry the read side uses (`isPlausibleMetricValue`). A non-finite + * value never passes, whatever the type: NaN is not an input to anything. + */ +export function classifyRangeRejection( + type: string, + value: number, +): MeasurementRangeRejection | null { + if (isPlausibleMetricValue(type, value)) return null; + if (!Number.isFinite(value)) return { type, direction: "not_finite" }; + const range = VALUE_RANGES[type]; + // `isPlausibleMetricValue` only answers false for a finite value when the + // type HAS a range, so the lookup cannot miss here. + return { + type, + direction: value < range!.min ? "below_min" : "above_max", + }; +} + +function readTally(): RangeRejectionTally { + const existing = getEvent()?.toJSON().meta?.[RANGE_REJECTED_META_KEY]; + if ( + existing !== null && + typeof existing === "object" && + Array.isArray((existing as RangeRejectionTally).buckets) + ) { + return existing as RangeRejectionTally; + } + return { total: 0, buckets: [] }; +} + +/** + * Fold refusals onto the ambient wide event. + * + * Merges with whatever the event already carries, so a provider that gates + * several batches (or reconciles row by row) accumulates instead of + * overwriting. Outside a wide-event context this is a no-op — the caller still + * drops the row, it simply has nowhere to report. + */ +export function recordRangeRejections( + source: string, + rejections: readonly MeasurementRangeRejection[], +): void { + if (rejections.length === 0) return; + const evt = getEvent(); + if (!evt) return; + + const tally = readTally(); + const firstForSource = !tally.buckets.some((b) => b.source === source); + + for (const rejection of rejections) { + const bucket = tally.buckets.find( + (b) => + b.source === source && + b.type === rejection.type && + b.direction === rejection.direction, + ); + if (bucket) bucket.count += 1; + else + tally.buckets.push({ + source, + type: rejection.type, + direction: rejection.direction, + count: 1, + }); + tally.total += 1; + } + + tally.buckets.sort( + (a, b) => + a.source.localeCompare(b.source) || + a.type.localeCompare(b.type) || + a.direction.localeCompare(b.direction), + ); + + annotate({ meta: { [RANGE_REJECTED_META_KEY]: tally } }); + if (firstForSource) { + evt.addWarning( + `${source}: refused reading(s) outside the declared plausibility range — ` + + `dropped, see meta.${RANGE_REJECTED_META_KEY}`, + ); + } +} + +/** + * Keep the rows whose value sits inside the metric's declared band; tally and + * drop the rest. + * + * `select` exists because the sync writers carry their own row shapes and none + * of them is worth reshaping at the call site: the gate reads the two fields + * it needs and returns the caller's own objects untouched. + */ +export function dropImplausibleMeasurements( + source: string, + rows: readonly T[], + select: (row: T) => { type: string; value: number }, +): T[] { + const kept: T[] = []; + const rejections: MeasurementRangeRejection[] = []; + for (const row of rows) { + const { type, value } = select(row); + const rejection = classifyRangeRejection(type, value); + if (rejection) rejections.push(rejection); + else kept.push(row); + } + recordRangeRejections(source, rejections); + return kept; +} diff --git a/src/lib/measurements/reconcile-external-measurement.ts b/src/lib/measurements/reconcile-external-measurement.ts index 031dda441..d4fb50784 100644 --- a/src/lib/measurements/reconcile-external-measurement.ts +++ b/src/lib/measurements/reconcile-external-measurement.ts @@ -4,6 +4,11 @@ import { type MeasurementAggregationProvenance, type Prisma as PrismaTypes, } from "@/generated/prisma/client"; +import { + classifyRangeRejection, + recordRangeRejections, + type MeasurementRangeRejection, +} from "@/lib/measurements/plausibility-gate"; const SAVEPOINT = "measurement_identity_reconcile"; @@ -50,6 +55,18 @@ export type MeasurementReconciliationVerdict = | { status: "failed"; error: { message: string; code?: string }; + } + | { + /** + * The incoming value sits outside the metric's declared plausibility + * band, so nothing was written and no existing row was touched. Not a + * failure: the write was refused on purpose and the caller must carry + * on with the rest of its batch. Every caller narrows on this status + * because it carries no `row` — the compiler is what makes a future + * provider handle it rather than dereference a row that does not exist. + */ + status: "rejected_range"; + rejection: MeasurementRangeRejection; }; export type FailedMeasurementReconciliation = Extract< @@ -219,12 +236,26 @@ async function retireCollision( * advisory lock serializes cooperative writers sharing either identity, and a * savepoint turns a non-benign database error into a structured `failed` * verdict without poisoning the caller's surrounding transaction. + * + * This is also the plausibility gate for every provider that reconciles rather + * than writing Prisma directly. Gating HERE rather than at each caller is the + * point: a provider added tomorrow inherits the refusal by construction, and + * cannot land an impossible reading by forgetting a check it never knew about. */ export async function reconcileExternalMeasurement( tx: PrismaTypes.TransactionClient, desired: ExternalMeasurementWrite, options: MeasurementReconciliationOptions = {}, ): Promise { + const rejection = classifyRangeRejection(desired.type, desired.value); + if (rejection) { + recordRangeRejections( + desired.source ? String(desired.source).toLowerCase() : "external", + [rejection], + ); + return { status: "rejected_range", rejection }; + } + await tx.$executeRawUnsafe(`SAVEPOINT ${SAVEPOINT}`); try { diff --git a/src/lib/nightscout/__tests__/sync-plausibility.test.ts b/src/lib/nightscout/__tests__/sync-plausibility.test.ts new file mode 100644 index 000000000..24c023eb7 --- /dev/null +++ b/src/lib/nightscout/__tests__/sync-plausibility.test.ts @@ -0,0 +1,115 @@ +/** + * The Nightscout pull writes BLOOD_GLUCOSE rows directly, so it does not + * inherit the shared reconciler's gate. It is also first-write-wins: a row + * that lands here is immutable, so an impossible sample never corrects itself + * on a later pass. These cases pin the refusal and its annotation. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { + upsertMock, + findManyMock, + emitArrivalsMock, + recomputeMock, + invalidateMock, +} = vi.hoisted(() => ({ + upsertMock: vi.fn(), + findManyMock: vi.fn(), + emitArrivalsMock: vi.fn(), + recomputeMock: vi.fn(), + invalidateMock: vi.fn(), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { measurement: { upsert: upsertMock, findMany: findManyMock } }, +})); + +vi.mock("@/lib/arrivals/measurement-emit", () => ({ + emitInsertedMeasurementArrivals: emitArrivalsMock, +})); + +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + collapseToTypeDayKeys: (rows: Array<{ type: string; measuredAt: Date }>) => + rows.map((r) => ({ type: r.type, measuredAt: r.measuredAt })), + recomputeBucketsForMeasurement: recomputeMock, +})); + +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: invalidateMock, +})); + +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate"; +import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate"; + +import { upsertNightscoutEntries } from "../sync"; + +/** BLOOD_GLUCOSE is declared 20–800 mg/dL. */ +const REAL_ENTRY = { id: "abc", sgv: 112, date: 1718000000000 }; +/** A sensor fault, or an mg/dL field carrying something else entirely. */ +const IMPOSSIBLE_ENTRY = { id: "def", sgv: 41_600, date: 1718000300000 }; + +let event: WideEventBuilder; + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +beforeEach(() => { + vi.clearAllMocks(); + findManyMock.mockResolvedValue([]); + upsertMock.mockResolvedValue({ id: "row-1" }); + emitArrivalsMock.mockResolvedValue(undefined); + recomputeMock.mockResolvedValue(undefined); + invalidateMock.mockResolvedValue(undefined); + event = new WideEventBuilder("background"); +}); + +describe("upsertNightscoutEntries — plausibility gate", () => { + it("writes a reading inside the metric's declared band", async () => { + const result = await eventStorage.run(event, () => + upsertNightscoutEntries("user-1", [REAL_ENTRY]), + ); + + expect(result).toEqual({ inserted: 1, failedRows: 0 }); + expect(upsertMock).toHaveBeenCalledTimes(1); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band reading and annotates the drop", async () => { + const result = await eventStorage.run(event, () => + upsertNightscoutEntries("user-1", [REAL_ENTRY, IMPOSSIBLE_ENTRY]), + ); + + expect(result).toEqual({ inserted: 1, failedRows: 0 }); + expect(upsertMock).toHaveBeenCalledTimes(1); + expect( + (upsertMock.mock.calls[0]![0] as { create: { value: number } }).create + .value, + ).toBe(112); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { + source: "nightscout", + type: "BLOOD_GLUCOSE", + direction: "above_max", + count: 1, + }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); + + it("writes nothing when the whole page is out of band", async () => { + const result = await eventStorage.run(event, () => + upsertNightscoutEntries("user-1", [IMPOSSIBLE_ENTRY]), + ); + + expect(result).toEqual({ inserted: 0, failedRows: 0 }); + expect(upsertMock).not.toHaveBeenCalled(); + expect(tally()?.total).toBe(1); + }); +}); diff --git a/src/lib/nightscout/sync.ts b/src/lib/nightscout/sync.ts index adef2de55..f92104550 100644 --- a/src/lib/nightscout/sync.ts +++ b/src/lib/nightscout/sync.ts @@ -22,6 +22,7 @@ import { prisma } from "@/lib/db"; import type { MeasurementType } from "@/generated/prisma/client"; import { getEvent } from "@/lib/logging/context"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { markSyncFailureRecorded, recordSyncFailure, @@ -221,7 +222,16 @@ export async function upsertNightscoutEntries( if (entries.length === 0) return { inserted: 0, failedRows: 0 }; const type = "BLOOD_GLUCOSE" as MeasurementType; - const mapped = entries.map((entry) => mapSgvEntryToMeasurement(entry)); + // A CGM that reports a glucose the app's own band calls impossible is + // reporting a sensor fault or a unit slip, not a reading. Drop it: the row + // would otherwise be immutable (first-write-wins), so an impossible sample + // that lands here never corrects itself on a later pass. + const mapped = dropImplausibleMeasurements( + "nightscout", + entries.map((entry) => mapSgvEntryToMeasurement(entry)), + (m) => ({ type, value: m.value }), + ); + if (mapped.length === 0) return { inserted: 0, failedRows: 0 }; // Probe which keys already exist so a re-confirmed reading (the upsert's // no-op update branch) is not miscounted as a fresh import and the arrival diff --git a/src/lib/oura/sync.ts b/src/lib/oura/sync.ts index 4dfefaa83..aa11c88f5 100644 --- a/src/lib/oura/sync.ts +++ b/src/lib/oura/sync.ts @@ -575,6 +575,10 @@ export async function upsertOuraMeasurements( for (let index = 0; index < readings.length; index++) { const reading = readings[index]!; const verdict = verdicts[index]!; + // Refused by the plausibility gate: nothing was written and nothing was + // touched, so the reading counts as neither imported nor a rollup dirt + // mark. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); } diff --git a/src/lib/polar/sync.ts b/src/lib/polar/sync.ts index 17b6505c5..8b44200fe 100644 --- a/src/lib/polar/sync.ts +++ b/src/lib/polar/sync.ts @@ -364,6 +364,10 @@ export async function upsertPolarMeasurements( for (let index = 0; index < readings.length; index++) { const reading = readings[index]!; const verdict = verdicts[index]!; + // Refused by the plausibility gate: nothing was written and nothing was + // touched, so the reading counts as neither imported nor a rollup dirt + // mark. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); } diff --git a/src/lib/whoop/sync-core.ts b/src/lib/whoop/sync-core.ts index 2e2e41212..464ab6718 100644 --- a/src/lib/whoop/sync-core.ts +++ b/src/lib/whoop/sync-core.ts @@ -480,6 +480,10 @@ export async function upsertWhoopMeasurements( for (let index = chunkStart; index < chunkEnd; index++) { const reading = readings[index]!; const verdict = verdicts[index - chunkStart]!; + // Refused by the plausibility gate: nothing was written and nothing was + // touched, so the reading counts as neither imported nor a rollup dirt + // mark. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); touchedTypes.add(dirty.type); diff --git a/src/lib/withings/__tests__/sync-plausibility.test.ts b/src/lib/withings/__tests__/sync-plausibility.test.ts new file mode 100644 index 000000000..8aa7572c7 --- /dev/null +++ b/src/lib/withings/__tests__/sync-plausibility.test.ts @@ -0,0 +1,261 @@ +/** + * Withings writes the measurement table directly — the measures leg row by + * row, the activity leg as a planned batch — so neither inherits the shared + * reconciler's gate. These cases pin that both legs now clear the app's own + * declared plausibility band before anything reaches Prisma, and that a + * refusal lands on the wide event rather than vanishing. + * + * The measures leg matters twice over: it does not only insert, it OVERWRITES + * the stored value of an existing slot, so an ungated impossible reading could + * replace a real one. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { + findFirst: vi.fn(), + findMany: vi.fn(), + create: vi.fn(), + createMany: vi.fn(), + update: vi.fn(), + }, + withingsConnection: { + findUnique: vi.fn(), + update: vi.fn(), + }, + $queryRaw: vi.fn(), + $transaction: vi.fn((ops: unknown) => + Array.isArray(ops) ? Promise.all(ops) : Promise.resolve(), + ), + }, +})); + +vi.mock("@/lib/crypto", () => ({ + decrypt: vi.fn((v: string) => v), + encrypt: vi.fn((v: string) => v), +})); + +vi.mock("../client", async () => { + const actual = await vi.importActual("../client"); + return { + ...actual, + fetchMeasurements: vi.fn(), + refreshAccessToken: vi.fn(), + subscribeWebhook: vi.fn(), + }; +}); + +vi.mock("../credentials", () => ({ + getUserWithingsCredentials: vi.fn(async () => ({ + clientId: "cid", + clientSecret: "secret", + })), +})); + +vi.mock("@/lib/integrations/status", () => ({ + isReauthRequired: vi.fn().mockResolvedValue(false), + parkIntegrationAtReauth: vi.fn(), + recordSyncFailure: vi.fn().mockResolvedValue(undefined), + recordSyncSuccess: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/rollups/measurement-rollups", async () => { + const actual = await vi.importActual< + typeof import("@/lib/rollups/measurement-rollups") + >("@/lib/rollups/measurement-rollups"); + return { + ...actual, + recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + }; +}); + +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserDashboardSnapshot: vi.fn(), +})); + +import { prisma } from "@/lib/db"; +import { eventStorage } from "@/lib/logging/context"; +import { WideEventBuilder } from "@/lib/logging/event-builder"; +import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate"; +import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate"; + +import { fetchMeasurements } from "../client"; +import { syncUserActivity } from "../sync-activity"; +import { syncUserMeasurements } from "../sync"; + +let event: WideEventBuilder; + +function tally(): RangeRejectionTally | undefined { + return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as + RangeRejectionTally | undefined; +} + +function installActivityFetch( + entries: Array<{ date: string; steps?: number; distance?: number }>, +) { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + status: 200, + headers: new Headers(), + json: async () => ({ + status: 0, + body: { activities: entries, more: false, offset: 0 }, + }), + })), + ); +} + +beforeEach(() => { + vi.clearAllMocks(); + event = new WideEventBuilder("background"); + vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({ + id: "conn-1", + withingsUserId: "wu-1", + accessToken: "enc-access", + refreshToken: "enc-refresh", + scope: "user.metrics,user.activity", + tokenExpiresAt: new Date(Date.now() + 60 * 60 * 1000), + lastSyncedAt: new Date(Date.now() - 60 * 60 * 1000), + } as never); + vi.mocked(prisma.withingsConnection.update).mockResolvedValue({} as never); + vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.measurement.create).mockResolvedValue({ + id: "m-1", + type: "PULSE", + measuredAt: new Date(), + } as never); + vi.mocked(prisma.measurement.createMany).mockResolvedValue({ + count: 1, + } as never); + vi.mocked(prisma.measurement.update).mockResolvedValue({} as never); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("syncUserMeasurements — plausibility gate", () => { + it("writes a reading inside the metric's declared band", async () => { + vi.mocked(fetchMeasurements).mockResolvedValue([ + { + type: "PULSE", + value: 112, + measuredAt: new Date("2026-05-16T08:00:00Z"), + }, + ] as never); + + const result = await eventStorage.run(event, () => + syncUserMeasurements("user-1"), + ); + + expect(result).toEqual({ imported: 1, failed: false }); + expect(prisma.measurement.create).toHaveBeenCalledTimes(1); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band reading and annotates the drop", async () => { + vi.mocked(fetchMeasurements).mockResolvedValue([ + { + type: "PULSE", + value: 112, + measuredAt: new Date("2026-05-16T08:00:00Z"), + }, + { + type: "PULSE", + value: 55643821.505, + measuredAt: new Date("2026-05-16T08:05:00Z"), + }, + ] as never); + + const result = await eventStorage.run(event, () => + syncUserMeasurements("user-1"), + ); + + // Only the real reading landed, and the run is still a clean success: one + // impossible sample must not strand the rest behind a held watermark. + expect(result).toEqual({ imported: 1, failed: false }); + expect(prisma.measurement.create).toHaveBeenCalledTimes(1); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { source: "withings", type: "PULSE", direction: "above_max", count: 1 }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); + + it("does not overwrite a stored reading with an out-of-band one", async () => { + // The slot already holds a real pulse; the provider re-sends the same + // instant carrying a decode slip. Ungated, the update branch rewrote the + // good value with the bad one. + vi.mocked(prisma.measurement.findFirst).mockResolvedValue({ + id: "existing-row", + } as never); + vi.mocked(fetchMeasurements).mockResolvedValue([ + { + type: "PULSE", + value: 55643821.505, + measuredAt: new Date("2026-05-16T08:00:00Z"), + }, + ] as never); + + const result = await eventStorage.run(event, () => + syncUserMeasurements("user-1"), + ); + + expect(result).toEqual({ imported: 0, failed: false }); + expect(prisma.measurement.update).not.toHaveBeenCalled(); + expect(tally()?.total).toBe(1); + }); +}); + +describe("syncUserActivity — plausibility gate", () => { + it("plans a daily total inside the metric's declared band", async () => { + installActivityFetch([{ date: "2026-05-16", steps: 8_400 }]); + + const imported = await eventStorage.run(event, () => + syncUserActivity("user-1"), + ); + + expect(imported).toBe(1); + expect(prisma.measurement.createMany).toHaveBeenCalledTimes(1); + expect(tally()).toBeUndefined(); + }); + + it("refuses an out-of-band daily total and annotates the drop", async () => { + // ACTIVITY_STEPS is declared 0–200 000. + installActivityFetch([ + { date: "2026-05-16", steps: 8_400 }, + { date: "2026-05-17", steps: 9_900_000 }, + ]); + + const imported = await eventStorage.run(event, () => + syncUserActivity("user-1"), + ); + + expect(imported).toBe(1); + const created = vi.mocked(prisma.measurement.createMany).mock.calls[0]?.[0] + ?.data as Array<{ value: number }>; + expect(created).toHaveLength(1); + expect(created[0]!.value).toBe(8_400); + expect(tally()).toEqual({ + total: 1, + buckets: [ + { + source: "withings", + type: "ACTIVITY_STEPS", + direction: "above_max", + count: 1, + }, + ], + }); + expect(event.getLevel()).toBe("warn"); + }); +}); diff --git a/src/lib/withings/sync-activity.ts b/src/lib/withings/sync-activity.ts index 869fb1d18..a4a0098b2 100644 --- a/src/lib/withings/sync-activity.ts +++ b/src/lib/withings/sync-activity.ts @@ -39,6 +39,7 @@ import { prisma } from "@/lib/db"; import { getEvent } from "@/lib/logging/context"; import { safeFetch } from "@/lib/safe-fetch"; import { getUnitForType } from "@/lib/validations/measurement"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { collapseToTypeDayKeys, recomputeBucketsForMeasurement, @@ -342,7 +343,7 @@ export async function syncUserActivity( value: number; externalId: string; } - const planned: PlannedRow[] = []; + const collected: PlannedRow[] = []; for (const entry of entries) { if (!entry.date) continue; const measuredAt = activityMeasuredAt(entry.date); @@ -352,7 +353,7 @@ export async function syncUserActivity( // valid (a day of rest still records 0 steps). Only skip // undefined / null / NaN. if (raw == null || !Number.isFinite(raw)) continue; - planned.push({ + collected.push({ type, measuredAt, value: raw, @@ -361,6 +362,15 @@ export async function syncUserActivity( } } + // A daily-activity total the app's own band calls impossible (a step count + // past 200 000, a negative distance) is a provider artefact, not a day. Drop + // it before the plan is read against the table, so it can neither insert a + // fresh impossible slot nor overwrite a good stored one. + const planned = dropImplausibleMeasurements("withings", collected, (p) => ({ + type: p.type, + value: p.value, + })); + if (planned.length > 0) { const slotKey = (type: MeasurementType, measuredAt: Date) => `${type}|${measuredAt.getTime()}`; diff --git a/src/lib/withings/sync-sleep.ts b/src/lib/withings/sync-sleep.ts index 840b5d8d3..65099ea7c 100644 --- a/src/lib/withings/sync-sleep.ts +++ b/src/lib/withings/sync-sleep.ts @@ -474,6 +474,9 @@ export async function syncUserSleep( if (verdict.status === "failed") { throw new MeasurementReconciliationError(verdict); } + // Refused by the plausibility gate: nothing was written and nothing + // was touched. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); } @@ -542,6 +545,9 @@ export async function syncUserSleep( if (verdict.status === "failed") { throw new MeasurementReconciliationError(verdict); } + // Refused by the plausibility gate: nothing was written and + // nothing was touched. The drop is already on the wide event. + if (verdict.status === "rejected_range") continue; for (const dirty of verdict.dirtyIdentities ?? []) { touched.push(dirty); } diff --git a/src/lib/withings/sync.ts b/src/lib/withings/sync.ts index fcee1cc7c..d500a030c 100644 --- a/src/lib/withings/sync.ts +++ b/src/lib/withings/sync.ts @@ -15,6 +15,7 @@ import { getUserWithingsCredentials } from "./credentials"; import { annotate, getEvent } from "@/lib/logging/context"; import { isP2002 } from "@/lib/prisma-errors"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; +import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { isReauthRequired, recordSyncFailure, @@ -229,14 +230,24 @@ export async function syncUserMeasurements( ) : new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days back - let measures: Awaited>; + let fetched: Awaited>; try { - measures = await fetchMeasurements(tokenInfo.accessToken, startDate); + fetched = await fetchMeasurements(tokenInfo.accessToken, startDate); } catch (err) { await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_MEASURES); throw err; } + // The app's own plausibility band applies to a provider reading exactly as + // it applies to a typed one. A remote that reports a pulse in the millions + // is reporting a decode slip, not a heartbeat, and the row would poison + // every mean and personal band derived from it. Refuse it here, before the + // update branch below can overwrite a good stored value with it. + const measures = dropImplausibleMeasurements("withings", fetched, (m) => ({ + type: m.type, + value: m.value, + })); + let imported = 0; // v1.4.39.1 — track every (type, measuredAt) we touched so the // persistent rollup tier can be re-folded at the end of the sync. From c2f67c96d4b48e6a241534cebc3fb2ca1a8f3525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 7 Aug 2026 23:58:34 +0200 Subject: [PATCH 7/9] fix(deps): float nanoid past an infinite-loop flaw A fresh advisory names nanoid below 3.3.17: a custom generator can loop forever when asked for a zero-size id. It reaches the tree only through next and postcss, but the audit gate rightly blocks on a known HIGH with a fix available. Bounded override in the established pattern; the audit drops from one high to none. --- pnpm-lock.yaml | 20 +++++++------------- pnpm-workspace.yaml | 1 + 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 92dc14bec..4f5efaea5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,7 @@ settings: overrides: dompurify@<=3.4.10: ^3.4.11 postcss@<8.5.10: ^8.5.10 + nanoid@<3.3.17: ^3.3.17 '@hono/node-server@<1.19.13': ^1.19.13 hono@<4.12.25: ^4.12.25 picomatch@<4.0.4: ^4.0.4 @@ -5556,13 +5557,8 @@ packages: nan@2.27.0: resolution: {integrity: sha512-hC+0LidcL3XE4rp1C4H54KujgXKzbfyTngZTwBByQxsOxCEKZT0MPQ4hOKUH2jU1OYstqdDH4onyHPDzcV0XdQ==} - nanoid@3.3.12: - resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} - engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true - - nanoid@3.3.16: - resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + nanoid@3.3.17: + resolution: {integrity: sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -12636,9 +12632,7 @@ snapshots: nan@2.27.0: optional: true - nanoid@3.3.12: {} - - nanoid@3.3.16: {} + nanoid@3.3.17: {} napi-postinstall@0.3.4: {} @@ -13045,19 +13039,19 @@ snapshots: postcss@8.5.14: dependencies: - nanoid: 3.3.12 + nanoid: 3.3.17 picocolors: 1.1.1 source-map-js: 1.2.1 postcss@8.5.20: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.17 picocolors: 1.1.1 source-map-js: 1.2.1 postcss@8.5.25: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.17 picocolors: 1.1.1 source-map-js: 1.2.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c7fabca7c..3fa8ec2d3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,7 @@ overrides: "dompurify@<=3.4.10": "^3.4.11" "postcss@<8.5.10": "^8.5.10" + "nanoid@<3.3.17": "^3.3.17" "@hono/node-server@<1.19.13": "^1.19.13" "hono@<4.12.25": "^4.12.25" "picomatch@<4.0.4": "^4.0.4" From 4bbf4c6d1050a8bb0d9c752b2b661735fecd0ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 8 Aug 2026 00:38:51 +0200 Subject: [PATCH 8/9] ci: three integration shards, not two Two shards cleared in about fourteen minutes on a quiet day, but with six pull requests running side by side the larger shard hit the same 25-minute guard the split was meant to escape. Three shards keep the biggest slice around sixty files, which clears the guard with real headroom on the slowest runner observed. The gate job is unchanged. --- .github/workflows/integration.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ceb9557f5..a4add73d3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,21 +10,21 @@ permissions: contents: read jobs: - # The suite outgrew a single runner: 128 files took ~13 minutes when the - # 25-minute guard was written, and the v1.37 sharing work took it to 173 - # files and past 25 minutes of wall clock, so every run died at the timeout - # with every test to that point green. The guard's own rule applies — split, - # don't raise — so the files run as two vitest shards. Each shard boots its - # own Postgres testcontainer; the timeout stays a hang detector, sized so a - # healthy half-suite clears it on a slow runner. + # The suite outgrew a single runner, and then a half-suite outgrew a slow + # one: two shards cleared in ~14 minutes on a quiet day, but with six pull + # requests running side by side the larger shard hit the same 25-minute + # guard the split was meant to escape. Three shards keep the biggest slice + # around 60 files, which clears the guard with real headroom even on the + # slowest runner observed. Each shard boots its own Postgres testcontainer; + # the timeout stays a hang detector, not a performance budget. shard: runs-on: ubuntu-latest timeout-minutes: 25 strategy: fail-fast: false matrix: - shard: [1, 2] - name: integration shard ${{ matrix.shard }}/2 + shard: [1, 2, 3] + name: integration shard ${{ matrix.shard }}/3 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -44,7 +44,7 @@ jobs: # vitest a stray positional argument and the shard flag never applies — # both "shards" then run the full suite. Proven by a paired local run # before this workflow shipped. - - run: pnpm exec vitest run --config vitest.integration.config.mts --shard=${{ matrix.shard }}/2 + - run: pnpm exec vitest run --config vitest.integration.config.mts --shard=${{ matrix.shard }}/3 env: # The integration suite boots a Postgres testcontainer and points # Prisma at it via DATABASE_URL set inside `tests/integration/setup.ts`. From 0849d59ecf445b4b1fed4d6e1b747919cdf601f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 8 Aug 2026 01:02:08 +0200 Subject: [PATCH 9/9] test(cycle): pin the fixture clock to the suite's clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two cycle suites compute their relative day keys with local Date arithmetic under TZ=UTC while the routes resolve today in the user's zone, falling back to Europe/Berlin when the fixture sets none. For two hours every evening the two clocks disagree about the date and every day-of-cycle assertion is off by one — the suite was red from 22:00 UTC to midnight and green the rest of the day. The fixtures now carry an explicit UTC timezone so the test's clock and the route's clock are the same clock at any hour. Reproduced red at 23:07 UTC, green with the pin at 23:19 UTC. --- tests/integration/cycle-flow-opens-cycle.test.ts | 8 ++++++++ tests/integration/cycle-routes.test.ts | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/tests/integration/cycle-flow-opens-cycle.test.ts b/tests/integration/cycle-flow-opens-cycle.test.ts index 5e81fda56..7a9fe7d84 100644 --- a/tests/integration/cycle-flow-opens-cycle.test.ts +++ b/tests/integration/cycle-flow-opens-cycle.test.ts @@ -70,6 +70,14 @@ beforeEach(async () => { username: "cycle-flow", email: "cycle-flow@example.test", gender: "FEMALE", + // The suite runs under TZ=UTC and computes its relative day keys with + // local Date arithmetic, i.e. in UTC. The route resolves "today" in the + // USER's zone and falls back to Europe/Berlin when none is set - one + // hour past 22:00 UTC the two clocks disagree about the date and every + // day-of-cycle assertion is off by one. Pinning the fixture zone to UTC + // makes the test's clock and the route's clock the same clock at any + // hour of the day. + timezone: "UTC", }, }); await loginAs(USER_ID); diff --git a/tests/integration/cycle-routes.test.ts b/tests/integration/cycle-routes.test.ts index 8dc05e92b..d75c8822f 100644 --- a/tests/integration/cycle-routes.test.ts +++ b/tests/integration/cycle-routes.test.ts @@ -69,6 +69,14 @@ beforeEach(async () => { username: "cycle-female", email: "cycle-female@example.test", gender: "FEMALE", + // The suite runs under TZ=UTC and computes its relative day keys with + // local Date arithmetic, i.e. in UTC. The route resolves "today" in the + // USER's zone and falls back to Europe/Berlin when none is set - one + // hour past 22:00 UTC the two clocks disagree about the date and every + // day-of-cycle assertion is off by one. Pinning the fixture zone to UTC + // makes the test's clock and the route's clock the same clock at any + // hour of the day. + timezone: "UTC", }, }); await prisma.user.create({ @@ -77,6 +85,7 @@ beforeEach(async () => { username: "cycle-male", email: "cycle-male@example.test", gender: "MALE", + timezone: "UTC", }, }); });