From 74dedee645233f144cd79390c1c11c92a2299787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 07:13:37 +0200 Subject: [PATCH 1/6] fix(import): write Apple Health spot rows without Prisma's per-call heap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each `createManyAndReturn` flush of the export import left about 5 MB of heap behind until some two dozen calls had piled up. A 300 000-record synthetic import under a 524 MB heap settled at 53 MB retained; a run of back-to-back 600-row calls with nothing between them reached 212 MB. Spot rows now go through the column-array `INSERT … SELECT FROM unnest` the restore already uses, with `ON CONFLICT DO NOTHING RETURNING`, which is what `createMany({ skipDuplicates })` emits. The same import holds 11.5 MB and finishes in 35 s instead of 55 s. Ids are minted in the cuid() shape, defaults are written as the client would write them, and integration tests compare the returned rows and the resulting table with `createManyAndReturn` for fresh rows, conflicts on either unique identity, and duplicates inside one call. --- ...asurement-write-plausibility-guard.test.ts | 36 +++- src/lib/export/measurement-bulk-insert.ts | 160 ++++++++++++++++- .../import-apple-health-export.test.ts | 56 ++++++ .../import-large-export-memory.test.ts | 19 ++ .../import-apple-health-export.ts | 75 ++++---- .../source-priority-whoop-applehealth.test.ts | 7 +- .../measurement-bulk-insert.test.ts | 164 ++++++++++++++++++ 7 files changed, 472 insertions(+), 45 deletions(-) diff --git a/src/__tests__/measurement-write-plausibility-guard.test.ts b/src/__tests__/measurement-write-plausibility-guard.test.ts index 88fbafee2..cc4428f9f 100644 --- a/src/__tests__/measurement-write-plausibility-guard.test.ts +++ b/src/__tests__/measurement-write-plausibility-guard.test.ts @@ -130,6 +130,22 @@ const WRITERS: Record = { const MEASUREMENT_WRITE = /[A-Za-z_$][\w$]*\s*\.\s*measurement\s*\.\s*(create|createMany|createManyAndReturn|upsert)\s*\(/; +/** + * A call into the column-array bulk insert, which writes around the Prisma + * delegate and so never matches {@link MEASUREMENT_WRITE}. The Apple export + * importer writes its spot rows only this way, and was invisible to this + * guard for as long as nothing looked for it. The definitions themselves are + * excluded by the lookbehind; the module that holds them is pinned below as + * the one place raw SQL may insert a measurement. + */ +const BULK_INSERT_CALL = /(? !p.startsWith("generated/")) @@ -143,7 +159,10 @@ function read(rel: string): string { } function discoveredWriters(): string[] { - return sourceFiles().filter((rel) => MEASUREMENT_WRITE.test(read(rel))); + return sourceFiles().filter((rel) => { + const text = read(rel); + return MEASUREMENT_WRITE.test(text) || BULK_INSERT_CALL.test(text); + }); } describe("every measurement writer declares how it meets the plausibility domain", () => { @@ -158,6 +177,21 @@ describe("every measurement writer declares how it meets the plausibility domain expect(discoveredWriters()).toEqual(Object.keys(WRITERS).sort()); }); + it("sees a writer that only calls the bulk insert", () => { + // The importer has no Prisma measurement write left; if the bulk-insert + // matcher stopped matching, it would drop out of the sweep silently. + const importer = "lib/measurements/import-apple-health-export.ts"; + expect(MEASUREMENT_WRITE.test(read(importer))).toBe(false); + expect(discoveredWriters()).toContain(importer); + }); + + it("raw SQL inserts measurements in the bulk-insert module and nowhere else", () => { + const raw = sourceFiles().filter((rel) => + RAW_MEASUREMENT_INSERT.test(read(rel)), + ); + expect(raw).toEqual([BULK_INSERT_MODULE]); + }); + it("every provider-fed writer reaches the gate module", () => { const gated = Object.entries(WRITERS) .filter(([, d]) => d.kind === "gate") diff --git a/src/lib/export/measurement-bulk-insert.ts b/src/lib/export/measurement-bulk-insert.ts index f1db771d9..4d0e7c21f 100644 --- a/src/lib/export/measurement-bulk-insert.ts +++ b/src/lib/export/measurement-bulk-insert.ts @@ -10,13 +10,28 @@ * `INSERT … SELECT FROM unnest(…)` with an array per column is a single small * statement whatever the batch size, and about four times faster. * + * The Apple Health export import writes its spot rows through the same + * statement (`insertNewMeasurementRows`), for the same reason. There the cost + * was per call rather than per shape: every `createManyAndReturn` of a + * 650-row flush left about 5 MB behind until roughly two dozen calls had + * accumulated, so a 300 000-record import settled at about 53 MB of retained + * heap and a run of back-to-back flushes reached well over 100 MB. Through + * this statement the same import holds a few megabytes. + * * `COLUMNS` maps every field of `MeasurementInsertRow` to its column and * type. It is typed as a complete record over the row's keys, so a field * added to the row without a column here does not compile, and the restore * builds its rows as `MeasurementInsertRow`, so a field it starts writing has * to be added to the row first. */ -import type { Prisma } from "@/generated/prisma/client"; +import { randomInt } from "node:crypto"; +import { hostname } from "node:os"; + +import type { + MeasurementType, + Prisma, + PrismaClient, +} from "@/generated/prisma/client"; /** One measurement as the restore writes it. */ export interface MeasurementInsertRow { @@ -134,3 +149,146 @@ export async function insertMeasurementRows( }); return tx.$executeRawUnsafe(MEASUREMENT_BULK_INSERT_SQL, ...arrays); } + +/** + * The same statement, skipping any row that collides with an existing one on + * either unique identity, and answering with the rows it did insert. + * + * `ON CONFLICT DO NOTHING` with no target is what Prisma emits for + * `createMany({ skipDuplicates: true })`: a row that meets either + * `(user_id, type, source, external_id)` or + * `(user_id, type, measured_at, source, sleep_stage)` is left out, a row that + * meets another row of the same statement is left out after the first, and + * nothing throws for either. + */ +export const MEASUREMENT_BULK_INSERT_SKIP_DUPLICATES_SQL = `${MEASUREMENT_BULK_INSERT_SQL} + ON CONFLICT DO NOTHING + RETURNING id, type::text AS "type", measured_at AS "measuredAt", external_id AS "externalId" +`; + +/** One row `insertNewMeasurementRows` wrote. */ +export interface InsertedMeasurementRow { + id: string; + type: MeasurementType; + measuredAt: Date; + externalId: string | null; +} + +/** A row for {@link insertNewMeasurementRows}: the id and the defaults are filled in. */ +export type NewMeasurementRow = Omit< + MeasurementInsertRow, + | "id" + | "valueMin" + | "valueMax" + | "notes" + | "notesEncrypted" + | "aggregationProvenance" + | "glucoseContext" + | "rhythmClassification" + | "syncVersion" + | "deletedAt" + | "createdAt" + | "updatedAt" +> & + Partial< + Pick< + MeasurementInsertRow, + | "valueMin" + | "valueMax" + | "notesEncrypted" + | "aggregationProvenance" + | "glucoseContext" + | "rhythmClassification" + > + >; + +/** + * Insert brand-new rows in one statement, skipping duplicates, and return the + * rows that landed. The replacement for + * `measurement.createManyAndReturn({ data, skipDuplicates: true })`. + * + * Every column the schema defaults is written with that default: a fresh + * cuid-shaped id, `sync_version` 1, not deleted, created and updated at the + * moment of the call, and null for every optional column the row does not + * carry. + */ +export async function insertNewMeasurementRows( + db: Pick | Prisma.TransactionClient, + rows: readonly NewMeasurementRow[], +): Promise { + if (rows.length === 0) return []; + // Stamped here, as the Prisma client stamps `@default(now())` and + // `@updatedAt`, rather than left to the database clock. + const now = new Date(); + const full: MeasurementInsertRow[] = rows.map((row) => ({ + id: newMeasurementId(), + userId: row.userId, + type: row.type, + value: row.value, + valueMin: row.valueMin ?? null, + valueMax: row.valueMax ?? null, + unit: row.unit, + source: row.source, + measuredAt: row.measuredAt, + notes: null, + notesEncrypted: row.notesEncrypted ?? null, + externalId: row.externalId, + externalSourceVersion: row.externalSourceVersion, + aggregationProvenance: row.aggregationProvenance ?? null, + glucoseContext: row.glucoseContext ?? null, + sleepStage: row.sleepStage, + rhythmClassification: row.rhythmClassification ?? null, + deviceType: row.deviceType, + syncVersion: 1, + deletedAt: null, + createdAt: now, + updatedAt: now, + })); + const arrays = FIELDS.map((field) => { + const kind = COLUMNS[field][1]; + return full.map((row) => cell(row[field], kind)); + }); + return db.$queryRawUnsafe( + MEASUREMENT_BULK_INSERT_SKIP_DUPLICATES_SQL, + ...arrays, + ); +} + +// ── Ids ────────────────────────────────────────────────────────────────── +// +// Prisma mints `@default(cuid())` in the client, so a row written around the +// client needs one minted here. Same shape as Prisma's own (cuid v1): `c`, the +// time in base 36, a rolling counter, a host fingerprint, and eight random +// characters, 25 characters in all. Ids stay roughly time-ordered, which the +// list reads rely on only as a tiebreaker. + +const BLOCK = 4; +const BLOCK_SPACE = 36 ** BLOCK; +let counter = randomInt(BLOCK_SPACE); + +function block(n: number): string { + return n.toString(36).padStart(BLOCK, "0").slice(-BLOCK); +} + +const FINGERPRINT = (() => { + const host = hostname(); + let sum = host.length + 36; + for (let i = 0; i < host.length; i++) sum += host.charCodeAt(i); + return ( + (process.pid % 1296).toString(36).padStart(2, "0") + + (sum % 1296).toString(36).padStart(2, "0") + ); +})(); + +/** A fresh id in the shape Prisma's `cuid()` default produces. */ +export function newMeasurementId(): string { + counter = (counter + 1) % BLOCK_SPACE; + return ( + "c" + + Date.now().toString(36).padStart(8, "0") + + block(counter) + + FINGERPRINT + + block(randomInt(BLOCK_SPACE)) + + block(randomInt(BLOCK_SPACE)) + ); +} diff --git a/src/lib/measurements/__tests__/import-apple-health-export.test.ts b/src/lib/measurements/__tests__/import-apple-health-export.test.ts index 607a332b2..17aa4b3c4 100644 --- a/src/lib/measurements/__tests__/import-apple-health-export.test.ts +++ b/src/lib/measurements/__tests__/import-apple-health-export.test.ts @@ -11,6 +11,27 @@ vi.mock("@/lib/daily/morning-refresh-trigger", () => ({ maybeEnqueueMorningRefresh: vi.fn().mockResolvedValue(undefined), })); +// The spot flush writes through one raw `INSERT … ON CONFLICT DO NOTHING +// RETURNING` statement. These fakes model the table in memory, so route that +// statement to the fake's own skip-duplicates insert; the SQL itself is pinned +// against a real Postgres in tests/integration/measurement-bulk-insert.test.ts. +vi.mock("@/lib/export/measurement-bulk-insert", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/export/measurement-bulk-insert") + >()), + insertNewMeasurementRows: vi.fn( + ( + db: { measurement: { createManyAndReturn: (args: unknown) => unknown } }, + rows: unknown[], + ) => + db.measurement.createManyAndReturn({ + data: rows, + skipDuplicates: true, + select: { id: true, type: true, measuredAt: true, externalId: true }, + }), + ), +})); + vi.mock("@/lib/arrivals/emit-shared", () => ({ emitDataArrival: vi.fn().mockResolvedValue(undefined), })); @@ -35,6 +56,7 @@ import { import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { maybeEnqueueMorningRefresh } from "@/lib/daily/morning-refresh-trigger"; import { emitDataArrival } from "@/lib/arrivals/emit-shared"; +import { insertNewMeasurementRows } from "@/lib/export/measurement-bulk-insert"; import { APPLE_HEALTH_TYPE_MAP, CUMULATIVE_HK_TYPES, @@ -731,6 +753,40 @@ describe("streamParseExportXml — cumulative source-day estimates", () => { }); }); +describe("streamParseExportXml — spot-row writer", () => { + it("writes spot rows through the bounded bulk insert, never createManyAndReturn", async () => { + const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-")); + const xmlPath = join(tmp, "export.xml"); + writeFileSync(xmlPath, tinyExportXml()); + const prisma = makeFakePrisma(); + const direct = vi.spyOn(prisma.measurement, "createManyAndReturn"); + const helper = vi.mocked(insertNewMeasurementRows); + helper.mockClear(); + + await streamParseExportXml({ + xmlPath, + userId: "user-1", + userTimezone: "Europe/Berlin", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prisma: prisma as any, + spotBatchSize: 2, + workoutBatchSize: 1, + }); + + expect(helper).toHaveBeenCalled(); + // Every createManyAndReturn the fake saw came through the helper's + // stand-in above; none came from the import itself. + expect(direct).toHaveBeenCalledTimes(helper.mock.calls.length); + const written = helper.mock.calls.flatMap(([, rows]) => [...rows]); + expect(written.map((r) => r.type)).toEqual( + expect.arrayContaining(["WEIGHT", "SLEEP_DURATION"]), + ); + expect(new Set(written.map((r) => r.source))).toEqual( + new Set(["APPLE_HEALTH"]), + ); + }); +}); + describe("streamParseExportXml — end-to-end", () => { it("ingests the tiny fixture into per-type / workout / deferred / unknown buckets", async () => { const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-")); diff --git a/src/lib/measurements/__tests__/import-large-export-memory.test.ts b/src/lib/measurements/__tests__/import-large-export-memory.test.ts index 941843fa6..e3c784041 100644 --- a/src/lib/measurements/__tests__/import-large-export-memory.test.ts +++ b/src/lib/measurements/__tests__/import-large-export-memory.test.ts @@ -43,6 +43,25 @@ vi.mock("@/lib/daily/morning-refresh-trigger", () => ({ maybeEnqueueMorningRefresh: async () => undefined, })); +// The spot flush writes through one raw `INSERT … ON CONFLICT DO NOTHING +// RETURNING` statement. These fakes model the table in memory, so route that +// statement to the fake's own skip-duplicates insert; the SQL itself is pinned +// against a real Postgres in tests/integration/measurement-bulk-insert.test.ts. +vi.mock("@/lib/export/measurement-bulk-insert", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/export/measurement-bulk-insert") + >()), + insertNewMeasurementRows: ( + db: { measurement: { createManyAndReturn: (args: unknown) => unknown } }, + rows: unknown[], + ) => + db.measurement.createManyAndReturn({ + data: rows, + skipDuplicates: true, + select: { id: true, type: true, measuredAt: true, externalId: true }, + }), +})); + vi.mock("@/lib/arrivals/emit-shared", () => ({ emitDataArrival: async () => undefined, })); diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index d175fc5c0..d76027feb 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -41,6 +41,11 @@ import { canonicalDailyTimestamp, } from "@/lib/measurements/drain-per-sample-cumulative"; import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement"; +import { + insertNewMeasurementRows, + type InsertedMeasurementRow, + type NewMeasurementRow, +} from "@/lib/export/measurement-bulk-insert"; import { resolveHkWorkoutSportType } from "@/lib/measurements/hk-workout-activity-type-map"; import { convertHkValue, @@ -309,7 +314,10 @@ export interface StreamParseInput { /** IANA timezone, used to anchor cumulative-type day-keys. */ userTimezone: string; /** Prisma client to flush rows through. */ - prisma: Pick; + prisma: Pick< + PrismaClient, + "measurement" | "workout" | "$transaction" | "$queryRawUnsafe" + >; /** * Live progress hook. Called every `PROGRESS_TICK_RECORDS` records * read, and once on terminal `done`. Best-effort; the parser @@ -457,55 +465,38 @@ export async function streamParseExportXml( type: MeasurementType; measuredAt: Date; }> = []; - const createData: Prisma.MeasurementCreateManyInput[] = chunk.map( - (row) => ({ - userId, - type: row.type, - value: row.value, - unit: row.unit, - source: "APPLE_HEALTH", - measuredAt: row.measuredAt, - externalId: row.externalId, - externalSourceVersion: row.externalSourceVersion, - sleepStage: row.sleepStage ?? null, - deviceType: row.deviceType, - }), - ); + // Written as one `INSERT … SELECT FROM unnest(…) ON CONFLICT DO NOTHING + // RETURNING` rather than `createManyAndReturn`: the Prisma call left about + // 5 MB of heap behind per flush until some two dozen had piled up, which + // is the import's whole budget in a small container. Same semantics — + // duplicates on either unique identity are skipped, never thrown — and the + // same answer: the rows that landed. See `insertNewMeasurementRows`. + const createData: NewMeasurementRow[] = chunk.map((row) => ({ + userId, + type: row.type, + value: row.value, + unit: row.unit, + source: "APPLE_HEALTH", + measuredAt: row.measuredAt, + externalId: row.externalId, + externalSourceVersion: row.externalSourceVersion, + sleepStage: row.sleepStage ?? null, + deviceType: row.deviceType, + })); const insertStartedAt = Date.now(); - let createdRows: Array<{ - id: string; - type: MeasurementType; - measuredAt: Date; - externalId: string | null; - }> = []; + let createdRows: InsertedMeasurementRow[] = []; const failedInsertIndexes = new Set(); try { - createdRows = await prisma.measurement.createManyAndReturn({ - data: createData, - skipDuplicates: true, - select: { - id: true, - type: true, - measuredAt: true, - externalId: true, - }, - }); + createdRows = await insertNewMeasurementRows(prisma, createData); } catch { // Retain the old per-sample failure isolation if an unexpected database // error rejects the bulk statement. Conflicts remain non-errors because - // every retry still uses skipDuplicates. + // every retry still skips duplicates. for (let index = 0; index < createData.length; index += 1) { try { - const created = await prisma.measurement.createManyAndReturn({ - data: [createData[index]], - skipDuplicates: true, - select: { - id: true, - type: true, - measuredAt: true, - externalId: true, - }, - }); + const created = await insertNewMeasurementRows(prisma, [ + createData[index], + ]); createdRows.push(...created); } catch { failedInsertIndexes.add(index); diff --git a/src/lib/sources/__tests__/source-priority-whoop-applehealth.test.ts b/src/lib/sources/__tests__/source-priority-whoop-applehealth.test.ts index aecd29f61..73996d51e 100644 --- a/src/lib/sources/__tests__/source-priority-whoop-applehealth.test.ts +++ b/src/lib/sources/__tests__/source-priority-whoop-applehealth.test.ts @@ -150,8 +150,13 @@ describe("WHOOP-written HealthKit records remain source-agnostic", () => { }); it("persists supported export.xml rows as APPLE_HEALTH, never as a raw writer label", () => { + // Spot rows are built as `NewMeasurementRow`s and written through the + // column-array bulk insert; the source is fixed where each row is built. expect(appleImporterSource).toMatch( - /measurement\.createManyAndReturn\([\s\S]*source:\s*"APPLE_HEALTH"/, + /const createData: NewMeasurementRow\[\] = chunk\.map\(\(row\) => \(\{[^}]*source:\s*"APPLE_HEALTH"/, + ); + expect(appleImporterSource).toMatch( + /insertNewMeasurementRows\(prisma, createData\)/, ); expect(appleImporterSource).not.toMatch( /source:\s*attrs\.(?:sourceName|device)/, diff --git a/tests/integration/measurement-bulk-insert.test.ts b/tests/integration/measurement-bulk-insert.test.ts index c0caae7b1..81379639e 100644 --- a/tests/integration/measurement-bulk-insert.test.ts +++ b/tests/integration/measurement-bulk-insert.test.ts @@ -10,7 +10,10 @@ import { beforeEach, describe, expect, it } from "vitest"; import { getPrismaClient, truncateAllTables } from "./setup"; import { insertMeasurementRows, + insertNewMeasurementRows, + newMeasurementId, type MeasurementInsertRow, + type NewMeasurementRow, } from "@/lib/export/measurement-bulk-insert"; const USER = "user-bulk-insert"; @@ -137,3 +140,164 @@ describe("insertMeasurementRows", () => { ); }); }); + +/** + * `insertNewMeasurementRows` replaced `createManyAndReturn({ skipDuplicates })` + * in the Apple Health export import. Each case below runs the same input + * through both and compares what they return and what they leave in the + * table, so a difference in either direction fails. + */ +describe("insertNewMeasurementRows", () => { + function spot( + externalId: string, + at: string, + overrides: Partial = {}, + ): NewMeasurementRow { + return { + userId: USER, + type: "PULSE", + value: 61, + unit: "bpm", + source: "APPLE_HEALTH", + measuredAt: new Date(at), + externalId, + externalSourceVersion: "17.4", + sleepStage: null, + deviceType: null, + ...overrides, + }; + } + + const SELECT = { + id: true, + type: true, + measuredAt: true, + externalId: true, + } as const; + + /** Run `rows` through Prisma, then through the raw path in a fresh table. */ + async function both(seed: NewMeasurementRow[], rows: NewMeasurementRow[]) { + const prisma = getPrismaClient(); + const run = async ( + write: (r: NewMeasurementRow[]) => Promise< + Array<{ + id: string; + type: string; + measuredAt: Date; + externalId: string | null; + }> + >, + ) => { + await prisma.measurement.deleteMany({ where: { userId: USER } }); + if (seed.length > 0) { + await prisma.measurement.createMany({ data: seed as never }); + } + const returned = await write(rows); + const table = await prisma.measurement.findMany({ + where: { userId: USER }, + orderBy: [{ measuredAt: "asc" }, { externalId: "asc" }], + }); + return { + returned: returned + .map(({ id: _id, ...rest }) => rest) + .sort((a, b) => `${a.externalId}`.localeCompare(`${b.externalId}`)), + table: table.map( + ({ id: _id, createdAt: _c, updatedAt: _u, ...rest }) => rest, + ), + ids: returned.map((r) => r.id), + }; + }; + const viaPrisma = await run((r) => + prisma.measurement.createManyAndReturn({ + data: r as never, + skipDuplicates: true, + select: SELECT, + }), + ); + const viaRaw = await run((r) => insertNewMeasurementRows(prisma, r)); + return { viaPrisma, viaRaw }; + } + + it("returns and stores exactly what createManyAndReturn does", async () => { + const rows = [ + spot("hr-1", "2026-03-01T08:00:00.123Z"), + spot("sleep-1", "2026-03-01T06:00:00.000Z", { + type: "SLEEP_DURATION", + value: 42.5, + unit: "minutes", + sleepStage: "REM", + deviceType: "watch", + }), + spot("spo2-1", "2026-03-01T09:00:00.000Z", { + type: "OXYGEN_SATURATION", + value: 97, + unit: "%", + externalSourceVersion: null, + }), + ]; + const { viaPrisma, viaRaw } = await both([], rows); + expect(viaRaw.returned).toEqual(viaPrisma.returned); + expect(viaRaw.table).toEqual(viaPrisma.table); + expect(viaRaw.returned).toHaveLength(3); + for (const r of viaRaw.returned) expect(r.measuredAt).toBeInstanceOf(Date); + }); + + it("skips a row that meets an existing row on either unique identity", async () => { + const seed = [ + // Same external id as `ext-dup`. + spot("ext-dup", "2026-03-02T08:00:00.000Z", { value: 70 }), + // Same (type, measured_at, source, sleep_stage) as `nat-dup`. + spot("seeded-natural", "2026-03-02T09:00:00.000Z", { value: 71 }), + ]; + const rows = [ + spot("ext-dup", "2026-03-02T10:00:00.000Z", { value: 80 }), + spot("nat-dup", "2026-03-02T09:00:00.000Z", { value: 81 }), + spot("fresh", "2026-03-02T11:00:00.000Z", { value: 82 }), + ]; + const { viaPrisma, viaRaw } = await both(seed, rows); + expect(viaRaw.returned).toEqual(viaPrisma.returned); + expect(viaRaw.table).toEqual(viaPrisma.table); + expect(viaRaw.returned.map((r) => r.externalId)).toEqual(["fresh"]); + }); + + it("keeps the first of two rows in one call that share an identity", async () => { + const rows = [ + spot("twin", "2026-03-03T08:00:00.000Z", { value: 90 }), + spot("twin", "2026-03-03T08:30:00.000Z", { value: 91 }), + spot("other", "2026-03-03T08:00:00.000Z", { value: 92 }), + ]; + const { viaPrisma, viaRaw } = await both([], rows); + expect(viaRaw.returned).toEqual(viaPrisma.returned); + expect(viaRaw.table).toEqual(viaPrisma.table); + expect(viaRaw.returned).toHaveLength(1); + }); + + it("mints ids in the shape the cuid() default produces", async () => { + const prisma = getPrismaClient(); + await prisma.measurement.deleteMany({ where: { userId: USER } }); + const viaPrisma = await prisma.measurement.create({ + data: spot("shape-prisma", "2026-03-04T08:00:00.000Z") as never, + }); + const [viaRaw] = await insertNewMeasurementRows(prisma, [ + spot("shape-raw", "2026-03-04T09:00:00.000Z"), + ]); + const shape = /^c[0-9a-z]{24}$/; + expect(viaPrisma.id).toMatch(shape); + expect(viaRaw!.id).toMatch(shape); + const ids = new Set(Array.from({ length: 10_000 }, newMeasurementId)); + expect(ids.size).toBe(10_000); + const stored = await prisma.measurement.findUniqueOrThrow({ + where: { id: viaRaw!.id }, + }); + expect(stored.syncVersion).toBe(1); + expect(stored.deletedAt).toBeNull(); + expect(Math.abs(Date.now() - stored.createdAt.getTime())).toBeLessThan( + 60_000, + ); + expect(stored.updatedAt.getTime()).toBe(stored.createdAt.getTime()); + }); + + it("answers an empty call without touching the database", async () => { + expect(await insertNewMeasurementRows(getPrismaClient(), [])).toEqual([]); + }); +}); From a7682a9fed7977a4c83e444b2e24b618dd0589bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 07:15:20 +0200 Subject: [PATCH 2/6] fix(measurements): name the batch validation refusal and the band an entry broke The validation 422 from `POST /api/measurements/batch` now carries `meta.errorCode: "measurement.batch.invalid"`, beside the existing `too_large` and `source_not_permitted` codes, so a client can file the rows as permanently refused instead of holding its anchor on a guess. A `value_out_of_range` entry now also carries `convertedValue`, the reading after conversion into the stored unit, and `range` (`min`, `max`, `unit`), the band it was checked against. Both appear only on that skip. A fraction sent where a percent was expected is now visible in the response instead of looking like a sensor glitch. Status codes, the envelope and every existing field are unchanged. --- docs/api/openapi.yaml | 30 ++- .../batch/__tests__/refusal-detail.test.ts | 231 ++++++++++++++++++ src/app/api/measurements/batch/route.ts | 55 ++++- src/lib/openapi/error-codes.ts | 1 + src/lib/openapi/routes/measurements.ts | 18 +- 5 files changed, 320 insertions(+), 15 deletions(-) create mode 100644 src/app/api/measurements/batch/__tests__/refusal-detail.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index b407f67b8..7938a5939 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -3936,9 +3936,11 @@ paths: "401": *a1 "409": *a7 "422": - description: The batch exceeded the 500-entry limit (`measurement.batch.too_large`), failed validation, or + description: "The batch exceeded the 500-entry limit (`measurement.batch.too_large`), failed validation + (`measurement.batch.invalid`, with every issue under `details.issues`), or (`measurement.batch.source_not_permitted`) carried an entry naming `APPLE_HEALTH` under a narrow - `measurements:write` credential. Nothing was written in any of the three cases. + `measurements:write` credential. Nothing was written in any of the three cases, and all three are permanent + for the batch as sent: resending it unchanged is refused the same way." content: application/json: schema: @@ -30284,7 +30286,7 @@ components: labs.update.linkedFieldsImmutable, labs.update.numericExpected, labs.update.qualitativeExpected, labs.update.referenceRangeInvalid, labs.update.sourceReferenceRangeInvalid; managed_profile — managed_profile.guardian.duplicate, managed_profile.guardian.managed_invitee, - managed_profile.guardian.required, managed_profile.not_found; measurement — + managed_profile.guardian.required, managed_profile.not_found; measurement — measurement.batch.invalid, measurement.batch.source_not_permitted, measurement.batch.too_large, measurement.bulk-delete.invalid, measurement.create.invalid, measurement.create.source_not_permitted, measurement.delete.too_large, measurement.duplicate_timestamp, measurement.restore.invalid, measurement.update.server_owned_source; @@ -39736,6 +39738,28 @@ components: not advance the entry cursor; the response is marked `Cache-Control: no-store`." reason: type: string + convertedValue: + description: "Only on a `value_out_of_range` skip (since v1.39.1): the entry's value after conversion into the stored + unit, which is the number the band was checked against. A HealthKit fraction that should have been a percent + shows up here a hundred times off." + type: number + range: + description: "Only on a `value_out_of_range` skip (since v1.39.1): the plausibility band that refused the entry, in the + stored unit. Both edges are inclusive. A skip is permanent for the value as sent; resending it unchanged + skips it again." + type: object + properties: + min: + type: number + max: + type: number + unit: + type: string + required: + - min + - max + - unit + additionalProperties: false required: - index - status diff --git a/src/app/api/measurements/batch/__tests__/refusal-detail.test.ts b/src/app/api/measurements/batch/__tests__/refusal-detail.test.ts new file mode 100644 index 000000000..921cda502 --- /dev/null +++ b/src/app/api/measurements/batch/__tests__/refusal-detail.test.ts @@ -0,0 +1,231 @@ +/** + * `POST /api/measurements/batch` — what a refusal tells the client. + * + * Two gaps the native client reported against v1.39.0: + * + * - The whole-batch validation 422 carried no `meta.errorCode`, so the + * client could not tell a permanent refusal from a transient one and had + * to hold its anchor for several syncs before giving up on the rows. + * The too-large and source refusals already had codes; this one now has + * `measurement.batch.invalid`. + * - A `value_out_of_range` skip said neither what the value became after + * conversion nor which band it broke, so a unit-mapping slip (a fraction + * sent where a percent was expected) looked the same as a sensor glitch. + * The entry now carries `convertedValue` and `range`. + * + * Both are additive: status codes, the envelope and every existing field + * are unchanged. + */ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { NextRequest } from "next/server"; + +vi.mock("@/lib/db", () => ({ + prisma: { + user: { update: vi.fn() }, + measurement: { findMany: vi.fn() }, + $transaction: vi.fn(async (fn: unknown) => { + if (typeof fn === "function") { + return (fn as (tx: unknown) => unknown)(prisma); + } + }), + }, +})); + +vi.mock("@/lib/measurements/reconcile-external-measurement", () => ({ + reconcileExternalMeasurement: vi.fn( + async (_tx: unknown, input: Record) => ({ + status: "inserted", + row: { id: "row-1", ...input }, + }), + ), +})); + +vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); +vi.mock("@/lib/auth/audit", () => ({ + auditLog: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: vi.fn() })); +vi.mock("@/lib/jobs/pr-detection", () => ({ + enqueuePrDetection: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/jobs/reminder-satisfy", () => ({ + enqueueReminderSatisfy: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserMeasurements: vi.fn(), +})); +vi.mock("@/lib/rollups/after-measurement-mutation", () => ({ + afterMeasurementMutation: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/daily/morning-refresh-trigger", () => ({ + maybeEnqueueMorningRefresh: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/arrivals/emit-shared", () => ({ + emitDataArrival: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("next/headers", () => ({ + headers: vi.fn(async () => ({ get: () => null })), + cookies: vi.fn(async () => ({ + get: () => undefined, + set: () => {}, + delete: () => {}, + })), +})); + +import { POST } from "../route"; +import { prisma } from "@/lib/db"; +import { getSession } from "@/lib/auth/session"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement"; + +const SESSION_OK = { + session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, + user: { id: "user-1", username: "testuser", role: "USER" as const }, +}; + +function post(body: unknown) { + return POST( + new NextRequest("http://localhost/api/measurements/batch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ); +} + +const AT = new Date(Date.now() - 3_600_000).toISOString(); + +function entry(overrides: Record = {}) { + return { + hkIdentifier: "HKQuantityTypeIdentifierHeartRate", + value: 64, + unit: "count/min", + startDate: AT, + endDate: AT, + externalId: "uuid-1", + ...overrides, + }; +} + +type EntryResult = { + index: number; + status: string; + reason?: string; + convertedValue?: number; + range?: { min: number; max: number; unit: string }; +}; + +async function entriesFor(entries: unknown[]): Promise { + const res = await post({ entries }); + expect(res.status).toBe(200); + return ((await res.json()) as { data: { entries: EntryResult[] } }).data + .entries; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(checkRateLimit).mockResolvedValue({ + allowed: true, + limit: 60, + remaining: 60, + resetAt: Date.now() + 60_000, + }); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([]); +}); + +describe("POST /api/measurements/batch — validation refusal code", () => { + it("names a malformed batch measurement.batch.invalid and keeps the 422 envelope", async () => { + const res = await post({ entries: [entry({ startDate: "yesterday" })] }); + expect(res.status).toBe(422); + const body = (await res.json()) as { + data: null; + error: string; + details: { issues: unknown[] }; + meta?: { errorCode?: string }; + }; + expect(body.data).toBeNull(); + expect(typeof body.error).toBe("string"); + expect(body.details.issues.length).toBeGreaterThan(0); + expect(body.meta?.errorCode).toBe("measurement.batch.invalid"); + }); + + it("names an empty entries array the same way", async () => { + const res = await post({ entries: [] }); + expect(res.status).toBe(422); + expect( + ((await res.json()) as { meta?: { errorCode?: string } }).meta?.errorCode, + ).toBe("measurement.batch.invalid"); + }); + + it("keeps measurement.batch.too_large for an over-cap batch", async () => { + const entries = Array.from({ length: 501 }, (_, i) => + entry({ externalId: `uuid-${i}` }), + ); + const res = await post({ entries }); + expect(res.status).toBe(422); + expect( + ((await res.json()) as { meta?: { errorCode?: string } }).meta?.errorCode, + ).toBe("measurement.batch.too_large"); + }); +}); + +describe("POST /api/measurements/batch — value_out_of_range detail", () => { + it("reports the converted value and the band it broke, in the stored unit", async () => { + // HealthKit reports body fat as a fraction; 0.9 becomes 90 %, above 80. + const [result] = await entriesFor([ + entry({ + hkIdentifier: "HKQuantityTypeIdentifierBodyFatPercentage", + value: 0.9, + unit: "%", + }), + ]); + expect(result).toEqual({ + index: 0, + status: "skipped", + reason: "value_out_of_range", + convertedValue: 90, + range: { min: 1, max: 80, unit: "%" }, + }); + }); + + it("reports the lower edge too", async () => { + const [result] = await entriesFor([entry({ value: 12 })]); + expect(result).toMatchObject({ + reason: "value_out_of_range", + convertedValue: 12, + range: { min: 20, max: 300, unit: "bpm" }, + }); + }); + + it("carries the same detail when the write-side gate is the one that refuses", async () => { + vi.mocked(reconcileExternalMeasurement).mockResolvedValueOnce({ + status: "rejected_range", + rejection: { type: "PULSE", direction: "above_max" }, + }); + const [result] = await entriesFor([entry({ value: 64 })]); + expect(result).toMatchObject({ + status: "skipped", + reason: "value_out_of_range", + convertedValue: 64, + range: { min: 20, max: 300, unit: "bpm" }, + }); + }); + + it("adds nothing to an entry that landed or was skipped for another reason", async () => { + const results = await entriesFor([ + entry({ externalId: "uuid-ok" }), + entry({ hkIdentifier: "HKQuantityTypeIdentifierNotAThing" }), + ]); + expect(results[0]).toEqual({ index: 0, status: "inserted" }); + expect(results[1]).toEqual({ + index: 1, + status: "skipped", + reason: "unmappable_identifier", + }); + }); +}); diff --git a/src/app/api/measurements/batch/route.ts b/src/app/api/measurements/batch/route.ts index 776e870b7..ffba6897d 100644 --- a/src/app/api/measurements/batch/route.ts +++ b/src/app/api/measurements/batch/route.ts @@ -71,7 +71,11 @@ import { UNSTABLE_EXTERNAL_ID_REASON, type UnstableExternalIdShape, } from "@/lib/validations/external-id"; -import { validateMeasurementRange } from "@/lib/validations/measurement"; +import { + VALUE_RANGES, + getUnitForType, + validateMeasurementRange, +} from "@/lib/validations/measurement"; import { deviceTypeEnum } from "@/lib/validations/source-priority"; import { checkRateLimit } from "@/lib/rate-limit"; import { enqueuePrDetection } from "@/lib/jobs/pr-detection"; @@ -216,6 +220,36 @@ interface EntryResult { index: number; status: EntryStatus; reason?: string; + /** + * Only on a `value_out_of_range` skip: the value after conversion into the + * stored unit, which is what the band was checked against. + */ + convertedValue?: number; + /** Only on a `value_out_of_range` skip: the band applied, in the stored unit. */ + range?: { min: number; max: number; unit: string }; +} + +/** + * The `value_out_of_range` skip, with what a client needs to say which limit + * applied. The value is the entry's own reading after conversion, handed back + * to the account that sent it; a fraction sent where a percent was expected + * shows up here as a converted value a hundred times too small or too large. + */ +function outOfRangeResult( + index: number, + type: string, + convertedValue: number, +): EntryResult { + const band = VALUE_RANGES[type]; + return { + index, + status: "skipped", + reason: "value_out_of_range", + ...(Number.isFinite(convertedValue) ? { convertedValue } : {}), + ...(band + ? { range: { min: band.min, max: band.max, unit: getUnitForType(type) } } + : {}), + }; } export const POST = apiHandler(withIdempotency<[NextRequest]>(postBatch)); @@ -276,10 +310,13 @@ async function postBatch(request: NextRequest): Promise { const parsed = batchPayloadSchema.safeParse(rawBody); if (!parsed.success) { + // The code is what lets a client tell this refusal from a transient one: + // the batch as sent will never be accepted, so resending it is pointless. return apiValidationError( parsed.error.issues[0]?.message ?? "Invalid batch", sanitiseZodIssues(parsed.error.issues), 422, + { errorCode: "measurement.batch.invalid" }, ); } @@ -363,11 +400,7 @@ async function postBatch(request: NextRequest): Promise { // sensor reading shouldn't poison an otherwise clean ingest. const rangeError = validateMeasurementRange(mapped.type, mapped.value); if (rangeError !== null) { - results[index] = { - index, - status: "skipped", - reason: "value_out_of_range", - }; + results[index] = outOfRangeResult(index, mapped.type, mapped.value); continue; } @@ -666,11 +699,11 @@ async function postBatch(request: NextRequest): Promise { // already applied above, so reaching here means the two disagreed — // report it exactly as the earlier guard would have, and let the // wide-event tally record which end refused. - results[p.index] = { - index: p.index, - status: "skipped", - reason: "value_out_of_range", - }; + results[p.index] = outOfRangeResult( + p.index, + p.row.type as string, + p.row.value as number, + ); continue; } diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 1957d5bb0..49d374343 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -198,6 +198,7 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "managed_profile.not_found", ], measurement: [ + "measurement.batch.invalid", "measurement.batch.source_not_permitted", "measurement.batch.too_large", "measurement.bulk-delete.invalid", diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index ffd097081..340673fc4 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -147,6 +147,22 @@ const batchEntryResult = z "`inserted`/`duplicate` — the row landed (advance the cursor). `updated` — a `stats:` aggregate overwrote an existing row. `skipped` — validation no-op; see `reason`. `failed` — retryable database failure that must not advance the entry cursor; the response is marked `Cache-Control: no-store`.", ), reason: z.string().optional(), + convertedValue: z + .number() + .optional() + .describe( + "Only on a `value_out_of_range` skip (since v1.39.1): the entry's value after conversion into the stored unit, which is the number the band was checked against. A HealthKit fraction that should have been a percent shows up here a hundred times off.", + ), + range: z + .object({ + min: z.number(), + max: z.number(), + unit: z.string(), + }) + .optional() + .describe( + "Only on a `value_out_of_range` skip (since v1.39.1): the plausibility band that refused the entry, in the stored unit. Both edges are inclusive. A skip is permanent for the value as sent; resending it unchanged skips it again.", + ), }) .meta({ id: "AppleHealthBatchEntryResult" }); @@ -1011,7 +1027,7 @@ export const measurementPaths: NonNullable = { ...stdResponses, "422": { description: - "The batch exceeded the 500-entry limit (`measurement.batch.too_large`), failed validation, or (`measurement.batch.source_not_permitted`) carried an entry naming `APPLE_HEALTH` under a narrow `measurements:write` credential. Nothing was written in any of the three cases.", + "The batch exceeded the 500-entry limit (`measurement.batch.too_large`), failed validation (`measurement.batch.invalid`, with every issue under `details.issues`), or (`measurement.batch.source_not_permitted`) carried an entry naming `APPLE_HEALTH` under a narrow `measurements:write` credential. Nothing was written in any of the three cases, and all three are permanent for the batch as sent: resending it unchanged is refused the same way.", content: { "application/json": { schema: errorEnvelope } }, }, }, From 0cc73d3c887b26028cd5008d9fed574b7464b18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 07:17:01 +0200 Subject: [PATCH 3/6] fix(ingest): treat upsert_failed as a retry in the nutrient and cycle batches The nutrient batch docs told a client to log and drop an `upsert_failed` entry. The route means the opposite: a write group failed on the server's side and nothing was stored. The docs now say to send the entry again on a later sync, and keep the three validation skips terminal. A retry under the same Idempotency-Key used to replay the cached failure, because the 200 carrying `upsert_failed` was cached like any other. Both the nutrient and the cycle bulk routes now mark that response `no-store`, as the measurement batch already does for `failed`, so the retry writes again. The cycle bulk route already answered its validation refusal with `cycle.bulk.invalid` and an over-cap batch with `cycle.bulk.too_large`. Both are now pinned by tests and named in the route's 422 response. --- docs/api/openapi.yaml | 30 ++++++- .../day-logs/bulk/__tests__/route.test.ts | 79 +++++++++++++++++++ src/app/api/cycle/day-logs/bulk/route.ts | 10 ++- .../nutrients/batch/__tests__/route.test.ts | 41 ++++++++++ src/app/api/nutrients/batch/route.ts | 15 +++- src/lib/openapi/routes/cycle.ts | 12 ++- src/lib/openapi/routes/nutrients.ts | 2 +- src/lib/validations/nutrients.ts | 2 +- 8 files changed, 181 insertions(+), 10 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 7938a5939..eb8909465 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -491,7 +491,15 @@ paths: "401": *a1 "403": *a3 "409": *a7 - "422": *a4 + "422": + description: "The whole batch was refused and nothing was written: more than 500 entries (`meta.errorCode` = + `cycle.bulk.too_large`, no issue list), or the body failed validation (`meta.errorCode` = + `cycle.bulk.invalid`, every issue under `details.issues`). Both are permanent for the batch as sent. A + problem with one entry never refuses the batch; it comes back as that entry's `skipped` status." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" "429": *a5 /api/cycle/period: post: @@ -16810,8 +16818,11 @@ paths: Upsert key is (day, nutrient): a re-post replaces the stored total and reports `updated`. Post the current + previous local day per sync; backfill 30 days on first enable. Max 500 entries; Idempotency-Key supported; rate limit 60/min. Per-entry skips (`unit_mismatch` | `value_out_of_range` | `day_invalid`) are terminal — log and - drop, do not retry the entry. Behind the opt-in `nutrients` module (403 `module.disabled` when off). Query - HealthKit with exactly the listed unit per code (`ug` = microgram; do not convert to IU): + drop, do not retry the entry. `upsert_failed` is not one of them: it is a server-side write failure for the + entry's whole write group, nothing was stored, and the entry should be sent again on a later sync; a response + carrying it is `Cache-Control: no-store` and is not replayed for its `Idempotency-Key`. Behind the opt-in + `nutrients` module (403 `module.disabled` when off). Query HealthKit with exactly the listed unit per code (`ug` + = microgram; do not convert to IU): | code | HealthKit identifier | unit | @@ -30760,6 +30771,12 @@ components: externalId: type: string reason: + description: "On a `skipped` entry. `unstable_external_id` is a verdict on the entry and terminal: resending it + unchanged is skipped again. `constraint` means the write met a database constraint (a key another + write claimed at the same moment, or a reference that no longer exists). `upsert_failed` is a + write failure on the server's side: nothing was stored, the entry should be sent again, and the + response carrying it is `Cache-Control: no-store`, so a retry under the same `Idempotency-Key` + writes again rather than replaying the failure." type: string required: - index @@ -64336,7 +64353,12 @@ components: - status additionalProperties: false description: "Per-entry ingest outcome. skipped reasons: unit_mismatch | value_out_of_range | day_invalid | - upsert_failed. Log and drop a skipped entry — do not retry it." + upsert_failed. The first three are verdicts on the entry itself and terminal: resending it unchanged is skipped + again, so log and drop it. `upsert_failed` is a write failure on the server's side, not a verdict on the entry: + nothing was stored for it, and it should be sent again on a later sync. A write failure takes out the whole + group it was written in (every new day total, or every existing one), so several entries can carry it at once. A + response carrying it is marked `Cache-Control: no-store` and is not kept for `Idempotency-Key` replay, so the + retry writes again rather than replaying the failure." NutrientOverviewEnvelope: type: object properties: diff --git a/src/app/api/cycle/day-logs/bulk/__tests__/route.test.ts b/src/app/api/cycle/day-logs/bulk/__tests__/route.test.ts index d75526e8e..8b59f8fec 100644 --- a/src/app/api/cycle/day-logs/bulk/__tests__/route.test.ts +++ b/src/app/api/cycle/day-logs/bulk/__tests__/route.test.ts @@ -203,3 +203,82 @@ describe("POST /api/cycle/day-logs/bulk — external-id stability floor", () => expect(upsertCycleDayLog).toHaveBeenCalledTimes(1); }); }); + +describe("POST /api/cycle/day-logs/bulk — refusal codes", () => { + /** + * The native client files rows under a permanent-refusal register as soon + * as it can tell the refusal is permanent. It can only tell from the code, + * so both whole-batch 422s are pinned here, with the envelope they ride. + */ + async function refusal(body: unknown) { + const res = await POST(postReq(body)); + return { + status: res.status, + body: (await res.json()) as { + data: null; + error: string; + details?: { issues: unknown[] }; + meta?: { errorCode?: string }; + }, + }; + } + + it("names a malformed batch cycle.bulk.invalid, with every issue listed", async () => { + const { status, body } = await refusal({ + entries: [{ date: "not-a-day" }, { date: "2026-13-40" }], + }); + expect(status).toBe(422); + expect(body.data).toBeNull(); + expect(body.meta?.errorCode).toBe("cycle.bulk.invalid"); + expect(body.details?.issues.length).toBeGreaterThanOrEqual(2); + expect(upsertCycleDayLog).not.toHaveBeenCalled(); + }); + + it("names a batch with no entries array cycle.bulk.invalid", async () => { + const { status, body } = await refusal({}); + expect(status).toBe(422); + expect(body.meta?.errorCode).toBe("cycle.bulk.invalid"); + }); + + it("names an over-cap batch cycle.bulk.too_large", async () => { + const entries = Array.from({ length: 501 }, (_, i) => ({ + date: "2026-01-01", + externalId: `e${i}`, + })); + const { status, body } = await refusal({ entries }); + expect(status).toBe(422); + expect(body.meta?.errorCode).toBe("cycle.bulk.too_large"); + }); +}); + +describe("POST /api/cycle/day-logs/bulk — a failed write is not cached", () => { + it("marks a response carrying upsert_failed no-store and leaves a clean one alone", async () => { + vi.mocked(upsertCycleDayLog).mockRejectedValueOnce(new Error("reset")); + const failed = await POST( + postReq({ + entries: [{ date: "2026-01-01", loggedAt: "2026-01-01T08:00:00.000Z" }], + }), + ); + expect(failed.status).toBe(200); + const body = (await failed.json()) as { + data: { entries: Array<{ status: string; reason?: string }> }; + }; + expect(body.data.entries[0]).toMatchObject({ + status: "skipped", + reason: "upsert_failed", + }); + expect(failed.headers.get("Cache-Control")).toBe("private, no-store"); + + vi.mocked(upsertCycleDayLog).mockResolvedValueOnce({ + id: "d1", + existed: false, + changed: true, + } as never); + const clean = await POST( + postReq({ + entries: [{ date: "2026-01-02", loggedAt: "2026-01-02T08:00:00.000Z" }], + }), + ); + expect(clean.headers.get("Cache-Control")).not.toBe("private, no-store"); + }); +}); diff --git a/src/app/api/cycle/day-logs/bulk/route.ts b/src/app/api/cycle/day-logs/bulk/route.ts index 890bbf240..93c56116b 100644 --- a/src/app/api/cycle/day-logs/bulk/route.ts +++ b/src/app/api/cycle/day-logs/bulk/route.ts @@ -249,7 +249,7 @@ async function postBulk(request: NextRequest): Promise { }, }); - return apiSuccess({ + const response = apiSuccess({ processed: entries.length, inserted, updated, @@ -257,4 +257,12 @@ async function postBulk(request: NextRequest): Promise { skipped, entries: results, }); + // `upsert_failed` is a write that did not happen, not a verdict on the + // entry, and the client sends it again. `no-store` keeps the idempotency + // layer from caching this answer, so a retry under the same key writes + // instead of replaying the failure. + if (results.some((r) => r.reason === "upsert_failed")) { + response.headers.set("Cache-Control", "private, no-store"); + } + return response; } diff --git a/src/app/api/nutrients/batch/__tests__/route.test.ts b/src/app/api/nutrients/batch/__tests__/route.test.ts index 09dba1152..9544efec6 100644 --- a/src/app/api/nutrients/batch/__tests__/route.test.ts +++ b/src/app/api/nutrients/batch/__tests__/route.test.ts @@ -69,6 +69,7 @@ import { checkRateLimit } from "@/lib/rate-limit"; import { requireModuleEnabled } from "@/lib/modules/gate"; import { auditLog } from "@/lib/auth/audit"; import { invalidateUserDashboardSnapshot } from "@/lib/cache/invalidate"; +import { nutrientEntryResultSchema } from "@/lib/validations/nutrients"; const SESSION_OK = { session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, @@ -449,6 +450,46 @@ describe("POST /api/nutrients/batch — write semantics", () => { spy.mockRestore(); }); + it("marks a response carrying upsert_failed no-store, so a retry with the same Idempotency-Key writes again", async () => { + const spy = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.mocked(prisma.nutrientIntakeDay.createMany).mockRejectedValue( + new Error("connection reset"), + ); + const failed = await POST( + postReq({ + entries: [ + { day: recentDay(), nutrient: "zinc", unit: "mg", amount: 10 }, + ], + }), + ); + expect(failed.status).toBe(200); + expect(failed.headers.get("Cache-Control")).toBe("private, no-store"); + spy.mockRestore(); + + vi.mocked(prisma.nutrientIntakeDay.createMany).mockResolvedValue({ + count: 1, + } as never); + const clean = await POST( + postReq({ + entries: [ + { day: recentDay(), nutrient: "zinc", unit: "mg", amount: 10 }, + ], + }), + ); + expect(clean.headers.get("Cache-Control")).not.toBe("private, no-store"); + }); + + it("documents upsert_failed as a transient write failure to retry, and the validation skips as terminal", () => { + const description = String( + (nutrientEntryResultSchema.meta() as { description?: string }) + .description, + ); + expect(description).toMatch(/`upsert_failed` is a write failure/); + expect(description).toMatch(/sent again on a later sync/); + expect(description).not.toMatch(/log and drop a skipped entry/i); + expect(description).toMatch(/unit_mismatch/); + }); + it("a race that drops one createMany row still counts consistently and is observable", async () => { vi.mocked(prisma.nutrientIntakeDay.createMany).mockResolvedValue({ count: 1, diff --git a/src/app/api/nutrients/batch/route.ts b/src/app/api/nutrients/batch/route.ts index cdfadcc57..fb15fc1a3 100644 --- a/src/app/api/nutrients/batch/route.ts +++ b/src/app/api/nutrients/batch/route.ts @@ -18,7 +18,10 @@ * per-row updates (existing pairs), the measurements-batch shape. A write * failure marks its WHOLE group `skipped upsert_failed` rather than * isolating the one entry that would have caused it (unavoidable once the - * write is batched) — the request itself never fails. + * write is batched) — the request itself never fails. That reason is + * transient, unlike the validation skips: the client sends those entries + * again, and the response is marked `no-store` so an idempotent retry is not + * answered from the cache. * * Module gate FIRST: the opt-in `nutrients` module (default off) * refuses ingest with the 403 `module.disabled` envelope, so a phone @@ -360,11 +363,19 @@ async function postBatch(request: NextRequest): Promise { }, }); - return apiSuccess({ + const response = apiSuccess({ processed: entries.length, inserted, updated, skipped, entries: results, }); + // `upsert_failed` is transient: nothing was stored for those entries and the + // client sends them again. `no-store` keeps the idempotency layer from + // caching this answer, so a retry under the same key writes instead of + // replaying the failure. The measurement batch does the same for `failed`. + if (skippedByReason.upsert_failed) { + response.headers.set("Cache-Control", "private, no-store"); + } + return response; } diff --git a/src/lib/openapi/routes/cycle.ts b/src/lib/openapi/routes/cycle.ts index 86306fa4d..a5ebfe264 100644 --- a/src/lib/openapi/routes/cycle.ts +++ b/src/lib/openapi/routes/cycle.ts @@ -420,7 +420,12 @@ const cycleBulkEntryResult = z.object({ status: z.enum(["inserted", "duplicate", "updated", "skipped"]), id: z.string().optional(), externalId: z.string().optional(), - reason: z.string().optional(), + reason: z + .string() + .optional() + .describe( + "On a `skipped` entry. `unstable_external_id` is a verdict on the entry and terminal: resending it unchanged is skipped again. `constraint` means the write met a database constraint (a key another write claimed at the same moment, or a reference that no longer exists). `upsert_failed` is a write failure on the server's side: nothing was stored, the entry should be sent again, and the response carrying it is `Cache-Control: no-store`, so a retry under the same `Idempotency-Key` writes again rather than replaying the failure.", + ), }); const cycleBulkResponse = z.object({ @@ -791,6 +796,11 @@ export const cyclePaths: NonNullable = { }, ...cycleDisabledResponse, ...stdResponses, + "422": { + description: + "The whole batch was refused and nothing was written: more than 500 entries (`meta.errorCode` = `cycle.bulk.too_large`, no issue list), or the body failed validation (`meta.errorCode` = `cycle.bulk.invalid`, every issue under `details.issues`). Both are permanent for the batch as sent. A problem with one entry never refuses the batch; it comes back as that entry's `skipped` status.", + content: { "application/json": { schema: errorEnvelope } }, + }, }, }, }, diff --git a/src/lib/openapi/routes/nutrients.ts b/src/lib/openapi/routes/nutrients.ts index 99f170029..a1256ad3e 100644 --- a/src/lib/openapi/routes/nutrients.ts +++ b/src/lib/openapi/routes/nutrients.ts @@ -56,7 +56,7 @@ export const nutrientPaths: NonNullable = { tags: ["Nutrients"], summary: "Ingest micronutrient day totals (v1.28)", description: - "Bulk upsert of supplement-style daily totals synced from HealthKit — vitamins, minerals, water, caffeine. Day totals only: compute HKStatisticsCollectionQuery cumulative sums day-anchored in the user's current IANA timezone; `day` is yyyy-MM-dd in that timezone (the same rule as the `stats:` day keys). Never post raw samples. Upsert key is (day, nutrient): a re-post replaces the stored total and reports `updated`. Post the current + previous local day per sync; backfill 30 days on first enable. Max 500 entries; Idempotency-Key supported; rate limit 60/min. Per-entry skips (`unit_mismatch` | `value_out_of_range` | `day_invalid`) are terminal — log and drop, do not retry the entry. Behind the opt-in `nutrients` module (403 `module.disabled` when off). Query HealthKit with exactly the listed unit per code (`ug` = microgram; do not convert to IU):\n\n| code | HealthKit identifier | unit |\n| --- | --- | --- |\n" + + "Bulk upsert of supplement-style daily totals synced from HealthKit — vitamins, minerals, water, caffeine. Day totals only: compute HKStatisticsCollectionQuery cumulative sums day-anchored in the user's current IANA timezone; `day` is yyyy-MM-dd in that timezone (the same rule as the `stats:` day keys). Never post raw samples. Upsert key is (day, nutrient): a re-post replaces the stored total and reports `updated`. Post the current + previous local day per sync; backfill 30 days on first enable. Max 500 entries; Idempotency-Key supported; rate limit 60/min. Per-entry skips (`unit_mismatch` | `value_out_of_range` | `day_invalid`) are terminal — log and drop, do not retry the entry. `upsert_failed` is not one of them: it is a server-side write failure for the entry's whole write group, nothing was stored, and the entry should be sent again on a later sync; a response carrying it is `Cache-Control: no-store` and is not replayed for its `Idempotency-Key`. Behind the opt-in `nutrients` module (403 `module.disabled` when off). Query HealthKit with exactly the listed unit per code (`ug` = microgram; do not convert to IU):\n\n| code | HealthKit identifier | unit |\n| --- | --- | --- |\n" + catalogTable, requestBody: { required: true, diff --git a/src/lib/validations/nutrients.ts b/src/lib/validations/nutrients.ts index a48fc76af..e65014ed6 100644 --- a/src/lib/validations/nutrients.ts +++ b/src/lib/validations/nutrients.ts @@ -64,7 +64,7 @@ export const nutrientEntryResultSchema = z .meta({ id: "NutrientEntryResult", description: - "Per-entry ingest outcome. skipped reasons: unit_mismatch | value_out_of_range | day_invalid | upsert_failed. Log and drop a skipped entry — do not retry it.", + "Per-entry ingest outcome. skipped reasons: unit_mismatch | value_out_of_range | day_invalid | upsert_failed. The first three are verdicts on the entry itself and terminal: resending it unchanged is skipped again, so log and drop it. `upsert_failed` is a write failure on the server's side, not a verdict on the entry: nothing was stored for it, and it should be sent again on a later sync. A write failure takes out the whole group it was written in (every new day total, or every existing one), so several entries can carry it at once. A response carrying it is marked `Cache-Control: no-store` and is not kept for `Idempotency-Key` replay, so the retry writes again rather than replaying the failure.", }); export const nutrientBatchResponseSchema = z From be112df972a28e37879e2ae323b182005db8ad0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 07:19:03 +0200 Subject: [PATCH 4/6] fix(profile): report the time zone the server uses on /api/user/profile `/api/auth/me` has reported the resolved zone, the stored one or the instance default when the stored value is unusable, while `/api/user/profile` returned the raw column. A client reading the profile could cut days differently from the server, and sending the value back in a PATCH was refused. Both reads, and the profile PATCH answer, now go through one `resolveStoredTimezone`. The field stays a string and always holds a valid IANA name. The web account editor already reads the resolved zone from `/api/auth/me` and saves through the validating timezone route, so it shows the zone in use and can only write a valid one. --- docs/api/openapi.yaml | 6 ++ src/app/api/auth/me/route.ts | 9 +- .../api/user/profile/__tests__/route.test.ts | 85 +++++++++++++++++++ src/app/api/user/profile/route.ts | 8 +- src/lib/openapi/routes/profile.ts | 12 ++- src/lib/tz/resolver.ts | 15 ++++ 6 files changed, 124 insertions(+), 11 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index eb8909465..2533625aa 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -47119,6 +47119,9 @@ components: - type: "null" timezone: type: string + description: "The IANA zone the server cuts this account's days in, the same value `/api/auth/me` reports: the stored + zone, or the instance default when the stored value is not a usable IANA name (since v1.39.1; before, the + raw stored value). Always a valid IANA name, so sending it back in a PATCH is accepted." timeFormat: type: string enum: @@ -47231,6 +47234,9 @@ components: - type: "null" timezone: type: string + description: "The IANA zone the server cuts this account's days in, the same value `/api/auth/me` reports: the stored + zone, or the instance default when the stored value is not a usable IANA name (since v1.39.1; before, the + raw stored value). Always a valid IANA name, so sending it back in a PATCH is accepted." timeFormat: type: string enum: diff --git a/src/app/api/auth/me/route.ts b/src/app/api/auth/me/route.ts index 0c7ac8244..2dfa2bc58 100644 --- a/src/app/api/auth/me/route.ts +++ b/src/app/api/auth/me/route.ts @@ -49,10 +49,7 @@ import { syncMfaEnrollCookie } from "@/lib/auth/mfa-enrollment"; import { buildAvatarUrl } from "@/lib/avatar"; import { decrypt } from "@/lib/crypto"; import { prisma } from "@/lib/db"; -import { - isValidTimezone, - resolveServerDefaultTimezone, -} from "@/lib/tz/resolver"; +import { resolveStoredTimezone } from "@/lib/tz/resolver"; import { isCycleEnabled } from "@/lib/cycle/gate"; import { resolveModuleMap, @@ -279,9 +276,7 @@ export const GET = apiHandler(async () => { // The zone the server cuts this account's days in: the stored zone, or // the instance default when the stored one is unusable. Clients bucket // by this value, so it must be the resolved one, not the raw column. - timezone: isValidTimezone(user.timezone) - ? user.timezone - : await resolveServerDefaultTimezone(), + timezone: await resolveStoredTimezone(user.timezone), onboardingCompletedAt: user.onboardingCompletedAt, onboardingTourCompleted: user.onboardingTourCompleted, // v1.39 (C1) — the needs-based setup flow, published additively: diff --git a/src/app/api/user/profile/__tests__/route.test.ts b/src/app/api/user/profile/__tests__/route.test.ts index f771581db..03567bd91 100644 --- a/src/app/api/user/profile/__tests__/route.test.ts +++ b/src/app/api/user/profile/__tests__/route.test.ts @@ -4,6 +4,7 @@ import { NextRequest } from "next/server"; vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn(), update: vi.fn() }, + appSettings: { findUnique: vi.fn() }, }, })); @@ -60,6 +61,12 @@ const SESSION_OK = { beforeEach(() => { vi.resetAllMocks(); vi.mocked(resolveModuleMap).mockResolvedValue(MODULE_MAP as never); + // The operator's default zone, which the server falls back to when a + // stored one is unusable. Distinct from the built-in default so a test can + // tell the resolver ran. + vi.mocked(prisma.appSettings.findUnique).mockResolvedValue({ + defaultUserTimezone: "America/Chicago", + } as never); }); const callGet = GET as unknown as (req: NextRequest) => Promise; @@ -438,3 +445,81 @@ describe("PATCH /api/user/profile", () => { ]); }); }); + +/** + * `/api/auth/me` has returned the zone the server actually cuts days in — + * the stored one, or the instance default when the stored value is unusable + * — while this alias returned the raw column. A client reading this endpoint + * could bucket a day differently from the server. Both now go through the + * same resolution. + */ +describe("/api/user/profile — timezone is the resolved zone", () => { + function profileRow(timezone: string | null) { + return { + username: "testuser", + displayName: null, + email: null, + dateOfBirth: null, + gender: null, + heightCm: null, + locale: null, + timezone, + moodReminderEnabled: false, + fullName: null, + insurerName: null, + insurerIkNumber: null, + insuranceNumberEncrypted: null, + }; + } + + async function getZone(stored: string | null): Promise { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue( + profileRow(stored) as never, + ); + const res = await callGet(makeGetReq()); + expect(res.status).toBe(200); + return ((await res.json()) as { data: { timezone: string } }).data.timezone; + } + + it("returns a usable stored zone as stored", async () => { + expect(await getZone("Asia/Tokyo")).toBe("Asia/Tokyo"); + }); + + it("returns the instance default in place of a bare offset", async () => { + expect(await getZone("+05:30")).toBe("America/Chicago"); + }); + + it("returns the instance default in place of a name no runtime knows", async () => { + expect(await getZone("Mars/Olympus_Mons")).toBe("America/Chicago"); + }); + + it("returns the instance default when the row is missing", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue(null); + const res = await callGet(makeGetReq()); + const body = (await res.json()) as { data: { timezone: string } }; + expect(body.data.timezone).toBe("America/Chicago"); + }); + + it("answers a PATCH that leaves an unusable zone untouched with the resolved one", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue(null); + vi.mocked(prisma.user.update).mockResolvedValue({ + ...profileRow("+05:30"), + id: "user-1", + role: "USER", + displayName: "Alex T.", + } as never); + const res = await PATCH( + new NextRequest("http://localhost/api/user/profile", { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ displayName: "Alex T." }), + }), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { data: { timezone: string } }; + expect(body.data.timezone).toBe("America/Chicago"); + }); +}); diff --git a/src/app/api/user/profile/route.ts b/src/app/api/user/profile/route.ts index d6e6f6a71..38e3dd77f 100644 --- a/src/app/api/user/profile/route.ts +++ b/src/app/api/user/profile/route.ts @@ -20,6 +20,7 @@ import { applyProfileUpdate } from "@/lib/auth/profile-update"; import { prisma } from "@/lib/db"; import { decrypt } from "@/lib/crypto"; import { resolveModuleMap } from "@/lib/modules/gate"; +import { resolveStoredTimezone } from "@/lib/tz/resolver"; export const dynamic = "force-dynamic"; @@ -77,7 +78,10 @@ export const GET = apiHandler(async () => { gender: dbUser?.gender ?? null, heightCm: dbUser?.heightCm ?? null, locale: dbUser?.locale ?? null, - timezone: dbUser?.timezone ?? "Europe/Berlin", + // The zone the server cuts this account's days in, exactly as + // `/api/auth/me` reports it: the stored zone, or the instance default + // when the stored value is unusable. Never the raw column. + timezone: await resolveStoredTimezone(dbUser?.timezone), timeFormat: dbUser?.timeFormat ?? "AUTO", dateFormat: dbUser?.dateFormat ?? "AUTO", moodReminderEnabled: dbUser?.moodReminderEnabled ?? false, @@ -118,7 +122,7 @@ export const PATCH = apiHandler(async (request: NextRequest) => { gender: result.user.gender, heightCm: result.user.heightCm, locale: result.user.locale, - timezone: result.user.timezone, + timezone: await resolveStoredTimezone(result.user.timezone), timeFormat: result.user.timeFormat, dateFormat: result.user.dateFormat, moodReminderEnabled: result.user.moodReminderEnabled, diff --git a/src/lib/openapi/routes/profile.ts b/src/lib/openapi/routes/profile.ts index f6b1dba67..b1fa0100d 100644 --- a/src/lib/openapi/routes/profile.ts +++ b/src/lib/openapi/routes/profile.ts @@ -498,7 +498,11 @@ const profileResponse = z gender: z.enum(["MALE", "FEMALE", "OTHER"]).nullable(), heightCm: z.number().nullable(), locale: z.string().nullable(), - timezone: z.string(), + timezone: z + .string() + .describe( + "The IANA zone the server cuts this account's days in, the same value `/api/auth/me` reports: the stored zone, or the instance default when the stored value is not a usable IANA name (since v1.39.1; before, the raw stored value). Always a valid IANA name, so sending it back in a PATCH is accepted.", + ), timeFormat: z .enum(["AUTO", "H12", "H24"]) .describe( @@ -542,7 +546,11 @@ const profileUpdateResponse = z gender: z.enum(["MALE", "FEMALE", "OTHER"]).nullable(), heightCm: z.number().nullable(), locale: z.string().nullable(), - timezone: z.string(), + timezone: z + .string() + .describe( + "The IANA zone the server cuts this account's days in, the same value `/api/auth/me` reports: the stored zone, or the instance default when the stored value is not a usable IANA name (since v1.39.1; before, the raw stored value). Always a valid IANA name, so sending it back in a PATCH is accepted.", + ), timeFormat: z .enum(["AUTO", "H12", "H24"]) .describe( diff --git a/src/lib/tz/resolver.ts b/src/lib/tz/resolver.ts index 1835432d5..6f3034dc5 100644 --- a/src/lib/tz/resolver.ts +++ b/src/lib/tz/resolver.ts @@ -144,6 +144,21 @@ export async function resolveServerDefaultTimezone(): Promise { return tz; } +/** + * The zone the server actually uses for a stored `User.timezone` value: the + * value itself when it is a usable IANA name, the instance default otherwise. + * + * Every response that reports an account's zone goes through this, so a + * client never buckets days by a value the server itself has set aside. + */ +export async function resolveStoredTimezone( + stored: string | null | undefined, +): Promise { + return stored && isValidTimezone(stored) + ? stored + : resolveServerDefaultTimezone(); +} + /** * Resolve the per-user display timezone. Reads `User.timezone`. Falls * back to the server default and then to "Europe/Berlin". Cached for From f443ab6615b530c9805e749e2d18a28a3bd3d81f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 07:31:13 +0200 Subject: [PATCH 5/6] feat(measurements): accept sleep stages by name on POST /api/measurements A bridge such as Tasker or Home Assistant could only send sleep stages through the batch route, in HealthKit's terms: the category identifier, a stage codepoint from 0 to 5, and a value it had to work out in minutes. It can now post one segment to `POST /api/measurements` with `type: "SLEEP_DURATION"`, a `sleepStage` of `IN_BED`, `ASLEEP`, `AWAKE`, `CORE`, `DEEP` or `REM`, and either `startDate` + `endDate` or `measuredAt` (the end) + `value` (minutes). The row is the one the batch route writes for the same segment, and the dedupe is the batch route's: the opposite-source same-reading check, then one reconcile over both unique identities. Without an `externalId` the server derives one from the stage and both instants, so a re-post answers 200 `duplicate`, a corrected start answers 200 `updated`, and a new segment answers 201 `inserted`. The narrow `measurements:write` token is accepted and still writes only MANUAL rows. Every other body on this route behaves as before. Requested in #972. --- docs/api/openapi.yaml | 205 +++++++++- src/app/api/measurements/route.ts | 190 +++++++++ src/lib/openapi/routes/measurements.ts | 44 +- src/lib/validations/sleep-stage-entry.ts | 210 ++++++++++ .../measurements-named-sleep-stage.test.ts | 376 ++++++++++++++++++ 5 files changed, 1020 insertions(+), 5 deletions(-) create mode 100644 src/lib/validations/sleep-stage-entry.ts create mode 100644 tests/integration/measurements-named-sleep-stage.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 2533625aa..ea39acef7 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -3753,6 +3753,12 @@ paths: in request order. Use `/api/measurements/batch` for Apple Health upload streams. + Sleep stages by name (since v1.39.1): an object with `type: SLEEP_DURATION`, a `sleepStage` of `IN_BED`, + `ASLEEP`, `AWAKE`, `CORE`, `DEEP` or `REM`, and the segment as `startDate` + `endDate` or as `measuredAt` (its + end) + `value` (minutes). It is stored and deduplicated exactly as the batch route stores a HealthKit sleep + sample, so a bridge can send stages without the HealthKit codepoints. See `NamedSleepStageEntry`. + + Also reachable with a narrow `measurements:write` Bearer, which is the supported way to push readings from a scale, a watch bridge or a home-automation rule. That credential is confined to its owner's own record — a request carrying the account selector is refused 403 before any sharing grant is read, however the record is @@ -3768,12 +3774,21 @@ paths: schema: anyOf: - $ref: "#/components/schemas/CreateMeasurementRequest" + - $ref: "#/components/schemas/NamedSleepStageEntry" - type: array items: $ref: "#/components/schemas/CreateMeasurementRequest" responses: + "200": + description: "Named sleep-stage body only: the segment was already stored (`status: duplicate`) or a stored one was + corrected in place (`status: updated`). Every other body answers 201 or an error, as before." + content: + application/json: + schema: + $ref: "#/components/schemas/NamedSleepStageDuplicateResponse" "201": - description: Created. A single resource for the object body; an array of resources, in request order, for the array body. + description: "Created. A single resource for the object body; an array of resources, in request order, for the array + body; for a named sleep-stage body, the stored row with `status: inserted`." content: application/json: schema: @@ -3782,7 +3797,7 @@ paths: "403": *a9 "409": description: A measurement with this data already exists — the `(userId, type, source, externalId)` dedup index rejected - the write. + the write. Never for a named sleep-stage body, which answers a re-post with 200 instead. content: application/json: schema: @@ -22124,6 +22139,61 @@ components: `glucoseContext` stays REQUIRED on a `BLOOD_GLUCOSE` row here: this is the hand-entry surface, where the person taking the reading knows whether it was fasting or after a meal. The bulk ingest paths (CSV import, JSON import, device sync) accept a contextless reading, because a sensor export classifies nothing per sample." + NamedSleepStageEntry: + type: object + properties: + type: + type: string + const: SLEEP_DURATION + sleepStage: + type: string + enum: + - IN_BED + - ASLEEP + - AWAKE + - CORE + - DEEP + - REM + startDate: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + endDate: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + measuredAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + value: + type: number + externalId: + type: string + minLength: 1 + maxLength: 120 + source: + default: MANUAL + type: string + enum: + - MANUAL + - APPLE_HEALTH + deviceType: + anyOf: + - type: string + minLength: 1 + maxLength: 32 + - type: "null" + required: + - type + - sleepStage + description: "One sleep-stage segment with the stage named (since v1.39.1), for a bridge that does not speak HealthKit. + Chosen whenever the object carries a string `sleepStage`. `type` is `SLEEP_DURATION`. The segment is either + `startDate` + `endDate`, or `measuredAt` (its END) + `value` (its length in minutes); a `value` sent beside the + dates must agree with them to within a minute. Stored as the batch route stores a HealthKit sleep sample: value + in minutes, `measuredAt` at the end, the stage in `sleepStage`. Without an `externalId` the server derives one + from the stage and both instants, so sending the same segment again is a `duplicate`, and a segment whose start + moved is `updated` rather than added." AppleHealthBatchRequest: type: object properties: @@ -39588,6 +39658,136 @@ components: - limit - offset additionalProperties: false + NamedSleepStageDuplicateResponse: + type: object + properties: + data: + $ref: "#/components/schemas/NamedSleepStageMeasurement" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + NamedSleepStageMeasurement: + type: object + properties: + id: + type: string + description: Stored row id, or a synthetic key (`day::`, `sleep-seg::`) on the collapsed list + modes. + type: + $ref: "#/components/schemas/MeasurementType" + value: + type: number + unit: + type: string + measuredAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + source: + $ref: "#/components/schemas/MeasurementSource" + notes: + anyOf: + - type: string + - type: "null" + userId: + type: string + valueMin: + description: Bucket MINIMUM — set only on an hourly heart-rate `stats:` row. + anyOf: + - type: number + - type: "null" + valueMax: + description: Bucket MAXIMUM — set only on an hourly heart-rate `stats:` row. + anyOf: + - type: number + - type: "null" + externalId: + description: External-system dedup key; null for manual entries. + anyOf: + - type: string + - type: "null" + externalSourceVersion: + anyOf: + - type: string + - type: "null" + glucoseContext: + description: Set only on BLOOD_GLUCOSE rows, and null there too when the writer recorded no classification (a sensor + export, a device sync). Never invent one. + anyOf: + - type: string + - type: "null" + sleepStage: + description: Set only on SLEEP_DURATION rows. + anyOf: + - type: string + - type: "null" + rhythmClassification: + description: Device classification verdict; set only on EVENT rows. + anyOf: + - type: string + - type: "null" + deviceType: + description: "`watch | band | ring | phone | scale | other | unknown`." + anyOf: + - type: string + - type: "null" + syncVersion: + description: Bumped on every server-side mutation; the last-writer-wins input for cross-device reconciliation. + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + deletedAt: + description: Soft-delete tombstone; null on a live row. + anyOf: + - type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + - type: "null" + createdAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + updatedAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + dayKey: + type: string + sampleCount: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + partial: + description: True on the one bucket whose day was cut off by the read cap, so its sum understates the real total. + type: boolean + status: + type: string + enum: + - inserted + - updated + - duplicate + description: "`inserted` — a new row (HTTP 201). `duplicate` — the segment was already stored, by this route or the + batch route, and nothing changed (HTTP 200). `updated` — a stored segment with the same stage, source and + end was corrected in place, for example a later start (HTTP 200)." + required: + - id + - type + - value + - unit + - measuredAt + - source + - status + additionalProperties: false + description: The stored SLEEP_DURATION row for a named sleep-stage segment, with what the call did to it. CreateMeasurementResponse: type: object properties: @@ -39597,6 +39797,7 @@ components: - type: array items: $ref: "#/components/schemas/MeasurementResource" + - $ref: "#/components/schemas/NamedSleepStageMeasurement" error: type: "null" meta: diff --git a/src/app/api/measurements/route.ts b/src/app/api/measurements/route.ts index a1eb337a1..6e9591cfc 100644 --- a/src/app/api/measurements/route.ts +++ b/src/app/api/measurements/route.ts @@ -54,6 +54,21 @@ import { pickMainNightAndNaps, } from "@/lib/analytics/sleep-night"; import { resolveUserTimezone } from "@/lib/tz/resolver"; +import { + isNamedSleepStageBody, + sleepStageEntrySchema, + toSleepStageSegment, +} from "@/lib/validations/sleep-stage-entry"; +import { + MeasurementReconciliationError, + reconcileExternalMeasurement, +} from "@/lib/measurements/reconcile-external-measurement"; +import { + MEASURED_AT_TOLERANCE_MS, + isMergeableSource, + isSameReadingAcrossSource, + oppositeMergeSource, +} from "@/lib/measurements/cross-source-merge"; import { NextRequest } from "next/server"; import type { MeasurementType, @@ -742,6 +757,13 @@ async function postMeasurement(request: NextRequest) { }); } + // One sleep-stage segment, the stage named rather than coded. A bridge + // (Tasker, Home Assistant) posts here with its narrow token; the row and + // its dedupe are the batch route's. See `sleep-stage-entry.ts`. + if (isNamedSleepStageBody(body)) { + return postNamedSleepStage(request, user.id, body); + } + // Batch mode (array of measurements, e.g. combined BP + Pulse) if (Array.isArray(body)) { const parsed = createBatchMeasurementSchema.safeParse({ @@ -1046,3 +1068,171 @@ async function postMeasurement(request: NextRequest) { return apiSuccess(shapeMeasurementNotes(measurement), 201); } + +/** + * `POST /api/measurements` with a named `sleepStage`: one segment, stored as + * the batch route stores a HealthKit sleep sample. + * + * The same three steps as the batch route, in the same order: a same-reading + * row of the opposite source (MANUAL against APPLE_HEALTH) makes this one a + * duplicate; otherwise `reconcileExternalMeasurement` settles both unique + * identities at once, reporting an exact external-id match as a duplicate + * and adopting a row that only shares the natural identity. So a bridge that + * re-posts the same segment gets one row, whichever path it used first. + * + * `201` with the stored row when it was inserted, `200` when it was already + * there (`duplicate`) or was corrected in place (`updated`). The row carries + * a `status` saying which. + */ +async function postNamedSleepStage( + request: NextRequest, + userId: string, + body: unknown, +): Promise { + const parsed = sleepStageEntrySchema.safeParse(body); + if (!parsed.success) { + annotate({ + action: { name: "measurements.create.sleep-stage.validation-failed" }, + meta: { issue_count: sanitiseZodIssues(parsed.error.issues).length }, + }); + return returnAllZodIssues(parsed.error, 422, { + errorCode: "measurement.create.invalid", + }); + } + + const segment = toSleepStageSegment(parsed.data); + const source = parsed.data.source as MeasurementSource; + const type: MeasurementType = "SLEEP_DURATION"; + + // Cross-source same reading, exactly as the batch route decides it. + if (isMergeableSource(source)) { + const candidates = await prisma.measurement.findMany({ + where: { + userId, + deletedAt: null, + type, + source: oppositeMergeSource(source), + measuredAt: { + gte: new Date(segment.end.getTime() - MEASURED_AT_TOLERANCE_MS), + lte: new Date(segment.end.getTime() + MEASURED_AT_TOLERANCE_MS), + }, + }, + select: { + id: true, + type: true, + source: true, + value: true, + measuredAt: true, + }, + }); + const twin = candidates.find((candidate) => + isSameReadingAcrossSource( + { type, source, value: segment.minutes, measuredAt: segment.end }, + candidate, + ), + ); + if (twin) { + return namedSleepStageResponse(twin.id, "duplicate", { + stage: segment.stage, + source, + crossSource: true, + }); + } + } + + const verdict = await prisma.$transaction((tx) => + reconcileExternalMeasurement( + tx, + { + userId, + type, + value: segment.minutes, + unit: getUnitForType(type), + source, + measuredAt: segment.end, + externalId: segment.externalId, + externalSourceVersion: null, + sleepStage: segment.stage, + deviceType: parsed.data.deviceType ?? null, + }, + { exactExternalMatch: "duplicate" }, + ), + ); + + if (verdict.status === "failed") { + throw new MeasurementReconciliationError(verdict); + } + if (verdict.status === "rejected_range") { + // The schema applies the same band, so this is the two gates disagreeing. + return apiError("Value out of plausible range", 422, { + errorCode: "measurement.create.invalid", + }); + } + + const status = + verdict.status === "inserted" + ? "inserted" + : verdict.status === "duplicate" + ? "duplicate" + : "updated"; + + if (status !== "duplicate") { + await auditLog("measurement.create", { + userId, + ipAddress: getClientIp(request), + details: { + type, + measurementId: verdict.row.id, + sleepStage: segment.stage, + status, + }, + }); + invalidateUserMeasurements(userId, { evict: true }); + fireAndForget(enqueueReminderSatisfy(userId), { + action: "reminder.satisfy.enqueue", + }); + if (status === "inserted") { + void emitInsertedMeasurementArrivals( + userId, + [verdict.row], + "manual", + ).catch(() => {}); + void maybeEnqueueMorningRefresh(userId, [segment.end]).catch(() => {}); + } + await afterMeasurementMutation(userId, [ + { type, measuredAt: segment.end }, + ...(verdict.dirtyIdentities ?? []), + ]); + } + + return namedSleepStageResponse(verdict.row.id, status, { + stage: segment.stage, + source, + crossSource: false, + }); +} + +async function namedSleepStageResponse( + id: string, + status: "inserted" | "updated" | "duplicate", + meta: { stage: string; source: string; crossSource: boolean }, +): Promise { + annotate({ + action: { + name: "measurement.create.sleep-stage", + entity_type: "measurement", + entity_id: id, + }, + meta: { + status, + stage: meta.stage, + source: meta.source, + cross_source_merge: meta.crossSource, + }, + }); + const row = await prisma.measurement.findUniqueOrThrow({ where: { id } }); + return apiSuccess( + { ...shapeMeasurementNotes(row), status }, + status === "inserted" ? 201 : 200, + ); +} diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index 340673fc4..067d61641 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -10,6 +10,7 @@ import type { ZodOpenApiObject } from "zod-openapi"; import { updateMeasurementSchema } from "@/lib/validations/measurement"; import { seriesBatchQuerySchema } from "@/lib/validations/series-batch"; import { deviceTypeEnum } from "@/lib/validations/source-priority"; +import { sleepStageEntrySchema } from "@/lib/validations/sleep-stage-entry"; import { MODULE_DISABLED_DESCRIPTION, createMeasurementSchema, @@ -329,6 +330,21 @@ export const measurementResource = z "Server-shaped measurement row. GET endpoints return the whole stored row with the decrypted note on `notes` and the `notesEncrypted` ciphertext stripped; the collapsed list modes synthesise rows carrying the display fields plus `dayKey` / `sampleCount`.", }); +/** The answer to a named-stage `POST /api/measurements` (since v1.39.1). */ +const namedSleepStageResult = measurementResource + .extend({ + status: z + .enum(["inserted", "updated", "duplicate"]) + .describe( + "`inserted` — a new row (HTTP 201). `duplicate` — the segment was already stored, by this route or the batch route, and nothing changed (HTTP 200). `updated` — a stored segment with the same stage, source and end was corrected in place, for example a later start (HTTP 200).", + ), + }) + .meta({ + id: "NamedSleepStageMeasurement", + description: + "The stored SLEEP_DURATION row for a named sleep-stage segment, with what the call did to it.", + }); + /** * The aggregate list modes (`aggregate=daily` and the coarser grains) return * pre-folded buckets, not stored rows — no `id`, `unit` or `source`. @@ -866,6 +882,7 @@ export const measurementPaths: NonNullable = { summary: "Create one measurement (or a small array)", description: "Single ingest. The body may also be a bare ARRAY of the same objects — the mode the iOS client uses for a combined blood-pressure + pulse or dual-value glucose write — in which case `data` is the array of created rows in request order. Use `/api/measurements/batch` for Apple Health upload streams.\n\n" + + "Sleep stages by name (since v1.39.1): an object with `type: SLEEP_DURATION`, a `sleepStage` of `IN_BED`, `ASLEEP`, `AWAKE`, `CORE`, `DEEP` or `REM`, and the segment as `startDate` + `endDate` or as `measuredAt` (its end) + `value` (minutes). It is stored and deduplicated exactly as the batch route stores a HealthKit sleep sample, so a bridge can send stages without the HealthKit codepoints. See `NamedSleepStageEntry`.\n\n" + "Also reachable with a narrow `measurements:write` Bearer, which is the supported way to push readings from a scale, a watch bridge or a home-automation rule. That credential is confined to its owner's own record — a request carrying the account selector is refused 403 before any sharing grant is read, however the record is actually shared — and to `MANUAL` attribution, so an explicit `APPLE_HEALTH` source is refused 422 (`measurement.create.source_not_permitted`). The GET on this same path, and the PUT and DELETE beside it, name no scope and so refuse it.", parameters: [idempotencyKeyParameter], requestBody: { @@ -874,6 +891,11 @@ export const measurementPaths: NonNullable = { "application/json": { schema: z.union([ createMeasurementSchema, + sleepStageEntrySchema.meta({ + id: "NamedSleepStageEntry", + description: + "One sleep-stage segment with the stage named (since v1.39.1), for a bridge that does not speak HealthKit. Chosen whenever the object carries a string `sleepStage`. `type` is `SLEEP_DURATION`. The segment is either `startDate` + `endDate`, or `measuredAt` (its END) + `value` (its length in minutes); a `value` sent beside the dates must agree with them to within a minute. Stored as the batch route stores a HealthKit sleep sample: value in minutes, `measuredAt` at the end, the stage in `sleepStage`. Without an `externalId` the server derives one from the stage and both instants, so sending the same segment again is a `duplicate`, and a segment whose start moved is `updated` rather than added.", + }), z.array(createMeasurementSchema), ]), }, @@ -884,19 +906,35 @@ export const measurementPaths: NonNullable = { ...recordRefusal(), "201": { description: - "Created. A single resource for the object body; an array of resources, in request order, for the array body.", + "Created. A single resource for the object body; an array of resources, in request order, for the array body; for a named sleep-stage body, the stored row with `status: inserted`.", content: { "application/json": { schema: dataEnvelope( - z.union([measurementResource, z.array(measurementResource)]), + z.union([ + measurementResource, + z.array(measurementResource), + namedSleepStageResult, + ]), "CreateMeasurementResponse", ), }, }, }, + "200": { + description: + "Named sleep-stage body only: the segment was already stored (`status: duplicate`) or a stored one was corrected in place (`status: updated`). Every other body answers 201 or an error, as before.", + content: { + "application/json": { + schema: dataEnvelope( + namedSleepStageResult, + "NamedSleepStageDuplicateResponse", + ), + }, + }, + }, "409": { description: - "A measurement with this data already exists — the `(userId, type, source, externalId)` dedup index rejected the write.", + "A measurement with this data already exists — the `(userId, type, source, externalId)` dedup index rejected the write. Never for a named sleep-stage body, which answers a re-post with 200 instead.", content: { "application/json": { schema: errorEnvelope } }, }, // After the spread: `stdResponses` carries a generic 422 that would diff --git a/src/lib/validations/sleep-stage-entry.ts b/src/lib/validations/sleep-stage-entry.ts new file mode 100644 index 000000000..3949c5390 --- /dev/null +++ b/src/lib/validations/sleep-stage-entry.ts @@ -0,0 +1,210 @@ +/** + * One sleep-stage segment, named rather than coded, for `POST /api/measurements`. + * + * Until v1.39.1 a stage could only arrive through the batch route, in the + * HealthKit dialect: `hkIdentifier: HKCategoryTypeIdentifierSleepAnalysis`, a + * `sleepStage` codepoint 0..5 whose meaning is Apple's, and a value the client + * had to work out in minutes. A bridge such as Tasker or Home Assistant knows + * none of that. This shape lets it say what it has: the stage by name, and the + * segment either as its start and end or as its end and length. + * + * The row that lands is the row the batch route would have written for the + * same segment: `SLEEP_DURATION`, value in minutes, `measuredAt` at the + * segment's END (Apple's convention, which every sleep reader assumes), the + * stage in `sleepStage`. Dedupe is the batch route's too: the + * `(type, source, externalId)` identity and the + * `(type, measuredAt, source, sleepStage)` identity, reconciled in one step. + */ +import { z } from "zod/v4"; + +import { validateEntryInstant } from "./entry-instant"; +import { assertStableExternalId } from "@/lib/validations/external-id"; +import { + validateMeasurementRange, + writableMeasurementSourceEnum, +} from "./measurement"; + +/** The stage names the `SleepStage` column stores, as a caller writes them. */ +export const SLEEP_STAGE_NAMES = [ + "IN_BED", + "ASLEEP", + "AWAKE", + "CORE", + "DEEP", + "REM", +] as const; + +export type SleepStageName = (typeof SLEEP_STAGE_NAMES)[number]; + +/** A difference of up to a minute between `value` and the dates is rounding. */ +const VALUE_TOLERANCE_MINUTES = 1; + +const instant = () => + validateEntryInstant( + z.iso.datetime({ offset: true }).transform((s) => new Date(s)), + ); + +export const sleepStageEntrySchema = z + .object({ + type: z.literal("SLEEP_DURATION"), + sleepStage: z.enum(SLEEP_STAGE_NAMES), + startDate: instant().optional(), + endDate: instant().optional(), + /** The segment's END, when it is sent as an end and a length. */ + measuredAt: instant().optional(), + /** Minutes. Required with `measuredAt`; optional with start and end. */ + value: z.number().finite().optional(), + externalId: z.string().min(1).max(120).optional(), + source: writableMeasurementSourceEnum.optional().default("MANUAL"), + deviceType: z.string().min(1).max(32).nullable().optional(), + }) + .superRefine((entry, ctx) => { + assertStableExternalId(entry, ctx); + + const hasDates = + entry.startDate !== undefined || entry.endDate !== undefined; + const hasEndAndLength = entry.measuredAt !== undefined; + if (hasDates && hasEndAndLength) { + ctx.addIssue({ + code: "custom", + path: ["measuredAt"], + message: + "Send the segment either as startDate and endDate, or as measuredAt (its end) and value (its length in minutes), not both", + }); + return; + } + if (hasDates) { + if (entry.startDate === undefined || entry.endDate === undefined) { + ctx.addIssue({ + code: "custom", + path: [entry.startDate === undefined ? "startDate" : "endDate"], + message: "startDate and endDate are sent together", + }); + return; + } + if (entry.endDate < entry.startDate) { + ctx.addIssue({ + code: "custom", + path: ["endDate"], + message: "endDate must not be before startDate", + }); + return; + } + const minutes = + (entry.endDate.getTime() - entry.startDate.getTime()) / 60_000; + if ( + entry.value !== undefined && + Math.abs(entry.value - minutes) > VALUE_TOLERANCE_MINUTES + ) { + ctx.addIssue({ + code: "custom", + path: ["value"], + message: + "value disagrees with startDate and endDate; send the dates alone, or a value that matches them", + }); + return; + } + if (validateMeasurementRange("SLEEP_DURATION", minutes) !== null) { + ctx.addIssue({ + code: "custom", + path: ["endDate"], + message: "A sleep segment lasts between 0 and 1440 minutes", + }); + } + return; + } + if (hasEndAndLength) { + if (entry.value === undefined) { + ctx.addIssue({ + code: "custom", + path: ["value"], + message: + "value (the segment's length in minutes) is required with measuredAt", + }); + return; + } + if (validateMeasurementRange("SLEEP_DURATION", entry.value) !== null) { + ctx.addIssue({ + code: "custom", + path: ["value"], + message: "A sleep segment lasts between 0 and 1440 minutes", + }); + } + return; + } + ctx.addIssue({ + code: "custom", + path: ["startDate"], + message: + "Send the segment as startDate and endDate, or as measuredAt (its end) and value (its length in minutes)", + }); + }); + +export type SleepStageEntry = z.infer; + +/** A parsed entry, reduced to the segment it describes. */ +export interface SleepStageSegment { + stage: SleepStageName; + start: Date; + end: Date; + minutes: number; + /** + * The caller's id, or one derived from the stage and the two instants, so a + * bridge that sends no id still re-posts onto the same row. + */ + externalId: string; +} + +/** + * The id a segment gets when the caller sends none. The stage and both + * instants, so the same segment sent twice is one row, and a segment whose + * start was corrected keeps its end (and so its natural identity) and is + * updated rather than duplicated. + */ +export function derivedSleepStageExternalId( + stage: SleepStageName, + start: Date, + end: Date, +): string { + return `sleep-stage:${stage}:${start.toISOString()}/${end.toISOString()}`; +} + +export function toSleepStageSegment(entry: SleepStageEntry): SleepStageSegment { + let start: Date; + let end: Date; + let minutes: number; + if (entry.startDate && entry.endDate) { + start = entry.startDate; + end = entry.endDate; + minutes = (end.getTime() - start.getTime()) / 60_000; + } else { + // The refinement guarantees the end-and-length form here. + end = entry.measuredAt!; + minutes = entry.value!; + start = new Date(end.getTime() - Math.round(minutes * 60_000)); + } + return { + stage: entry.sleepStage, + start, + end, + minutes, + externalId: + entry.externalId ?? + derivedSleepStageExternalId(entry.sleepStage, start, end), + }; +} + +/** + * Whether a single-object body asks for the named-stage form. Only a string + * `sleepStage` does. The plain create schema never read the field, so a body + * that carried one before was stored without its stage; it is now stored with + * it, or refused with the reason. + */ +export function isNamedSleepStageBody(body: unknown): boolean { + return ( + typeof body === "object" && + body !== null && + !Array.isArray(body) && + typeof (body as { sleepStage?: unknown }).sleepStage === "string" + ); +} diff --git a/tests/integration/measurements-named-sleep-stage.test.ts b/tests/integration/measurements-named-sleep-stage.test.ts new file mode 100644 index 000000000..b3462f040 --- /dev/null +++ b/tests/integration/measurements-named-sleep-stage.test.ts @@ -0,0 +1,376 @@ +/** + * `POST /api/measurements` with a named `sleepStage`, against real Postgres. + * + * A bridge (Tasker, Home Assistant) sends one segment with the stage by name + * and its narrow `measurements:write` token. What this pins: + * + * - the row is the row the batch route writes for the same segment: + * SLEEP_DURATION, minutes, `measuredAt` at the segment's end, the stage; + * - re-posting the same segment is one row, whether the bridge sends an id + * or not, and whichever of the two routes it used first; + * - a corrected start updates the row instead of adding a second one; + * - the scoped token's source rule still holds: MANUAL, never APPLE_HEALTH; + * - a body the shape does not describe is a 422 with every issue listed. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +process.env.API_TOKEN_HMAC_KEY ??= + "test-hmac-key-named-sleep-stage-ingest-32-bytes-min-0123456789"; + +const { hashToken } = await import("@/lib/auth/hmac"); + +const USER_ID = "user-named-sleep-stage"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const { POST } = await import("@/app/api/measurements/route"); +const { POST: POST_BATCH } = await import("@/app/api/measurements/batch/route"); + +let token = ""; + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + cookieJar.clear(); + headerJar.clear(); + await prisma.user.create({ + data: { + id: USER_ID, + username: "named-sleep", + email: "named-sleep@example.test", + timezone: "UTC", + }, + }); + token = `hlk_sleepstage_${"0".repeat(48)}`; + await prisma.apiToken.create({ + data: { + userId: USER_ID, + name: "bridge", + tokenHash: hashToken(token), + permissions: ["measurements:write"], + }, + }); + headerJar.set("authorization", `Bearer ${token}`); +}); + +function post(body: unknown, url = "https://health.example/api/measurements") { + return new NextRequest(url, { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + }, + body: JSON.stringify(body), + }); +} + +const END = new Date(Date.now() - 6 * 3_600_000); +END.setUTCSeconds(0, 0); +const START = new Date(END.getTime() - 42 * 60_000); + +function rows() { + return getPrismaClient().measurement.findMany({ + where: { userId: USER_ID, type: "SLEEP_DURATION" }, + orderBy: { measuredAt: "asc" }, + }); +} + +type Created = { + data: { + id: string; + status: string; + type: string; + value: number; + unit: string; + sleepStage: string | null; + source: string; + measuredAt: string; + externalId: string | null; + }; +}; + +describe("POST /api/measurements — named sleep stage", () => { + it("stores a start/end segment as the batch route would", async () => { + const res = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + expect(res.status).toBe(201); + const body = (await res.json()) as Created; + expect(body.data).toMatchObject({ + status: "inserted", + type: "SLEEP_DURATION", + value: 42, + unit: "minutes", + sleepStage: "REM", + source: "MANUAL", + measuredAt: END.toISOString(), + }); + const [row] = await rows(); + expect(row).toMatchObject({ + value: 42, + unit: "minutes", + sleepStage: "REM", + source: "MANUAL", + }); + expect(row!.measuredAt.toISOString()).toBe(END.toISOString()); + expect(row!.externalId).toBe( + `sleep-stage:REM:${START.toISOString()}/${END.toISOString()}`, + ); + }); + + it("accepts the end-and-length form and lands on the same row", async () => { + const first = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "DEEP", + measuredAt: END.toISOString(), + value: 42, + }), + ); + expect(first.status).toBe(201); + const second = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "DEEP", + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + expect(second.status).toBe(200); + expect(((await second.json()) as Created).data.status).toBe("duplicate"); + expect(await rows()).toHaveLength(1); + }); + + it("answers a re-post of the same segment with the stored row, not a second one", async () => { + const body = { + type: "SLEEP_DURATION", + sleepStage: "CORE", + startDate: START.toISOString(), + endDate: END.toISOString(), + }; + const first = (await (await POST(post(body))).json()) as Created; + const again = await POST(post(body)); + expect(again.status).toBe(200); + const second = (await again.json()) as Created; + expect(second.data.status).toBe("duplicate"); + expect(second.data.id).toBe(first.data.id); + expect(await rows()).toHaveLength(1); + }); + + it("updates the row when the same segment comes back with a corrected start", async () => { + await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "AWAKE", + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + const corrected = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "AWAKE", + startDate: new Date(START.getTime() + 12 * 60_000).toISOString(), + endDate: END.toISOString(), + }), + ); + expect(corrected.status).toBe(200); + expect(((await corrected.json()) as Created).data.status).toBe("updated"); + const all = await rows(); + expect(all).toHaveLength(1); + expect(all[0]!.value).toBe(30); + }); + + it("keeps two stages that end at the same instant apart", async () => { + for (const sleepStage of ["IN_BED", "ASLEEP"]) { + const res = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage, + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + expect(res.status).toBe(201); + } + expect((await rows()).map((r) => r.sleepStage).sort()).toEqual([ + "ASLEEP", + "IN_BED", + ]); + }); + + it("dedupes against the batch route: same source, same id, one row", async () => { + const externalId = "tasker-night-1-rem"; + const viaBatch = await POST_BATCH( + post( + { + entries: [ + { + hkIdentifier: "HKCategoryTypeIdentifierSleepAnalysis", + value: 42, + unit: "min", + startDate: START.toISOString(), + endDate: END.toISOString(), + sleepStage: 5, + externalId, + }, + ], + }, + "https://health.example/api/measurements/batch", + ), + ); + expect(viaBatch.status).toBe(200); + const viaNamed = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + externalId, + }), + ); + expect(viaNamed.status).toBe(200); + expect(((await viaNamed.json()) as Created).data.status).toBe("duplicate"); + const all = await rows(); + expect(all).toHaveLength(1); + expect(all[0]).toMatchObject({ sleepStage: "REM", source: "MANUAL" }); + }); + + it("treats a same-reading APPLE_HEALTH row as the same segment", async () => { + await getPrismaClient().measurement.create({ + data: { + userId: USER_ID, + type: "SLEEP_DURATION", + value: 42, + unit: "minutes", + source: "APPLE_HEALTH", + measuredAt: END, + sleepStage: "REM", + externalId: "hk-uuid-1", + }, + }); + const res = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + expect(res.status).toBe(200); + expect(((await res.json()) as Created).data.status).toBe("duplicate"); + expect(await rows()).toHaveLength(1); + }); + + it("refuses APPLE_HEALTH from the scoped token", async () => { + const res = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + source: "APPLE_HEALTH", + }), + ); + expect(res.status).toBe(422); + const body = (await res.json()) as { meta?: { errorCode?: string } }; + expect(body.meta?.errorCode).toBe( + "measurement.create.source_not_permitted", + ); + expect(await rows()).toHaveLength(0); + }); + + it("refuses an unknown stage, a missing end, a mismatched value and an unstable id, listing each", async () => { + for (const body of [ + { + type: "SLEEP_DURATION", + sleepStage: "LIGHT", + startDate: START.toISOString(), + endDate: END.toISOString(), + }, + { + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + }, + { + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + value: 90, + }, + { + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + externalId: "0xdeadbeef", + }, + { + type: "WEIGHT", + sleepStage: "REM", + measuredAt: END.toISOString(), + value: 70, + }, + ]) { + const res = await POST(post(body)); + expect(res.status).toBe(422); + const parsed = (await res.json()) as { + details?: { issues: unknown[] }; + meta?: { errorCode?: string }; + }; + expect(parsed.meta?.errorCode).toBe("measurement.create.invalid"); + expect(parsed.details?.issues.length).toBeGreaterThan(0); + } + expect(await rows()).toHaveLength(0); + }); + + it("leaves the plain single create untouched", async () => { + const res = await POST( + post({ + type: "SLEEP_DURATION", + value: 420, + measuredAt: END.toISOString(), + }), + ); + expect(res.status).toBe(201); + const body = (await res.json()) as { + data: Record; + }; + expect(body.data).not.toHaveProperty("status"); + expect(body.data.sleepStage).toBeNull(); + }); +}); From 7e654175b6f04b375654611fd55a20c76a458164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 07:32:18 +0200 Subject: [PATCH 6/6] test(contract): pin the v1.39.1 wire changes as additive for shipped clients Decoders written against the shapes the shipped iPhone builds read still decode every payload this release changes, and the old shapes still satisfy the new document: the new batch-entry fields and the error code stay optional, the profile time zone stays a required string, and the named sleep-stage answer sits beside the old create response rather than in place of it. --- src/__tests__/v1391-additive-contract.test.ts | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 src/__tests__/v1391-additive-contract.test.ts diff --git a/src/__tests__/v1391-additive-contract.test.ts b/src/__tests__/v1391-additive-contract.test.ts new file mode 100644 index 000000000..06ed6b9b6 --- /dev/null +++ b/src/__tests__/v1391-additive-contract.test.ts @@ -0,0 +1,253 @@ +/** + * v1.39.1 — every wire change in this release is additive for the iPhone app + * already in people's hands (1.0.3 in the App Store, 1.0.4 on TestFlight). + * + * What changed on shapes those builds decode: + * + * - `AppleHealthBatchEntryResult` gained `convertedValue` and `range`, on a + * `value_out_of_range` skip only; + * - the batch validation 422 gained `meta.errorCode`; + * - `ProfileResponse.timezone` now holds the resolved zone (same type); + * - the nutrient and cycle batch responses gained a `Cache-Control` header + * on one outcome, and the nutrient entry's description changed. + * + * Two directions are pinned, because "additive" is a claim about both: + * + * 1. A payload the new server sends still decodes with a decoder written + * against the old shape. The decoders below are that: the fields the + * shipped client reads non-optionally, with their types, and nothing + * else — unknown keys are ignored, as Swift's `Decodable` ignores them. + * 2. A payload of the old shape still satisfies the new document: no field + * a shipped client might omit or never see became required. + * + * The document is built from the registry rather than read off disk; + * `openapi:check` already pins the YAML to exactly this build. + * + * ## Mutation check + * + * Drop `.optional()` from `convertedValue` or `range` in the registry and the + * `required` assertion goes red. Rename a field the old decoder reads and the + * decode assertion goes red. + */ +import { describe, expect, it } from "vitest"; +import { z } from "zod/v4"; + +import { buildOpenApiDocument } from "@/lib/openapi/registry"; + +type Schema = { + type?: string | string[]; + properties?: Record; + required?: string[]; + additionalProperties?: boolean | Schema; + $ref?: string; + anyOf?: Schema[]; +}; + +const doc = buildOpenApiDocument() as unknown as { + components: { schemas: Record }; +}; +const schemas = doc.components.schemas; + +function component(name: string): Schema { + const found = schemas[name]; + if (!found) throw new Error(`component ${name} is missing`); + return found; +} + +// ── The shapes as 1.0.3 decodes them ──────────────────────────────────── + +const oldBatchEntry = z.object({ + index: z.number().int(), + status: z.string(), + reason: z.string().optional(), +}); + +const oldBatchResponse = z.object({ + data: z.object({ + processed: z.number(), + inserted: z.number(), + updated: z.number(), + duplicates: z.number(), + failed: z.number(), + skipped: z.array(z.object({ index: z.number(), reason: z.string() })), + entries: z.array(oldBatchEntry), + }), +}); + +const oldErrorEnvelope = z.object({ + data: z.null(), + error: z.string(), +}); + +const oldProfile = z.object({ + data: z.object({ + username: z.string(), + email: z.string().nullable(), + timezone: z.string(), + timeFormat: z.string(), + dateFormat: z.string(), + moodReminderEnabled: z.boolean(), + }), +}); + +const oldNutrientResponse = z.object({ + data: z.object({ + processed: z.number(), + inserted: z.number(), + updated: z.number(), + skipped: z.array(z.object({ index: z.number(), reason: z.string() })), + entries: z.array( + z.object({ + index: z.number(), + status: z.string(), + reason: z.string().optional(), + }), + ), + }), +}); + +// ── Fixtures ──────────────────────────────────────────────────────────── + +/** A v1.39.0 batch answer: no range detail anywhere. */ +const OLD_BATCH = { + data: { + processed: 2, + inserted: 1, + updated: 0, + duplicates: 0, + failed: 0, + skipped: [{ index: 1, reason: "value_out_of_range" }], + entries: [ + { index: 0, status: "inserted" }, + { index: 1, status: "skipped", reason: "value_out_of_range" }, + ], + }, + error: null, +}; + +/** The same batch answered by v1.39.1. */ +const NEW_BATCH = { + data: { + ...OLD_BATCH.data, + entries: [ + { index: 0, status: "inserted" }, + { + index: 1, + status: "skipped", + reason: "value_out_of_range", + convertedValue: 90, + range: { min: 1, max: 80, unit: "%" }, + }, + ], + }, + error: null, +}; + +const NEW_BATCH_422 = { + data: null, + error: "Invalid input: expected string, received undefined", + details: { + issues: [{ path: ["entries", 0, "unit"], message: "Required" }], + }, + meta: { errorCode: "measurement.batch.invalid" }, +}; + +const NEW_PROFILE = { + data: { + username: "someone", + displayName: null, + email: null, + dateOfBirth: null, + gender: null, + heightCm: null, + locale: null, + timezone: "Europe/Berlin", + timeFormat: "AUTO", + dateFormat: "AUTO", + moodReminderEnabled: false, + fullName: null, + insurerName: null, + insurerIkNumber: null, + insuranceNumber: null, + modules: {}, + }, + error: null, +}; + +const NEW_NUTRIENTS = { + data: { + processed: 1, + inserted: 0, + updated: 0, + skipped: [{ index: 0, reason: "upsert_failed" }], + entries: [{ index: 0, status: "skipped", reason: "upsert_failed" }], + }, + error: null, +}; + +describe("v1.39.1 — shipped decoders still decode what the server now sends", () => { + it("the batch answer, with the new range detail", () => { + expect(oldBatchResponse.safeParse(NEW_BATCH).success).toBe(true); + }); + + it("the batch validation 422, with its new code", () => { + expect(oldErrorEnvelope.safeParse(NEW_BATCH_422).success).toBe(true); + }); + + it("the profile, whose timezone is still a string", () => { + expect(oldProfile.safeParse(NEW_PROFILE).success).toBe(true); + }); + + it("the nutrient batch answer carrying upsert_failed", () => { + expect(oldNutrientResponse.safeParse(NEW_NUTRIENTS).success).toBe(true); + }); +}); + +describe("v1.39.1 — the new document still admits the old shapes", () => { + it("adds convertedValue and range to the batch entry as optional fields", () => { + const entry = component("AppleHealthBatchEntryResult"); + expect(entry.properties?.convertedValue?.type).toBe("number"); + expect( + Object.keys(entry.properties?.range?.properties ?? {}).sort(), + ).toEqual(["max", "min", "unit"]); + expect([...(entry.required ?? [])].sort()).toEqual(["index", "status"]); + }); + + it("an old batch entry carries every field the new document requires", () => { + const required = component("AppleHealthBatchEntryResult").required ?? []; + for (const e of OLD_BATCH.data.entries) { + for (const key of required) expect(e).toHaveProperty(key); + } + }); + + it("keeps the error envelope's meta optional", () => { + const envelope = component("ErrorEnvelope"); + expect(envelope.required ?? []).not.toContain("meta"); + expect(envelope.properties?.meta?.required ?? []).not.toContain( + "errorCode", + ); + }); + + it("keeps the profile timezone a required string", () => { + const profile = component("ProfileResponse"); + expect(profile.properties?.timezone?.type).toBe("string"); + expect(profile.required).toContain("timezone"); + }); + + it("keeps the nutrient entry result's shape", () => { + const entry = component("NutrientEntryResult"); + expect(Object.keys(entry.properties ?? {}).sort()).toEqual([ + "index", + "reason", + "status", + ]); + expect([...(entry.required ?? [])].sort()).toEqual(["index", "status"]); + }); + + it("puts the named sleep-stage answer beside the old create response, not in place of it", () => { + const named = component("NamedSleepStageMeasurement"); + expect(named.required).toContain("status"); + const resource = component("MeasurementResource"); + expect(resource.properties).not.toHaveProperty("status"); + }); +});