diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 190b9430e..92f353364 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -491,7 +491,15 @@ paths: "401": *a1 "403": *a3 "409": *a7 - "422": *a4 + "422": + description: "The whole batch was refused and nothing was written: more than 500 entries (`meta.errorCode` = + `cycle.bulk.too_large`, no issue list), or the body failed validation (`meta.errorCode` = + `cycle.bulk.invalid`, every issue under `details.issues`). Both are permanent for the batch as sent. A + problem with one entry never refuses the batch; it comes back as that entry's `skipped` status." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" "429": *a5 /api/cycle/period: post: @@ -3750,6 +3758,12 @@ paths: in request order. Use `/api/measurements/batch` for Apple Health upload streams. + Sleep stages by name (since v1.39.1): an object with `type: SLEEP_DURATION`, a `sleepStage` of `IN_BED`, + `ASLEEP`, `AWAKE`, `CORE`, `DEEP` or `REM`, and the segment as `startDate` + `endDate` or as `measuredAt` (its + end) + `value` (minutes). It is stored and deduplicated exactly as the batch route stores a HealthKit sleep + sample, so a bridge can send stages without the HealthKit codepoints. See `NamedSleepStageEntry`. + + Also reachable with a narrow `measurements:write` Bearer, which is the supported way to push readings from a scale, a watch bridge or a home-automation rule. That credential is confined to its owner's own record — a request carrying the account selector is refused 403 before any sharing grant is read, however the record is @@ -3767,12 +3781,21 @@ paths: schema: anyOf: - $ref: "#/components/schemas/CreateMeasurementRequest" + - $ref: "#/components/schemas/NamedSleepStageEntry" - type: array items: $ref: "#/components/schemas/CreateMeasurementRequest" responses: + "200": + description: "Named sleep-stage body only: the segment was already stored (`status: duplicate`) or a stored one was + corrected in place (`status: updated`). Every other body answers 201 or an error, as before." + content: + application/json: + schema: + $ref: "#/components/schemas/NamedSleepStageDuplicateResponse" "201": - description: Created. A single resource for the object body; an array of resources, in request order, for the array body. + description: "Created. A single resource for the object body; an array of resources, in request order, for the array + body; for a named sleep-stage body, the stored row with `status: inserted`." content: application/json: schema: @@ -3781,7 +3804,7 @@ paths: "403": *a9 "409": description: A measurement with this data already exists — the `(userId, type, source, externalId)` dedup index rejected - the write. + the write. Never for a named sleep-stage body, which answers a re-post with 200 instead. content: application/json: schema: @@ -3945,9 +3968,11 @@ paths: "401": *a1 "409": *a7 "422": - description: The batch exceeded the 500-entry limit (`measurement.batch.too_large`), failed validation, or - (`measurement.batch.source_not_permitted`) carried an entry naming a source — any source — under a narrow - `measurements:write` credential. Nothing was written in any of the three cases. + description: "The batch exceeded the 500-entry limit (`measurement.batch.too_large`), failed validation + (`measurement.batch.invalid`, with every issue under `details.issues`), or + (`measurement.batch.source_not_permitted`) carried an entry naming a source, any source, under a narrow + `measurements:write` credential. Nothing was written in any of the three cases, and all three are permanent + for the batch as sent: resending it unchanged is refused the same way." content: application/json: schema: @@ -16855,8 +16880,11 @@ paths: Upsert key is (day, nutrient): a re-post replaces the stored total and reports `updated`. Post the current + previous local day per sync; backfill 30 days on first enable. Max 500 entries; Idempotency-Key supported; rate limit 60/min. Per-entry skips (`unit_mismatch` | `value_out_of_range` | `day_invalid`) are terminal — log and - drop, do not retry the entry. Behind the opt-in `nutrients` module (403 `module.disabled` when off). Query - HealthKit with exactly the listed unit per code (`ug` = microgram; do not convert to IU): + drop, do not retry the entry. `upsert_failed` is not one of them: it is a server-side write failure for the + entry's whole write group, nothing was stored, and the entry should be sent again on a later sync; a response + carrying it is `Cache-Control: no-store` and is not replayed for its `Idempotency-Key`. Behind the opt-in + `nutrients` module (403 `module.disabled` when off). Query HealthKit with exactly the listed unit per code (`ug` + = microgram; do not convert to IU): | code | HealthKit identifier | unit | @@ -22158,6 +22186,61 @@ components: `glucoseContext` stays REQUIRED on a `BLOOD_GLUCOSE` row here: this is the hand-entry surface, where the person taking the reading knows whether it was fasting or after a meal. The bulk ingest paths (CSV import, JSON import, device sync) accept a contextless reading, because a sensor export classifies nothing per sample." + NamedSleepStageEntry: + type: object + properties: + type: + type: string + const: SLEEP_DURATION + sleepStage: + type: string + enum: + - IN_BED + - ASLEEP + - AWAKE + - CORE + - DEEP + - REM + startDate: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + endDate: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + measuredAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + value: + type: number + externalId: + type: string + minLength: 1 + maxLength: 120 + source: + default: MANUAL + type: string + enum: + - MANUAL + - APPLE_HEALTH + deviceType: + anyOf: + - type: string + minLength: 1 + maxLength: 32 + - type: "null" + required: + - type + - sleepStage + description: "One sleep-stage segment with the stage named (since v1.39.1), for a bridge that does not speak HealthKit. + Chosen whenever the object carries a string `sleepStage`. `type` is `SLEEP_DURATION`. The segment is either + `startDate` + `endDate`, or `measuredAt` (its END) + `value` (its length in minutes); a `value` sent beside the + dates must agree with them to within a minute. Stored as the batch route stores a HealthKit sleep sample: value + in minutes, `measuredAt` at the end, the stage in `sleepStage`. Without an `externalId` the server derives one + from the stage and both instants, so sending the same segment again is a `duplicate`, and a segment whose start + moved is `updated` rather than added." AppleHealthBatchRequest: type: object properties: @@ -30331,7 +30414,7 @@ components: labs.update.linkedFieldsImmutable, labs.update.numericExpected, labs.update.qualitativeExpected, labs.update.referenceRangeInvalid, labs.update.sourceReferenceRangeInvalid; managed_profile — managed_profile.guardian.duplicate, managed_profile.guardian.managed_invitee, - managed_profile.guardian.required, managed_profile.not_found; measurement — + managed_profile.guardian.required, managed_profile.not_found; measurement — measurement.batch.invalid, measurement.batch.source_not_permitted, measurement.batch.too_large, measurement.bulk-delete.invalid, measurement.create.invalid, measurement.create.source_not_permitted, measurement.delete.too_large, measurement.duplicate_timestamp, measurement.restore.invalid, measurement.update.server_owned_source; @@ -30805,6 +30888,12 @@ components: externalId: type: string reason: + description: "On a `skipped` entry. `unstable_external_id` is a verdict on the entry and terminal: resending it + unchanged is skipped again. `constraint` means the write met a database constraint (a key another + write claimed at the same moment, or a reference that no longer exists). `upsert_failed` is a + write failure on the server's side: nothing was stored, the entry should be sent again, and the + response carrying it is `Cache-Control: no-store`, so a retry under the same `Idempotency-Key` + writes again rather than replaying the failure." type: string required: - index @@ -39652,6 +39741,136 @@ components: - limit - offset additionalProperties: false + NamedSleepStageDuplicateResponse: + type: object + properties: + data: + $ref: "#/components/schemas/NamedSleepStageMeasurement" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + NamedSleepStageMeasurement: + type: object + properties: + id: + type: string + description: Stored row id, or a synthetic key (`day::`, `sleep-seg::`) on the collapsed list + modes. + type: + $ref: "#/components/schemas/MeasurementType" + value: + type: number + unit: + type: string + measuredAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + source: + $ref: "#/components/schemas/MeasurementSource" + notes: + anyOf: + - type: string + - type: "null" + userId: + type: string + valueMin: + description: Bucket MINIMUM — set only on an hourly heart-rate `stats:` row. + anyOf: + - type: number + - type: "null" + valueMax: + description: Bucket MAXIMUM — set only on an hourly heart-rate `stats:` row. + anyOf: + - type: number + - type: "null" + externalId: + description: External-system dedup key; null for manual entries. + anyOf: + - type: string + - type: "null" + externalSourceVersion: + anyOf: + - type: string + - type: "null" + glucoseContext: + description: Set only on BLOOD_GLUCOSE rows, and null there too when the writer recorded no classification (a sensor + export, a device sync). Never invent one. + anyOf: + - type: string + - type: "null" + sleepStage: + description: Set only on SLEEP_DURATION rows. + anyOf: + - type: string + - type: "null" + rhythmClassification: + description: Device classification verdict; set only on EVENT rows. + anyOf: + - type: string + - type: "null" + deviceType: + description: "`watch | band | ring | phone | scale | other | unknown`." + anyOf: + - type: string + - type: "null" + syncVersion: + description: Bumped on every server-side mutation; the last-writer-wins input for cross-device reconciliation. + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + deletedAt: + description: Soft-delete tombstone; null on a live row. + anyOf: + - type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + - type: "null" + createdAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + updatedAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + dayKey: + type: string + sampleCount: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + partial: + description: True on the one bucket whose day was cut off by the read cap, so its sum understates the real total. + type: boolean + status: + type: string + enum: + - inserted + - updated + - duplicate + description: "`inserted` — a new row (HTTP 201). `duplicate` — the segment was already stored, by this route or the + batch route, and nothing changed (HTTP 200). `updated` — a stored segment with the same stage, source and + end was corrected in place, for example a later start (HTTP 200)." + required: + - id + - type + - value + - unit + - measuredAt + - source + - status + additionalProperties: false + description: The stored SLEEP_DURATION row for a named sleep-stage segment, with what the call did to it. CreateMeasurementResponse: type: object properties: @@ -39661,6 +39880,7 @@ components: - type: array items: $ref: "#/components/schemas/MeasurementResource" + - $ref: "#/components/schemas/NamedSleepStageMeasurement" error: type: "null" meta: @@ -39819,6 +40039,28 @@ components: not advance the entry cursor; the response is marked `Cache-Control: no-store`." reason: type: string + convertedValue: + description: "Only on a `value_out_of_range` skip (since v1.39.1): the entry's value after conversion into the stored + unit, which is the number the band was checked against. A HealthKit fraction that should have been a percent + shows up here a hundred times off." + type: number + range: + description: "Only on a `value_out_of_range` skip (since v1.39.1): the plausibility band that refused the entry, in the + stored unit. Both edges are inclusive. A skip is permanent for the value as sent; resending it unchanged + skips it again." + type: object + properties: + min: + type: number + max: + type: number + unit: + type: string + required: + - min + - max + - unit + additionalProperties: false required: - index - status @@ -47237,6 +47479,9 @@ components: - type: "null" timezone: type: string + description: "The IANA zone the server cuts this account's days in, the same value `/api/auth/me` reports: the stored + zone, or the instance default when the stored value is not a usable IANA name (since v1.39.1; before, the + raw stored value). Always a valid IANA name, so sending it back in a PATCH is accepted." timeFormat: type: string enum: @@ -47349,6 +47594,9 @@ components: - type: "null" timezone: type: string + description: "The IANA zone the server cuts this account's days in, the same value `/api/auth/me` reports: the stored + zone, or the instance default when the stored value is not a usable IANA name (since v1.39.1; before, the + raw stored value). Always a valid IANA name, so sending it back in a PATCH is accepted." timeFormat: type: string enum: @@ -64471,7 +64719,12 @@ components: - status additionalProperties: false description: "Per-entry ingest outcome. skipped reasons: unit_mismatch | value_out_of_range | day_invalid | - upsert_failed. Log and drop a skipped entry — do not retry it." + upsert_failed. The first three are verdicts on the entry itself and terminal: resending it unchanged is skipped + again, so log and drop it. `upsert_failed` is a write failure on the server's side, not a verdict on the entry: + nothing was stored for it, and it should be sent again on a later sync. A write failure takes out the whole + group it was written in (every new day total, or every existing one), so several entries can carry it at once. A + response carrying it is marked `Cache-Control: no-store` and is not kept for `Idempotency-Key` replay, so the + retry writes again rather than replaying the failure." NutrientOverviewEnvelope: type: object properties: diff --git a/src/__tests__/measurement-write-plausibility-guard.test.ts b/src/__tests__/measurement-write-plausibility-guard.test.ts index 88fbafee2..cc4428f9f 100644 --- a/src/__tests__/measurement-write-plausibility-guard.test.ts +++ b/src/__tests__/measurement-write-plausibility-guard.test.ts @@ -130,6 +130,22 @@ const WRITERS: Record = { const MEASUREMENT_WRITE = /[A-Za-z_$][\w$]*\s*\.\s*measurement\s*\.\s*(create|createMany|createManyAndReturn|upsert)\s*\(/; +/** + * A call into the column-array bulk insert, which writes around the Prisma + * delegate and so never matches {@link MEASUREMENT_WRITE}. The Apple export + * importer writes its spot rows only this way, and was invisible to this + * guard for as long as nothing looked for it. The definitions themselves are + * excluded by the lookbehind; the module that holds them is pinned below as + * the one place raw SQL may insert a measurement. + */ +const BULK_INSERT_CALL = /(? !p.startsWith("generated/")) @@ -143,7 +159,10 @@ function read(rel: string): string { } function discoveredWriters(): string[] { - return sourceFiles().filter((rel) => MEASUREMENT_WRITE.test(read(rel))); + return sourceFiles().filter((rel) => { + const text = read(rel); + return MEASUREMENT_WRITE.test(text) || BULK_INSERT_CALL.test(text); + }); } describe("every measurement writer declares how it meets the plausibility domain", () => { @@ -158,6 +177,21 @@ describe("every measurement writer declares how it meets the plausibility domain expect(discoveredWriters()).toEqual(Object.keys(WRITERS).sort()); }); + it("sees a writer that only calls the bulk insert", () => { + // The importer has no Prisma measurement write left; if the bulk-insert + // matcher stopped matching, it would drop out of the sweep silently. + const importer = "lib/measurements/import-apple-health-export.ts"; + expect(MEASUREMENT_WRITE.test(read(importer))).toBe(false); + expect(discoveredWriters()).toContain(importer); + }); + + it("raw SQL inserts measurements in the bulk-insert module and nowhere else", () => { + const raw = sourceFiles().filter((rel) => + RAW_MEASUREMENT_INSERT.test(read(rel)), + ); + expect(raw).toEqual([BULK_INSERT_MODULE]); + }); + it("every provider-fed writer reaches the gate module", () => { const gated = Object.entries(WRITERS) .filter(([, d]) => d.kind === "gate") diff --git a/src/__tests__/v1391-additive-contract.test.ts b/src/__tests__/v1391-additive-contract.test.ts new file mode 100644 index 000000000..06ed6b9b6 --- /dev/null +++ b/src/__tests__/v1391-additive-contract.test.ts @@ -0,0 +1,253 @@ +/** + * v1.39.1 — every wire change in this release is additive for the iPhone app + * already in people's hands (1.0.3 in the App Store, 1.0.4 on TestFlight). + * + * What changed on shapes those builds decode: + * + * - `AppleHealthBatchEntryResult` gained `convertedValue` and `range`, on a + * `value_out_of_range` skip only; + * - the batch validation 422 gained `meta.errorCode`; + * - `ProfileResponse.timezone` now holds the resolved zone (same type); + * - the nutrient and cycle batch responses gained a `Cache-Control` header + * on one outcome, and the nutrient entry's description changed. + * + * Two directions are pinned, because "additive" is a claim about both: + * + * 1. A payload the new server sends still decodes with a decoder written + * against the old shape. The decoders below are that: the fields the + * shipped client reads non-optionally, with their types, and nothing + * else — unknown keys are ignored, as Swift's `Decodable` ignores them. + * 2. A payload of the old shape still satisfies the new document: no field + * a shipped client might omit or never see became required. + * + * The document is built from the registry rather than read off disk; + * `openapi:check` already pins the YAML to exactly this build. + * + * ## Mutation check + * + * Drop `.optional()` from `convertedValue` or `range` in the registry and the + * `required` assertion goes red. Rename a field the old decoder reads and the + * decode assertion goes red. + */ +import { describe, expect, it } from "vitest"; +import { z } from "zod/v4"; + +import { buildOpenApiDocument } from "@/lib/openapi/registry"; + +type Schema = { + type?: string | string[]; + properties?: Record; + required?: string[]; + additionalProperties?: boolean | Schema; + $ref?: string; + anyOf?: Schema[]; +}; + +const doc = buildOpenApiDocument() as unknown as { + components: { schemas: Record }; +}; +const schemas = doc.components.schemas; + +function component(name: string): Schema { + const found = schemas[name]; + if (!found) throw new Error(`component ${name} is missing`); + return found; +} + +// ── The shapes as 1.0.3 decodes them ──────────────────────────────────── + +const oldBatchEntry = z.object({ + index: z.number().int(), + status: z.string(), + reason: z.string().optional(), +}); + +const oldBatchResponse = z.object({ + data: z.object({ + processed: z.number(), + inserted: z.number(), + updated: z.number(), + duplicates: z.number(), + failed: z.number(), + skipped: z.array(z.object({ index: z.number(), reason: z.string() })), + entries: z.array(oldBatchEntry), + }), +}); + +const oldErrorEnvelope = z.object({ + data: z.null(), + error: z.string(), +}); + +const oldProfile = z.object({ + data: z.object({ + username: z.string(), + email: z.string().nullable(), + timezone: z.string(), + timeFormat: z.string(), + dateFormat: z.string(), + moodReminderEnabled: z.boolean(), + }), +}); + +const oldNutrientResponse = z.object({ + data: z.object({ + processed: z.number(), + inserted: z.number(), + updated: z.number(), + skipped: z.array(z.object({ index: z.number(), reason: z.string() })), + entries: z.array( + z.object({ + index: z.number(), + status: z.string(), + reason: z.string().optional(), + }), + ), + }), +}); + +// ── Fixtures ──────────────────────────────────────────────────────────── + +/** A v1.39.0 batch answer: no range detail anywhere. */ +const OLD_BATCH = { + data: { + processed: 2, + inserted: 1, + updated: 0, + duplicates: 0, + failed: 0, + skipped: [{ index: 1, reason: "value_out_of_range" }], + entries: [ + { index: 0, status: "inserted" }, + { index: 1, status: "skipped", reason: "value_out_of_range" }, + ], + }, + error: null, +}; + +/** The same batch answered by v1.39.1. */ +const NEW_BATCH = { + data: { + ...OLD_BATCH.data, + entries: [ + { index: 0, status: "inserted" }, + { + index: 1, + status: "skipped", + reason: "value_out_of_range", + convertedValue: 90, + range: { min: 1, max: 80, unit: "%" }, + }, + ], + }, + error: null, +}; + +const NEW_BATCH_422 = { + data: null, + error: "Invalid input: expected string, received undefined", + details: { + issues: [{ path: ["entries", 0, "unit"], message: "Required" }], + }, + meta: { errorCode: "measurement.batch.invalid" }, +}; + +const NEW_PROFILE = { + data: { + username: "someone", + displayName: null, + email: null, + dateOfBirth: null, + gender: null, + heightCm: null, + locale: null, + timezone: "Europe/Berlin", + timeFormat: "AUTO", + dateFormat: "AUTO", + moodReminderEnabled: false, + fullName: null, + insurerName: null, + insurerIkNumber: null, + insuranceNumber: null, + modules: {}, + }, + error: null, +}; + +const NEW_NUTRIENTS = { + data: { + processed: 1, + inserted: 0, + updated: 0, + skipped: [{ index: 0, reason: "upsert_failed" }], + entries: [{ index: 0, status: "skipped", reason: "upsert_failed" }], + }, + error: null, +}; + +describe("v1.39.1 — shipped decoders still decode what the server now sends", () => { + it("the batch answer, with the new range detail", () => { + expect(oldBatchResponse.safeParse(NEW_BATCH).success).toBe(true); + }); + + it("the batch validation 422, with its new code", () => { + expect(oldErrorEnvelope.safeParse(NEW_BATCH_422).success).toBe(true); + }); + + it("the profile, whose timezone is still a string", () => { + expect(oldProfile.safeParse(NEW_PROFILE).success).toBe(true); + }); + + it("the nutrient batch answer carrying upsert_failed", () => { + expect(oldNutrientResponse.safeParse(NEW_NUTRIENTS).success).toBe(true); + }); +}); + +describe("v1.39.1 — the new document still admits the old shapes", () => { + it("adds convertedValue and range to the batch entry as optional fields", () => { + const entry = component("AppleHealthBatchEntryResult"); + expect(entry.properties?.convertedValue?.type).toBe("number"); + expect( + Object.keys(entry.properties?.range?.properties ?? {}).sort(), + ).toEqual(["max", "min", "unit"]); + expect([...(entry.required ?? [])].sort()).toEqual(["index", "status"]); + }); + + it("an old batch entry carries every field the new document requires", () => { + const required = component("AppleHealthBatchEntryResult").required ?? []; + for (const e of OLD_BATCH.data.entries) { + for (const key of required) expect(e).toHaveProperty(key); + } + }); + + it("keeps the error envelope's meta optional", () => { + const envelope = component("ErrorEnvelope"); + expect(envelope.required ?? []).not.toContain("meta"); + expect(envelope.properties?.meta?.required ?? []).not.toContain( + "errorCode", + ); + }); + + it("keeps the profile timezone a required string", () => { + const profile = component("ProfileResponse"); + expect(profile.properties?.timezone?.type).toBe("string"); + expect(profile.required).toContain("timezone"); + }); + + it("keeps the nutrient entry result's shape", () => { + const entry = component("NutrientEntryResult"); + expect(Object.keys(entry.properties ?? {}).sort()).toEqual([ + "index", + "reason", + "status", + ]); + expect([...(entry.required ?? [])].sort()).toEqual(["index", "status"]); + }); + + it("puts the named sleep-stage answer beside the old create response, not in place of it", () => { + const named = component("NamedSleepStageMeasurement"); + expect(named.required).toContain("status"); + const resource = component("MeasurementResource"); + expect(resource.properties).not.toHaveProperty("status"); + }); +}); diff --git a/src/app/api/auth/me/route.ts b/src/app/api/auth/me/route.ts index 0c7ac8244..2dfa2bc58 100644 --- a/src/app/api/auth/me/route.ts +++ b/src/app/api/auth/me/route.ts @@ -49,10 +49,7 @@ import { syncMfaEnrollCookie } from "@/lib/auth/mfa-enrollment"; import { buildAvatarUrl } from "@/lib/avatar"; import { decrypt } from "@/lib/crypto"; import { prisma } from "@/lib/db"; -import { - isValidTimezone, - resolveServerDefaultTimezone, -} from "@/lib/tz/resolver"; +import { resolveStoredTimezone } from "@/lib/tz/resolver"; import { isCycleEnabled } from "@/lib/cycle/gate"; import { resolveModuleMap, @@ -279,9 +276,7 @@ export const GET = apiHandler(async () => { // The zone the server cuts this account's days in: the stored zone, or // the instance default when the stored one is unusable. Clients bucket // by this value, so it must be the resolved one, not the raw column. - timezone: isValidTimezone(user.timezone) - ? user.timezone - : await resolveServerDefaultTimezone(), + timezone: await resolveStoredTimezone(user.timezone), onboardingCompletedAt: user.onboardingCompletedAt, onboardingTourCompleted: user.onboardingTourCompleted, // v1.39 (C1) — the needs-based setup flow, published additively: diff --git a/src/app/api/cycle/day-logs/bulk/__tests__/route.test.ts b/src/app/api/cycle/day-logs/bulk/__tests__/route.test.ts index d75526e8e..8b59f8fec 100644 --- a/src/app/api/cycle/day-logs/bulk/__tests__/route.test.ts +++ b/src/app/api/cycle/day-logs/bulk/__tests__/route.test.ts @@ -203,3 +203,82 @@ describe("POST /api/cycle/day-logs/bulk — external-id stability floor", () => expect(upsertCycleDayLog).toHaveBeenCalledTimes(1); }); }); + +describe("POST /api/cycle/day-logs/bulk — refusal codes", () => { + /** + * The native client files rows under a permanent-refusal register as soon + * as it can tell the refusal is permanent. It can only tell from the code, + * so both whole-batch 422s are pinned here, with the envelope they ride. + */ + async function refusal(body: unknown) { + const res = await POST(postReq(body)); + return { + status: res.status, + body: (await res.json()) as { + data: null; + error: string; + details?: { issues: unknown[] }; + meta?: { errorCode?: string }; + }, + }; + } + + it("names a malformed batch cycle.bulk.invalid, with every issue listed", async () => { + const { status, body } = await refusal({ + entries: [{ date: "not-a-day" }, { date: "2026-13-40" }], + }); + expect(status).toBe(422); + expect(body.data).toBeNull(); + expect(body.meta?.errorCode).toBe("cycle.bulk.invalid"); + expect(body.details?.issues.length).toBeGreaterThanOrEqual(2); + expect(upsertCycleDayLog).not.toHaveBeenCalled(); + }); + + it("names a batch with no entries array cycle.bulk.invalid", async () => { + const { status, body } = await refusal({}); + expect(status).toBe(422); + expect(body.meta?.errorCode).toBe("cycle.bulk.invalid"); + }); + + it("names an over-cap batch cycle.bulk.too_large", async () => { + const entries = Array.from({ length: 501 }, (_, i) => ({ + date: "2026-01-01", + externalId: `e${i}`, + })); + const { status, body } = await refusal({ entries }); + expect(status).toBe(422); + expect(body.meta?.errorCode).toBe("cycle.bulk.too_large"); + }); +}); + +describe("POST /api/cycle/day-logs/bulk — a failed write is not cached", () => { + it("marks a response carrying upsert_failed no-store and leaves a clean one alone", async () => { + vi.mocked(upsertCycleDayLog).mockRejectedValueOnce(new Error("reset")); + const failed = await POST( + postReq({ + entries: [{ date: "2026-01-01", loggedAt: "2026-01-01T08:00:00.000Z" }], + }), + ); + expect(failed.status).toBe(200); + const body = (await failed.json()) as { + data: { entries: Array<{ status: string; reason?: string }> }; + }; + expect(body.data.entries[0]).toMatchObject({ + status: "skipped", + reason: "upsert_failed", + }); + expect(failed.headers.get("Cache-Control")).toBe("private, no-store"); + + vi.mocked(upsertCycleDayLog).mockResolvedValueOnce({ + id: "d1", + existed: false, + changed: true, + } as never); + const clean = await POST( + postReq({ + entries: [{ date: "2026-01-02", loggedAt: "2026-01-02T08:00:00.000Z" }], + }), + ); + expect(clean.headers.get("Cache-Control")).not.toBe("private, no-store"); + }); +}); diff --git a/src/app/api/cycle/day-logs/bulk/route.ts b/src/app/api/cycle/day-logs/bulk/route.ts index 890bbf240..93c56116b 100644 --- a/src/app/api/cycle/day-logs/bulk/route.ts +++ b/src/app/api/cycle/day-logs/bulk/route.ts @@ -249,7 +249,7 @@ async function postBulk(request: NextRequest): Promise { }, }); - return apiSuccess({ + const response = apiSuccess({ processed: entries.length, inserted, updated, @@ -257,4 +257,12 @@ async function postBulk(request: NextRequest): Promise { skipped, entries: results, }); + // `upsert_failed` is a write that did not happen, not a verdict on the + // entry, and the client sends it again. `no-store` keeps the idempotency + // layer from caching this answer, so a retry under the same key writes + // instead of replaying the failure. + if (results.some((r) => r.reason === "upsert_failed")) { + response.headers.set("Cache-Control", "private, no-store"); + } + return response; } diff --git a/src/app/api/measurements/batch/__tests__/refusal-detail.test.ts b/src/app/api/measurements/batch/__tests__/refusal-detail.test.ts new file mode 100644 index 000000000..921cda502 --- /dev/null +++ b/src/app/api/measurements/batch/__tests__/refusal-detail.test.ts @@ -0,0 +1,231 @@ +/** + * `POST /api/measurements/batch` — what a refusal tells the client. + * + * Two gaps the native client reported against v1.39.0: + * + * - The whole-batch validation 422 carried no `meta.errorCode`, so the + * client could not tell a permanent refusal from a transient one and had + * to hold its anchor for several syncs before giving up on the rows. + * The too-large and source refusals already had codes; this one now has + * `measurement.batch.invalid`. + * - A `value_out_of_range` skip said neither what the value became after + * conversion nor which band it broke, so a unit-mapping slip (a fraction + * sent where a percent was expected) looked the same as a sensor glitch. + * The entry now carries `convertedValue` and `range`. + * + * Both are additive: status codes, the envelope and every existing field + * are unchanged. + */ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { NextRequest } from "next/server"; + +vi.mock("@/lib/db", () => ({ + prisma: { + user: { update: vi.fn() }, + measurement: { findMany: vi.fn() }, + $transaction: vi.fn(async (fn: unknown) => { + if (typeof fn === "function") { + return (fn as (tx: unknown) => unknown)(prisma); + } + }), + }, +})); + +vi.mock("@/lib/measurements/reconcile-external-measurement", () => ({ + reconcileExternalMeasurement: vi.fn( + async (_tx: unknown, input: Record) => ({ + status: "inserted", + row: { id: "row-1", ...input }, + }), + ), +})); + +vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); +vi.mock("@/lib/auth/audit", () => ({ + auditLog: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: vi.fn() })); +vi.mock("@/lib/jobs/pr-detection", () => ({ + enqueuePrDetection: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/jobs/reminder-satisfy", () => ({ + enqueueReminderSatisfy: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserMeasurements: vi.fn(), +})); +vi.mock("@/lib/rollups/after-measurement-mutation", () => ({ + afterMeasurementMutation: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/daily/morning-refresh-trigger", () => ({ + maybeEnqueueMorningRefresh: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/arrivals/emit-shared", () => ({ + emitDataArrival: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("next/headers", () => ({ + headers: vi.fn(async () => ({ get: () => null })), + cookies: vi.fn(async () => ({ + get: () => undefined, + set: () => {}, + delete: () => {}, + })), +})); + +import { POST } from "../route"; +import { prisma } from "@/lib/db"; +import { getSession } from "@/lib/auth/session"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement"; + +const SESSION_OK = { + session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, + user: { id: "user-1", username: "testuser", role: "USER" as const }, +}; + +function post(body: unknown) { + return POST( + new NextRequest("http://localhost/api/measurements/batch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ); +} + +const AT = new Date(Date.now() - 3_600_000).toISOString(); + +function entry(overrides: Record = {}) { + return { + hkIdentifier: "HKQuantityTypeIdentifierHeartRate", + value: 64, + unit: "count/min", + startDate: AT, + endDate: AT, + externalId: "uuid-1", + ...overrides, + }; +} + +type EntryResult = { + index: number; + status: string; + reason?: string; + convertedValue?: number; + range?: { min: number; max: number; unit: string }; +}; + +async function entriesFor(entries: unknown[]): Promise { + const res = await post({ entries }); + expect(res.status).toBe(200); + return ((await res.json()) as { data: { entries: EntryResult[] } }).data + .entries; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(checkRateLimit).mockResolvedValue({ + allowed: true, + limit: 60, + remaining: 60, + resetAt: Date.now() + 60_000, + }); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([]); +}); + +describe("POST /api/measurements/batch — validation refusal code", () => { + it("names a malformed batch measurement.batch.invalid and keeps the 422 envelope", async () => { + const res = await post({ entries: [entry({ startDate: "yesterday" })] }); + expect(res.status).toBe(422); + const body = (await res.json()) as { + data: null; + error: string; + details: { issues: unknown[] }; + meta?: { errorCode?: string }; + }; + expect(body.data).toBeNull(); + expect(typeof body.error).toBe("string"); + expect(body.details.issues.length).toBeGreaterThan(0); + expect(body.meta?.errorCode).toBe("measurement.batch.invalid"); + }); + + it("names an empty entries array the same way", async () => { + const res = await post({ entries: [] }); + expect(res.status).toBe(422); + expect( + ((await res.json()) as { meta?: { errorCode?: string } }).meta?.errorCode, + ).toBe("measurement.batch.invalid"); + }); + + it("keeps measurement.batch.too_large for an over-cap batch", async () => { + const entries = Array.from({ length: 501 }, (_, i) => + entry({ externalId: `uuid-${i}` }), + ); + const res = await post({ entries }); + expect(res.status).toBe(422); + expect( + ((await res.json()) as { meta?: { errorCode?: string } }).meta?.errorCode, + ).toBe("measurement.batch.too_large"); + }); +}); + +describe("POST /api/measurements/batch — value_out_of_range detail", () => { + it("reports the converted value and the band it broke, in the stored unit", async () => { + // HealthKit reports body fat as a fraction; 0.9 becomes 90 %, above 80. + const [result] = await entriesFor([ + entry({ + hkIdentifier: "HKQuantityTypeIdentifierBodyFatPercentage", + value: 0.9, + unit: "%", + }), + ]); + expect(result).toEqual({ + index: 0, + status: "skipped", + reason: "value_out_of_range", + convertedValue: 90, + range: { min: 1, max: 80, unit: "%" }, + }); + }); + + it("reports the lower edge too", async () => { + const [result] = await entriesFor([entry({ value: 12 })]); + expect(result).toMatchObject({ + reason: "value_out_of_range", + convertedValue: 12, + range: { min: 20, max: 300, unit: "bpm" }, + }); + }); + + it("carries the same detail when the write-side gate is the one that refuses", async () => { + vi.mocked(reconcileExternalMeasurement).mockResolvedValueOnce({ + status: "rejected_range", + rejection: { type: "PULSE", direction: "above_max" }, + }); + const [result] = await entriesFor([entry({ value: 64 })]); + expect(result).toMatchObject({ + status: "skipped", + reason: "value_out_of_range", + convertedValue: 64, + range: { min: 20, max: 300, unit: "bpm" }, + }); + }); + + it("adds nothing to an entry that landed or was skipped for another reason", async () => { + const results = await entriesFor([ + entry({ externalId: "uuid-ok" }), + entry({ hkIdentifier: "HKQuantityTypeIdentifierNotAThing" }), + ]); + expect(results[0]).toEqual({ index: 0, status: "inserted" }); + expect(results[1]).toEqual({ + index: 1, + status: "skipped", + reason: "unmappable_identifier", + }); + }); +}); diff --git a/src/app/api/measurements/batch/route.ts b/src/app/api/measurements/batch/route.ts index 278e4959a..40c0b3e20 100644 --- a/src/app/api/measurements/batch/route.ts +++ b/src/app/api/measurements/batch/route.ts @@ -72,7 +72,11 @@ import { UNSTABLE_EXTERNAL_ID_REASON, type UnstableExternalIdShape, } from "@/lib/validations/external-id"; -import { validateMeasurementRange } from "@/lib/validations/measurement"; +import { + VALUE_RANGES, + getUnitForType, + validateMeasurementRange, +} from "@/lib/validations/measurement"; import { deviceTypeEnum } from "@/lib/validations/source-priority"; import { checkRateLimit } from "@/lib/rate-limit"; import { enqueuePrDetection } from "@/lib/jobs/pr-detection"; @@ -217,6 +221,36 @@ interface EntryResult { index: number; status: EntryStatus; reason?: string; + /** + * Only on a `value_out_of_range` skip: the value after conversion into the + * stored unit, which is what the band was checked against. + */ + convertedValue?: number; + /** Only on a `value_out_of_range` skip: the band applied, in the stored unit. */ + range?: { min: number; max: number; unit: string }; +} + +/** + * The `value_out_of_range` skip, with what a client needs to say which limit + * applied. The value is the entry's own reading after conversion, handed back + * to the account that sent it; a fraction sent where a percent was expected + * shows up here as a converted value a hundred times too small or too large. + */ +function outOfRangeResult( + index: number, + type: string, + convertedValue: number, +): EntryResult { + const band = VALUE_RANGES[type]; + return { + index, + status: "skipped", + reason: "value_out_of_range", + ...(Number.isFinite(convertedValue) ? { convertedValue } : {}), + ...(band + ? { range: { min: band.min, max: band.max, unit: getUnitForType(type) } } + : {}), + }; } export const POST = apiHandler(withIdempotency<[NextRequest]>(postBatch)); @@ -277,10 +311,13 @@ async function postBatch(request: NextRequest): Promise { const parsed = batchPayloadSchema.safeParse(rawBody); if (!parsed.success) { + // The code is what lets a client tell this refusal from a transient one: + // the batch as sent will never be accepted, so resending it is pointless. return apiValidationError( parsed.error.issues[0]?.message ?? "Invalid batch", sanitiseZodIssues(parsed.error.issues), 422, + { errorCode: "measurement.batch.invalid" }, ); } @@ -373,11 +410,7 @@ async function postBatch(request: NextRequest): Promise { // sensor reading shouldn't poison an otherwise clean ingest. const rangeError = validateMeasurementRange(mapped.type, mapped.value); if (rangeError !== null) { - results[index] = { - index, - status: "skipped", - reason: "value_out_of_range", - }; + results[index] = outOfRangeResult(index, mapped.type, mapped.value); continue; } @@ -678,11 +711,11 @@ async function postBatch(request: NextRequest): Promise { // already applied above, so reaching here means the two disagreed — // report it exactly as the earlier guard would have, and let the // wide-event tally record which end refused. - results[p.index] = { - index: p.index, - status: "skipped", - reason: "value_out_of_range", - }; + results[p.index] = outOfRangeResult( + p.index, + p.row.type as string, + p.row.value as number, + ); continue; } diff --git a/src/app/api/measurements/route.ts b/src/app/api/measurements/route.ts index 6767b1ad7..b62d4319f 100644 --- a/src/app/api/measurements/route.ts +++ b/src/app/api/measurements/route.ts @@ -55,6 +55,21 @@ import { pickMainNightAndNaps, } from "@/lib/analytics/sleep-night"; import { resolveUserTimezone } from "@/lib/tz/resolver"; +import { + isNamedSleepStageBody, + sleepStageEntrySchema, + toSleepStageSegment, +} from "@/lib/validations/sleep-stage-entry"; +import { + MeasurementReconciliationError, + reconcileExternalMeasurement, +} from "@/lib/measurements/reconcile-external-measurement"; +import { + MEASURED_AT_TOLERANCE_MS, + isMergeableSource, + isSameReadingAcrossSource, + oppositeMergeSource, +} from "@/lib/measurements/cross-source-merge"; import { NextRequest } from "next/server"; import type { MeasurementType, @@ -765,6 +780,13 @@ async function postMeasurement(request: NextRequest) { ); } + // One sleep-stage segment, the stage named rather than coded. A bridge + // (Tasker, Home Assistant) posts here with its narrow token; the row and + // its dedupe are the batch route's. See `sleep-stage-entry.ts`. + if (isNamedSleepStageBody(body)) { + return postNamedSleepStage(request, user.id, body, scoped); + } + // Batch mode (array of measurements, e.g. combined BP + Pulse) if (Array.isArray(body)) { const parsed = createBatchMeasurementSchema.safeParse({ @@ -1084,3 +1106,177 @@ async function postMeasurement(request: NextRequest) { return apiSuccess(shapeMeasurementNotes(measurement), 201); } + +/** + * `POST /api/measurements` with a named `sleepStage`: one segment, stored as + * the batch route stores a HealthKit sleep sample. + * + * The same three steps as the batch route, in the same order: a same-reading + * row of the opposite source (MANUAL against APPLE_HEALTH) makes this one a + * duplicate; otherwise `reconcileExternalMeasurement` settles both unique + * identities at once, reporting an exact external-id match as a duplicate + * and adopting a row that only shares the natural identity. So a bridge that + * re-posts the same segment gets one row, whichever path it used first. + * + * `201` with the stored row when it was inserted, `200` when it was already + * there (`duplicate`) or was corrected in place (`updated`). The row carries + * a `status` saying which. + */ +async function postNamedSleepStage( + request: NextRequest, + userId: string, + body: unknown, + scoped: boolean, +): Promise { + const parsed = sleepStageEntrySchema.safeParse(body); + if (!parsed.success) { + annotate({ + action: { name: "measurements.create.sleep-stage.validation-failed" }, + meta: { issue_count: sanitiseZodIssues(parsed.error.issues).length }, + }); + return returnAllZodIssues(parsed.error, 422, { + errorCode: "measurement.create.invalid", + }); + } + + const segment = toSleepStageSegment(parsed.data); + // Under a scoped credential the source is resolved from the credential, + // as on the other arms of this route: the refusal above already returned + // on any body that named one. + const source = ( + scoped ? EXTERNAL_SOURCE : parsed.data.source + ) as MeasurementSource; + const type: MeasurementType = "SLEEP_DURATION"; + + // Cross-source same reading, exactly as the batch route decides it. + if (isMergeableSource(source)) { + const candidates = await prisma.measurement.findMany({ + where: { + userId, + deletedAt: null, + type, + source: oppositeMergeSource(source), + measuredAt: { + gte: new Date(segment.end.getTime() - MEASURED_AT_TOLERANCE_MS), + lte: new Date(segment.end.getTime() + MEASURED_AT_TOLERANCE_MS), + }, + }, + select: { + id: true, + type: true, + source: true, + value: true, + measuredAt: true, + }, + }); + const twin = candidates.find((candidate) => + isSameReadingAcrossSource( + { type, source, value: segment.minutes, measuredAt: segment.end }, + candidate, + ), + ); + if (twin) { + return namedSleepStageResponse(twin.id, "duplicate", { + stage: segment.stage, + source, + crossSource: true, + }); + } + } + + const verdict = await prisma.$transaction((tx) => + reconcileExternalMeasurement( + tx, + { + userId, + type, + value: segment.minutes, + unit: getUnitForType(type), + source, + measuredAt: segment.end, + externalId: segment.externalId, + externalSourceVersion: null, + sleepStage: segment.stage, + deviceType: parsed.data.deviceType ?? null, + }, + { exactExternalMatch: "duplicate" }, + ), + ); + + if (verdict.status === "failed") { + throw new MeasurementReconciliationError(verdict); + } + if (verdict.status === "rejected_range") { + // The schema applies the same band, so this is the two gates disagreeing. + return apiError("Value out of plausible range", 422, { + errorCode: "measurement.create.invalid", + }); + } + + const status = + verdict.status === "inserted" + ? "inserted" + : verdict.status === "duplicate" + ? "duplicate" + : "updated"; + + if (status !== "duplicate") { + await auditLog("measurement.create", { + userId, + ipAddress: getClientIp(request), + details: { + type, + measurementId: verdict.row.id, + sleepStage: segment.stage, + status, + }, + }); + invalidateUserMeasurements(userId, { evict: true }); + fireAndForget(enqueueReminderSatisfy(userId), { + action: "reminder.satisfy.enqueue", + }); + if (status === "inserted") { + void emitInsertedMeasurementArrivals( + userId, + [verdict.row], + "manual", + ).catch(() => {}); + void maybeEnqueueMorningRefresh(userId, [segment.end]).catch(() => {}); + } + await afterMeasurementMutation(userId, [ + { type, measuredAt: segment.end }, + ...(verdict.dirtyIdentities ?? []), + ]); + } + + return namedSleepStageResponse(verdict.row.id, status, { + stage: segment.stage, + source, + crossSource: false, + }); +} + +async function namedSleepStageResponse( + id: string, + status: "inserted" | "updated" | "duplicate", + meta: { stage: string; source: string; crossSource: boolean }, +): Promise { + annotate({ + action: { + name: "measurement.create.sleep-stage", + entity_type: "measurement", + entity_id: id, + }, + meta: { + status, + stage: meta.stage, + source: meta.source, + cross_source_merge: meta.crossSource, + }, + }); + const row = await prisma.measurement.findUniqueOrThrow({ where: { id } }); + return apiSuccess( + { ...shapeMeasurementNotes(row), status }, + status === "inserted" ? 201 : 200, + ); +} diff --git a/src/app/api/nutrients/batch/__tests__/route.test.ts b/src/app/api/nutrients/batch/__tests__/route.test.ts index 09dba1152..9544efec6 100644 --- a/src/app/api/nutrients/batch/__tests__/route.test.ts +++ b/src/app/api/nutrients/batch/__tests__/route.test.ts @@ -69,6 +69,7 @@ import { checkRateLimit } from "@/lib/rate-limit"; import { requireModuleEnabled } from "@/lib/modules/gate"; import { auditLog } from "@/lib/auth/audit"; import { invalidateUserDashboardSnapshot } from "@/lib/cache/invalidate"; +import { nutrientEntryResultSchema } from "@/lib/validations/nutrients"; const SESSION_OK = { session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, @@ -449,6 +450,46 @@ describe("POST /api/nutrients/batch — write semantics", () => { spy.mockRestore(); }); + it("marks a response carrying upsert_failed no-store, so a retry with the same Idempotency-Key writes again", async () => { + const spy = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.mocked(prisma.nutrientIntakeDay.createMany).mockRejectedValue( + new Error("connection reset"), + ); + const failed = await POST( + postReq({ + entries: [ + { day: recentDay(), nutrient: "zinc", unit: "mg", amount: 10 }, + ], + }), + ); + expect(failed.status).toBe(200); + expect(failed.headers.get("Cache-Control")).toBe("private, no-store"); + spy.mockRestore(); + + vi.mocked(prisma.nutrientIntakeDay.createMany).mockResolvedValue({ + count: 1, + } as never); + const clean = await POST( + postReq({ + entries: [ + { day: recentDay(), nutrient: "zinc", unit: "mg", amount: 10 }, + ], + }), + ); + expect(clean.headers.get("Cache-Control")).not.toBe("private, no-store"); + }); + + it("documents upsert_failed as a transient write failure to retry, and the validation skips as terminal", () => { + const description = String( + (nutrientEntryResultSchema.meta() as { description?: string }) + .description, + ); + expect(description).toMatch(/`upsert_failed` is a write failure/); + expect(description).toMatch(/sent again on a later sync/); + expect(description).not.toMatch(/log and drop a skipped entry/i); + expect(description).toMatch(/unit_mismatch/); + }); + it("a race that drops one createMany row still counts consistently and is observable", async () => { vi.mocked(prisma.nutrientIntakeDay.createMany).mockResolvedValue({ count: 1, diff --git a/src/app/api/nutrients/batch/route.ts b/src/app/api/nutrients/batch/route.ts index cdfadcc57..fb15fc1a3 100644 --- a/src/app/api/nutrients/batch/route.ts +++ b/src/app/api/nutrients/batch/route.ts @@ -18,7 +18,10 @@ * per-row updates (existing pairs), the measurements-batch shape. A write * failure marks its WHOLE group `skipped upsert_failed` rather than * isolating the one entry that would have caused it (unavoidable once the - * write is batched) — the request itself never fails. + * write is batched) — the request itself never fails. That reason is + * transient, unlike the validation skips: the client sends those entries + * again, and the response is marked `no-store` so an idempotent retry is not + * answered from the cache. * * Module gate FIRST: the opt-in `nutrients` module (default off) * refuses ingest with the 403 `module.disabled` envelope, so a phone @@ -360,11 +363,19 @@ async function postBatch(request: NextRequest): Promise { }, }); - return apiSuccess({ + const response = apiSuccess({ processed: entries.length, inserted, updated, skipped, entries: results, }); + // `upsert_failed` is transient: nothing was stored for those entries and the + // client sends them again. `no-store` keeps the idempotency layer from + // caching this answer, so a retry under the same key writes instead of + // replaying the failure. The measurement batch does the same for `failed`. + if (skippedByReason.upsert_failed) { + response.headers.set("Cache-Control", "private, no-store"); + } + return response; } diff --git a/src/app/api/user/profile/__tests__/route.test.ts b/src/app/api/user/profile/__tests__/route.test.ts index f771581db..03567bd91 100644 --- a/src/app/api/user/profile/__tests__/route.test.ts +++ b/src/app/api/user/profile/__tests__/route.test.ts @@ -4,6 +4,7 @@ import { NextRequest } from "next/server"; vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn(), update: vi.fn() }, + appSettings: { findUnique: vi.fn() }, }, })); @@ -60,6 +61,12 @@ const SESSION_OK = { beforeEach(() => { vi.resetAllMocks(); vi.mocked(resolveModuleMap).mockResolvedValue(MODULE_MAP as never); + // The operator's default zone, which the server falls back to when a + // stored one is unusable. Distinct from the built-in default so a test can + // tell the resolver ran. + vi.mocked(prisma.appSettings.findUnique).mockResolvedValue({ + defaultUserTimezone: "America/Chicago", + } as never); }); const callGet = GET as unknown as (req: NextRequest) => Promise; @@ -438,3 +445,81 @@ describe("PATCH /api/user/profile", () => { ]); }); }); + +/** + * `/api/auth/me` has returned the zone the server actually cuts days in — + * the stored one, or the instance default when the stored value is unusable + * — while this alias returned the raw column. A client reading this endpoint + * could bucket a day differently from the server. Both now go through the + * same resolution. + */ +describe("/api/user/profile — timezone is the resolved zone", () => { + function profileRow(timezone: string | null) { + return { + username: "testuser", + displayName: null, + email: null, + dateOfBirth: null, + gender: null, + heightCm: null, + locale: null, + timezone, + moodReminderEnabled: false, + fullName: null, + insurerName: null, + insurerIkNumber: null, + insuranceNumberEncrypted: null, + }; + } + + async function getZone(stored: string | null): Promise { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue( + profileRow(stored) as never, + ); + const res = await callGet(makeGetReq()); + expect(res.status).toBe(200); + return ((await res.json()) as { data: { timezone: string } }).data.timezone; + } + + it("returns a usable stored zone as stored", async () => { + expect(await getZone("Asia/Tokyo")).toBe("Asia/Tokyo"); + }); + + it("returns the instance default in place of a bare offset", async () => { + expect(await getZone("+05:30")).toBe("America/Chicago"); + }); + + it("returns the instance default in place of a name no runtime knows", async () => { + expect(await getZone("Mars/Olympus_Mons")).toBe("America/Chicago"); + }); + + it("returns the instance default when the row is missing", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue(null); + const res = await callGet(makeGetReq()); + const body = (await res.json()) as { data: { timezone: string } }; + expect(body.data.timezone).toBe("America/Chicago"); + }); + + it("answers a PATCH that leaves an unusable zone untouched with the resolved one", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue(null); + vi.mocked(prisma.user.update).mockResolvedValue({ + ...profileRow("+05:30"), + id: "user-1", + role: "USER", + displayName: "Alex T.", + } as never); + const res = await PATCH( + new NextRequest("http://localhost/api/user/profile", { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ displayName: "Alex T." }), + }), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { data: { timezone: string } }; + expect(body.data.timezone).toBe("America/Chicago"); + }); +}); diff --git a/src/app/api/user/profile/route.ts b/src/app/api/user/profile/route.ts index d6e6f6a71..38e3dd77f 100644 --- a/src/app/api/user/profile/route.ts +++ b/src/app/api/user/profile/route.ts @@ -20,6 +20,7 @@ import { applyProfileUpdate } from "@/lib/auth/profile-update"; import { prisma } from "@/lib/db"; import { decrypt } from "@/lib/crypto"; import { resolveModuleMap } from "@/lib/modules/gate"; +import { resolveStoredTimezone } from "@/lib/tz/resolver"; export const dynamic = "force-dynamic"; @@ -77,7 +78,10 @@ export const GET = apiHandler(async () => { gender: dbUser?.gender ?? null, heightCm: dbUser?.heightCm ?? null, locale: dbUser?.locale ?? null, - timezone: dbUser?.timezone ?? "Europe/Berlin", + // The zone the server cuts this account's days in, exactly as + // `/api/auth/me` reports it: the stored zone, or the instance default + // when the stored value is unusable. Never the raw column. + timezone: await resolveStoredTimezone(dbUser?.timezone), timeFormat: dbUser?.timeFormat ?? "AUTO", dateFormat: dbUser?.dateFormat ?? "AUTO", moodReminderEnabled: dbUser?.moodReminderEnabled ?? false, @@ -118,7 +122,7 @@ export const PATCH = apiHandler(async (request: NextRequest) => { gender: result.user.gender, heightCm: result.user.heightCm, locale: result.user.locale, - timezone: result.user.timezone, + timezone: await resolveStoredTimezone(result.user.timezone), timeFormat: result.user.timeFormat, dateFormat: result.user.dateFormat, moodReminderEnabled: result.user.moodReminderEnabled, diff --git a/src/lib/export/measurement-bulk-insert.ts b/src/lib/export/measurement-bulk-insert.ts index f1db771d9..4d0e7c21f 100644 --- a/src/lib/export/measurement-bulk-insert.ts +++ b/src/lib/export/measurement-bulk-insert.ts @@ -10,13 +10,28 @@ * `INSERT … SELECT FROM unnest(…)` with an array per column is a single small * statement whatever the batch size, and about four times faster. * + * The Apple Health export import writes its spot rows through the same + * statement (`insertNewMeasurementRows`), for the same reason. There the cost + * was per call rather than per shape: every `createManyAndReturn` of a + * 650-row flush left about 5 MB behind until roughly two dozen calls had + * accumulated, so a 300 000-record import settled at about 53 MB of retained + * heap and a run of back-to-back flushes reached well over 100 MB. Through + * this statement the same import holds a few megabytes. + * * `COLUMNS` maps every field of `MeasurementInsertRow` to its column and * type. It is typed as a complete record over the row's keys, so a field * added to the row without a column here does not compile, and the restore * builds its rows as `MeasurementInsertRow`, so a field it starts writing has * to be added to the row first. */ -import type { Prisma } from "@/generated/prisma/client"; +import { randomInt } from "node:crypto"; +import { hostname } from "node:os"; + +import type { + MeasurementType, + Prisma, + PrismaClient, +} from "@/generated/prisma/client"; /** One measurement as the restore writes it. */ export interface MeasurementInsertRow { @@ -134,3 +149,146 @@ export async function insertMeasurementRows( }); return tx.$executeRawUnsafe(MEASUREMENT_BULK_INSERT_SQL, ...arrays); } + +/** + * The same statement, skipping any row that collides with an existing one on + * either unique identity, and answering with the rows it did insert. + * + * `ON CONFLICT DO NOTHING` with no target is what Prisma emits for + * `createMany({ skipDuplicates: true })`: a row that meets either + * `(user_id, type, source, external_id)` or + * `(user_id, type, measured_at, source, sleep_stage)` is left out, a row that + * meets another row of the same statement is left out after the first, and + * nothing throws for either. + */ +export const MEASUREMENT_BULK_INSERT_SKIP_DUPLICATES_SQL = `${MEASUREMENT_BULK_INSERT_SQL} + ON CONFLICT DO NOTHING + RETURNING id, type::text AS "type", measured_at AS "measuredAt", external_id AS "externalId" +`; + +/** One row `insertNewMeasurementRows` wrote. */ +export interface InsertedMeasurementRow { + id: string; + type: MeasurementType; + measuredAt: Date; + externalId: string | null; +} + +/** A row for {@link insertNewMeasurementRows}: the id and the defaults are filled in. */ +export type NewMeasurementRow = Omit< + MeasurementInsertRow, + | "id" + | "valueMin" + | "valueMax" + | "notes" + | "notesEncrypted" + | "aggregationProvenance" + | "glucoseContext" + | "rhythmClassification" + | "syncVersion" + | "deletedAt" + | "createdAt" + | "updatedAt" +> & + Partial< + Pick< + MeasurementInsertRow, + | "valueMin" + | "valueMax" + | "notesEncrypted" + | "aggregationProvenance" + | "glucoseContext" + | "rhythmClassification" + > + >; + +/** + * Insert brand-new rows in one statement, skipping duplicates, and return the + * rows that landed. The replacement for + * `measurement.createManyAndReturn({ data, skipDuplicates: true })`. + * + * Every column the schema defaults is written with that default: a fresh + * cuid-shaped id, `sync_version` 1, not deleted, created and updated at the + * moment of the call, and null for every optional column the row does not + * carry. + */ +export async function insertNewMeasurementRows( + db: Pick | Prisma.TransactionClient, + rows: readonly NewMeasurementRow[], +): Promise { + if (rows.length === 0) return []; + // Stamped here, as the Prisma client stamps `@default(now())` and + // `@updatedAt`, rather than left to the database clock. + const now = new Date(); + const full: MeasurementInsertRow[] = rows.map((row) => ({ + id: newMeasurementId(), + userId: row.userId, + type: row.type, + value: row.value, + valueMin: row.valueMin ?? null, + valueMax: row.valueMax ?? null, + unit: row.unit, + source: row.source, + measuredAt: row.measuredAt, + notes: null, + notesEncrypted: row.notesEncrypted ?? null, + externalId: row.externalId, + externalSourceVersion: row.externalSourceVersion, + aggregationProvenance: row.aggregationProvenance ?? null, + glucoseContext: row.glucoseContext ?? null, + sleepStage: row.sleepStage, + rhythmClassification: row.rhythmClassification ?? null, + deviceType: row.deviceType, + syncVersion: 1, + deletedAt: null, + createdAt: now, + updatedAt: now, + })); + const arrays = FIELDS.map((field) => { + const kind = COLUMNS[field][1]; + return full.map((row) => cell(row[field], kind)); + }); + return db.$queryRawUnsafe( + MEASUREMENT_BULK_INSERT_SKIP_DUPLICATES_SQL, + ...arrays, + ); +} + +// ── Ids ────────────────────────────────────────────────────────────────── +// +// Prisma mints `@default(cuid())` in the client, so a row written around the +// client needs one minted here. Same shape as Prisma's own (cuid v1): `c`, the +// time in base 36, a rolling counter, a host fingerprint, and eight random +// characters, 25 characters in all. Ids stay roughly time-ordered, which the +// list reads rely on only as a tiebreaker. + +const BLOCK = 4; +const BLOCK_SPACE = 36 ** BLOCK; +let counter = randomInt(BLOCK_SPACE); + +function block(n: number): string { + return n.toString(36).padStart(BLOCK, "0").slice(-BLOCK); +} + +const FINGERPRINT = (() => { + const host = hostname(); + let sum = host.length + 36; + for (let i = 0; i < host.length; i++) sum += host.charCodeAt(i); + return ( + (process.pid % 1296).toString(36).padStart(2, "0") + + (sum % 1296).toString(36).padStart(2, "0") + ); +})(); + +/** A fresh id in the shape Prisma's `cuid()` default produces. */ +export function newMeasurementId(): string { + counter = (counter + 1) % BLOCK_SPACE; + return ( + "c" + + Date.now().toString(36).padStart(8, "0") + + block(counter) + + FINGERPRINT + + block(randomInt(BLOCK_SPACE)) + + block(randomInt(BLOCK_SPACE)) + ); +} diff --git a/src/lib/measurements/__tests__/import-apple-health-export.test.ts b/src/lib/measurements/__tests__/import-apple-health-export.test.ts index 607a332b2..17aa4b3c4 100644 --- a/src/lib/measurements/__tests__/import-apple-health-export.test.ts +++ b/src/lib/measurements/__tests__/import-apple-health-export.test.ts @@ -11,6 +11,27 @@ vi.mock("@/lib/daily/morning-refresh-trigger", () => ({ maybeEnqueueMorningRefresh: vi.fn().mockResolvedValue(undefined), })); +// The spot flush writes through one raw `INSERT … ON CONFLICT DO NOTHING +// RETURNING` statement. These fakes model the table in memory, so route that +// statement to the fake's own skip-duplicates insert; the SQL itself is pinned +// against a real Postgres in tests/integration/measurement-bulk-insert.test.ts. +vi.mock("@/lib/export/measurement-bulk-insert", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/export/measurement-bulk-insert") + >()), + insertNewMeasurementRows: vi.fn( + ( + db: { measurement: { createManyAndReturn: (args: unknown) => unknown } }, + rows: unknown[], + ) => + db.measurement.createManyAndReturn({ + data: rows, + skipDuplicates: true, + select: { id: true, type: true, measuredAt: true, externalId: true }, + }), + ), +})); + vi.mock("@/lib/arrivals/emit-shared", () => ({ emitDataArrival: vi.fn().mockResolvedValue(undefined), })); @@ -35,6 +56,7 @@ import { import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { maybeEnqueueMorningRefresh } from "@/lib/daily/morning-refresh-trigger"; import { emitDataArrival } from "@/lib/arrivals/emit-shared"; +import { insertNewMeasurementRows } from "@/lib/export/measurement-bulk-insert"; import { APPLE_HEALTH_TYPE_MAP, CUMULATIVE_HK_TYPES, @@ -731,6 +753,40 @@ describe("streamParseExportXml — cumulative source-day estimates", () => { }); }); +describe("streamParseExportXml — spot-row writer", () => { + it("writes spot rows through the bounded bulk insert, never createManyAndReturn", async () => { + const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-")); + const xmlPath = join(tmp, "export.xml"); + writeFileSync(xmlPath, tinyExportXml()); + const prisma = makeFakePrisma(); + const direct = vi.spyOn(prisma.measurement, "createManyAndReturn"); + const helper = vi.mocked(insertNewMeasurementRows); + helper.mockClear(); + + await streamParseExportXml({ + xmlPath, + userId: "user-1", + userTimezone: "Europe/Berlin", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prisma: prisma as any, + spotBatchSize: 2, + workoutBatchSize: 1, + }); + + expect(helper).toHaveBeenCalled(); + // Every createManyAndReturn the fake saw came through the helper's + // stand-in above; none came from the import itself. + expect(direct).toHaveBeenCalledTimes(helper.mock.calls.length); + const written = helper.mock.calls.flatMap(([, rows]) => [...rows]); + expect(written.map((r) => r.type)).toEqual( + expect.arrayContaining(["WEIGHT", "SLEEP_DURATION"]), + ); + expect(new Set(written.map((r) => r.source))).toEqual( + new Set(["APPLE_HEALTH"]), + ); + }); +}); + describe("streamParseExportXml — end-to-end", () => { it("ingests the tiny fixture into per-type / workout / deferred / unknown buckets", async () => { const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-")); diff --git a/src/lib/measurements/__tests__/import-large-export-memory.test.ts b/src/lib/measurements/__tests__/import-large-export-memory.test.ts index 941843fa6..e3c784041 100644 --- a/src/lib/measurements/__tests__/import-large-export-memory.test.ts +++ b/src/lib/measurements/__tests__/import-large-export-memory.test.ts @@ -43,6 +43,25 @@ vi.mock("@/lib/daily/morning-refresh-trigger", () => ({ maybeEnqueueMorningRefresh: async () => undefined, })); +// The spot flush writes through one raw `INSERT … ON CONFLICT DO NOTHING +// RETURNING` statement. These fakes model the table in memory, so route that +// statement to the fake's own skip-duplicates insert; the SQL itself is pinned +// against a real Postgres in tests/integration/measurement-bulk-insert.test.ts. +vi.mock("@/lib/export/measurement-bulk-insert", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/export/measurement-bulk-insert") + >()), + insertNewMeasurementRows: ( + db: { measurement: { createManyAndReturn: (args: unknown) => unknown } }, + rows: unknown[], + ) => + db.measurement.createManyAndReturn({ + data: rows, + skipDuplicates: true, + select: { id: true, type: true, measuredAt: true, externalId: true }, + }), +})); + vi.mock("@/lib/arrivals/emit-shared", () => ({ emitDataArrival: async () => undefined, })); diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index d175fc5c0..d76027feb 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -41,6 +41,11 @@ import { canonicalDailyTimestamp, } from "@/lib/measurements/drain-per-sample-cumulative"; import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement"; +import { + insertNewMeasurementRows, + type InsertedMeasurementRow, + type NewMeasurementRow, +} from "@/lib/export/measurement-bulk-insert"; import { resolveHkWorkoutSportType } from "@/lib/measurements/hk-workout-activity-type-map"; import { convertHkValue, @@ -309,7 +314,10 @@ export interface StreamParseInput { /** IANA timezone, used to anchor cumulative-type day-keys. */ userTimezone: string; /** Prisma client to flush rows through. */ - prisma: Pick; + prisma: Pick< + PrismaClient, + "measurement" | "workout" | "$transaction" | "$queryRawUnsafe" + >; /** * Live progress hook. Called every `PROGRESS_TICK_RECORDS` records * read, and once on terminal `done`. Best-effort; the parser @@ -457,55 +465,38 @@ export async function streamParseExportXml( type: MeasurementType; measuredAt: Date; }> = []; - const createData: Prisma.MeasurementCreateManyInput[] = chunk.map( - (row) => ({ - userId, - type: row.type, - value: row.value, - unit: row.unit, - source: "APPLE_HEALTH", - measuredAt: row.measuredAt, - externalId: row.externalId, - externalSourceVersion: row.externalSourceVersion, - sleepStage: row.sleepStage ?? null, - deviceType: row.deviceType, - }), - ); + // Written as one `INSERT … SELECT FROM unnest(…) ON CONFLICT DO NOTHING + // RETURNING` rather than `createManyAndReturn`: the Prisma call left about + // 5 MB of heap behind per flush until some two dozen had piled up, which + // is the import's whole budget in a small container. Same semantics — + // duplicates on either unique identity are skipped, never thrown — and the + // same answer: the rows that landed. See `insertNewMeasurementRows`. + const createData: NewMeasurementRow[] = chunk.map((row) => ({ + userId, + type: row.type, + value: row.value, + unit: row.unit, + source: "APPLE_HEALTH", + measuredAt: row.measuredAt, + externalId: row.externalId, + externalSourceVersion: row.externalSourceVersion, + sleepStage: row.sleepStage ?? null, + deviceType: row.deviceType, + })); const insertStartedAt = Date.now(); - let createdRows: Array<{ - id: string; - type: MeasurementType; - measuredAt: Date; - externalId: string | null; - }> = []; + let createdRows: InsertedMeasurementRow[] = []; const failedInsertIndexes = new Set(); try { - createdRows = await prisma.measurement.createManyAndReturn({ - data: createData, - skipDuplicates: true, - select: { - id: true, - type: true, - measuredAt: true, - externalId: true, - }, - }); + createdRows = await insertNewMeasurementRows(prisma, createData); } catch { // Retain the old per-sample failure isolation if an unexpected database // error rejects the bulk statement. Conflicts remain non-errors because - // every retry still uses skipDuplicates. + // every retry still skips duplicates. for (let index = 0; index < createData.length; index += 1) { try { - const created = await prisma.measurement.createManyAndReturn({ - data: [createData[index]], - skipDuplicates: true, - select: { - id: true, - type: true, - measuredAt: true, - externalId: true, - }, - }); + const created = await insertNewMeasurementRows(prisma, [ + createData[index], + ]); createdRows.push(...created); } catch { failedInsertIndexes.add(index); diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 1957d5bb0..49d374343 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -198,6 +198,7 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "managed_profile.not_found", ], measurement: [ + "measurement.batch.invalid", "measurement.batch.source_not_permitted", "measurement.batch.too_large", "measurement.bulk-delete.invalid", diff --git a/src/lib/openapi/routes/cycle.ts b/src/lib/openapi/routes/cycle.ts index fd37bebd5..2410b7f6d 100644 --- a/src/lib/openapi/routes/cycle.ts +++ b/src/lib/openapi/routes/cycle.ts @@ -437,7 +437,12 @@ const cycleBulkEntryResult = z.object({ status: z.enum(["inserted", "duplicate", "updated", "skipped"]), id: z.string().optional(), externalId: z.string().optional(), - reason: z.string().optional(), + reason: z + .string() + .optional() + .describe( + "On a `skipped` entry. `unstable_external_id` is a verdict on the entry and terminal: resending it unchanged is skipped again. `constraint` means the write met a database constraint (a key another write claimed at the same moment, or a reference that no longer exists). `upsert_failed` is a write failure on the server's side: nothing was stored, the entry should be sent again, and the response carrying it is `Cache-Control: no-store`, so a retry under the same `Idempotency-Key` writes again rather than replaying the failure.", + ), }); const cycleBulkResponse = z.object({ @@ -808,6 +813,11 @@ export const cyclePaths: NonNullable = { }, ...cycleDisabledResponse, ...stdResponses, + "422": { + description: + "The whole batch was refused and nothing was written: more than 500 entries (`meta.errorCode` = `cycle.bulk.too_large`, no issue list), or the body failed validation (`meta.errorCode` = `cycle.bulk.invalid`, every issue under `details.issues`). Both are permanent for the batch as sent. A problem with one entry never refuses the batch; it comes back as that entry's `skipped` status.", + content: { "application/json": { schema: errorEnvelope } }, + }, }, }, }, diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index e6dc81dff..d07c7cfec 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -10,6 +10,7 @@ import type { ZodOpenApiObject } from "zod-openapi"; import { updateMeasurementSchema } from "@/lib/validations/measurement"; import { seriesBatchQuerySchema } from "@/lib/validations/series-batch"; import { deviceTypeEnum } from "@/lib/validations/source-priority"; +import { sleepStageEntrySchema } from "@/lib/validations/sleep-stage-entry"; import { MODULE_DISABLED_DESCRIPTION, createMeasurementSchema, @@ -147,6 +148,22 @@ const batchEntryResult = z "`inserted`/`duplicate` — the row landed (advance the cursor). `updated` — a `stats:` aggregate overwrote an existing row. `skipped` — validation no-op; see `reason`. `failed` — retryable database failure that must not advance the entry cursor; the response is marked `Cache-Control: no-store`.", ), reason: z.string().optional(), + convertedValue: z + .number() + .optional() + .describe( + "Only on a `value_out_of_range` skip (since v1.39.1): the entry's value after conversion into the stored unit, which is the number the band was checked against. A HealthKit fraction that should have been a percent shows up here a hundred times off.", + ), + range: z + .object({ + min: z.number(), + max: z.number(), + unit: z.string(), + }) + .optional() + .describe( + "Only on a `value_out_of_range` skip (since v1.39.1): the plausibility band that refused the entry, in the stored unit. Both edges are inclusive. A skip is permanent for the value as sent; resending it unchanged skips it again.", + ), }) .meta({ id: "AppleHealthBatchEntryResult" }); @@ -313,6 +330,21 @@ export const measurementResource = z "Server-shaped measurement row. GET endpoints return the whole stored row with the decrypted note on `notes` and the `notesEncrypted` ciphertext stripped; the collapsed list modes synthesise rows carrying the display fields plus `dayKey` / `sampleCount`.", }); +/** The answer to a named-stage `POST /api/measurements` (since v1.39.1). */ +const namedSleepStageResult = measurementResource + .extend({ + status: z + .enum(["inserted", "updated", "duplicate"]) + .describe( + "`inserted` — a new row (HTTP 201). `duplicate` — the segment was already stored, by this route or the batch route, and nothing changed (HTTP 200). `updated` — a stored segment with the same stage, source and end was corrected in place, for example a later start (HTTP 200).", + ), + }) + .meta({ + id: "NamedSleepStageMeasurement", + description: + "The stored SLEEP_DURATION row for a named sleep-stage segment, with what the call did to it.", + }); + /** * The aggregate list modes (`aggregate=daily` and the coarser grains) return * pre-folded buckets, not stored rows — no `id`, `unit` or `source`. @@ -850,6 +882,7 @@ export const measurementPaths: NonNullable = { summary: "Create one measurement (or a small array)", description: "Single ingest. The body may also be a bare ARRAY of the same objects — the mode the iOS client uses for a combined blood-pressure + pulse or dual-value glucose write — in which case `data` is the array of created rows in request order. Use `/api/measurements/batch` for Apple Health upload streams.\n\n" + + "Sleep stages by name (since v1.39.1): an object with `type: SLEEP_DURATION`, a `sleepStage` of `IN_BED`, `ASLEEP`, `AWAKE`, `CORE`, `DEEP` or `REM`, and the segment as `startDate` + `endDate` or as `measuredAt` (its end) + `value` (minutes). It is stored and deduplicated exactly as the batch route stores a HealthKit sleep sample, so a bridge can send stages without the HealthKit codepoints. See `NamedSleepStageEntry`.\n\n" + "Also reachable with a narrow `measurements:write` Bearer, which is the supported way to push readings from a scale, a watch bridge or a home-automation rule. That credential is confined to its owner's own record — a request carrying the account selector is refused 403 before any sharing grant is read, however the record is actually shared — and it may not attribute a source at all: rows it writes carry `source: EXTERNAL`, resolved from the credential, so a bridged reading stays distinguishable from a hand-typed one. A body naming ANY source, `MANUAL` included, is refused 422 (`measurement.create.source_not_permitted`). The GET on this same path, and the PUT and DELETE beside it, name no scope and so refuse it — though the rows themselves stay editable by their owner, unlike a connected provider's.", parameters: [idempotencyKeyParameter], requestBody: { @@ -858,6 +891,11 @@ export const measurementPaths: NonNullable = { "application/json": { schema: z.union([ createMeasurementSchema, + sleepStageEntrySchema.meta({ + id: "NamedSleepStageEntry", + description: + "One sleep-stage segment with the stage named (since v1.39.1), for a bridge that does not speak HealthKit. Chosen whenever the object carries a string `sleepStage`. `type` is `SLEEP_DURATION`. The segment is either `startDate` + `endDate`, or `measuredAt` (its END) + `value` (its length in minutes); a `value` sent beside the dates must agree with them to within a minute. Stored as the batch route stores a HealthKit sleep sample: value in minutes, `measuredAt` at the end, the stage in `sleepStage`. Without an `externalId` the server derives one from the stage and both instants, so sending the same segment again is a `duplicate`, and a segment whose start moved is `updated` rather than added.", + }), z.array(createMeasurementSchema), ]), }, @@ -868,19 +906,35 @@ export const measurementPaths: NonNullable = { ...recordRefusal(), "201": { description: - "Created. A single resource for the object body; an array of resources, in request order, for the array body.", + "Created. A single resource for the object body; an array of resources, in request order, for the array body; for a named sleep-stage body, the stored row with `status: inserted`.", content: { "application/json": { schema: dataEnvelope( - z.union([measurementResource, z.array(measurementResource)]), + z.union([ + measurementResource, + z.array(measurementResource), + namedSleepStageResult, + ]), "CreateMeasurementResponse", ), }, }, }, + "200": { + description: + "Named sleep-stage body only: the segment was already stored (`status: duplicate`) or a stored one was corrected in place (`status: updated`). Every other body answers 201 or an error, as before.", + content: { + "application/json": { + schema: dataEnvelope( + namedSleepStageResult, + "NamedSleepStageDuplicateResponse", + ), + }, + }, + }, "409": { description: - "A measurement with this data already exists — the `(userId, type, source, externalId)` dedup index rejected the write.", + "A measurement with this data already exists — the `(userId, type, source, externalId)` dedup index rejected the write. Never for a named sleep-stage body, which answers a re-post with 200 instead.", content: { "application/json": { schema: errorEnvelope } }, }, // After the spread: `stdResponses` carries a generic 422 that would @@ -1011,7 +1065,7 @@ export const measurementPaths: NonNullable = { ...stdResponses, "422": { description: - "The batch exceeded the 500-entry limit (`measurement.batch.too_large`), failed validation, or (`measurement.batch.source_not_permitted`) carried an entry naming a source — any source — under a narrow `measurements:write` credential. Nothing was written in any of the three cases.", + "The batch exceeded the 500-entry limit (`measurement.batch.too_large`), failed validation (`measurement.batch.invalid`, with every issue under `details.issues`), or (`measurement.batch.source_not_permitted`) carried an entry naming a source, any source, under a narrow `measurements:write` credential. Nothing was written in any of the three cases, and all three are permanent for the batch as sent: resending it unchanged is refused the same way.", content: { "application/json": { schema: errorEnvelope } }, }, }, diff --git a/src/lib/openapi/routes/nutrients.ts b/src/lib/openapi/routes/nutrients.ts index 99f170029..a1256ad3e 100644 --- a/src/lib/openapi/routes/nutrients.ts +++ b/src/lib/openapi/routes/nutrients.ts @@ -56,7 +56,7 @@ export const nutrientPaths: NonNullable = { tags: ["Nutrients"], summary: "Ingest micronutrient day totals (v1.28)", description: - "Bulk upsert of supplement-style daily totals synced from HealthKit — vitamins, minerals, water, caffeine. Day totals only: compute HKStatisticsCollectionQuery cumulative sums day-anchored in the user's current IANA timezone; `day` is yyyy-MM-dd in that timezone (the same rule as the `stats:` day keys). Never post raw samples. Upsert key is (day, nutrient): a re-post replaces the stored total and reports `updated`. Post the current + previous local day per sync; backfill 30 days on first enable. Max 500 entries; Idempotency-Key supported; rate limit 60/min. Per-entry skips (`unit_mismatch` | `value_out_of_range` | `day_invalid`) are terminal — log and drop, do not retry the entry. Behind the opt-in `nutrients` module (403 `module.disabled` when off). Query HealthKit with exactly the listed unit per code (`ug` = microgram; do not convert to IU):\n\n| code | HealthKit identifier | unit |\n| --- | --- | --- |\n" + + "Bulk upsert of supplement-style daily totals synced from HealthKit — vitamins, minerals, water, caffeine. Day totals only: compute HKStatisticsCollectionQuery cumulative sums day-anchored in the user's current IANA timezone; `day` is yyyy-MM-dd in that timezone (the same rule as the `stats:` day keys). Never post raw samples. Upsert key is (day, nutrient): a re-post replaces the stored total and reports `updated`. Post the current + previous local day per sync; backfill 30 days on first enable. Max 500 entries; Idempotency-Key supported; rate limit 60/min. Per-entry skips (`unit_mismatch` | `value_out_of_range` | `day_invalid`) are terminal — log and drop, do not retry the entry. `upsert_failed` is not one of them: it is a server-side write failure for the entry's whole write group, nothing was stored, and the entry should be sent again on a later sync; a response carrying it is `Cache-Control: no-store` and is not replayed for its `Idempotency-Key`. Behind the opt-in `nutrients` module (403 `module.disabled` when off). Query HealthKit with exactly the listed unit per code (`ug` = microgram; do not convert to IU):\n\n| code | HealthKit identifier | unit |\n| --- | --- | --- |\n" + catalogTable, requestBody: { required: true, diff --git a/src/lib/openapi/routes/profile.ts b/src/lib/openapi/routes/profile.ts index f6b1dba67..b1fa0100d 100644 --- a/src/lib/openapi/routes/profile.ts +++ b/src/lib/openapi/routes/profile.ts @@ -498,7 +498,11 @@ const profileResponse = z gender: z.enum(["MALE", "FEMALE", "OTHER"]).nullable(), heightCm: z.number().nullable(), locale: z.string().nullable(), - timezone: z.string(), + timezone: z + .string() + .describe( + "The IANA zone the server cuts this account's days in, the same value `/api/auth/me` reports: the stored zone, or the instance default when the stored value is not a usable IANA name (since v1.39.1; before, the raw stored value). Always a valid IANA name, so sending it back in a PATCH is accepted.", + ), timeFormat: z .enum(["AUTO", "H12", "H24"]) .describe( @@ -542,7 +546,11 @@ const profileUpdateResponse = z gender: z.enum(["MALE", "FEMALE", "OTHER"]).nullable(), heightCm: z.number().nullable(), locale: z.string().nullable(), - timezone: z.string(), + timezone: z + .string() + .describe( + "The IANA zone the server cuts this account's days in, the same value `/api/auth/me` reports: the stored zone, or the instance default when the stored value is not a usable IANA name (since v1.39.1; before, the raw stored value). Always a valid IANA name, so sending it back in a PATCH is accepted.", + ), timeFormat: z .enum(["AUTO", "H12", "H24"]) .describe( diff --git a/src/lib/sources/__tests__/source-priority-whoop-applehealth.test.ts b/src/lib/sources/__tests__/source-priority-whoop-applehealth.test.ts index aecd29f61..73996d51e 100644 --- a/src/lib/sources/__tests__/source-priority-whoop-applehealth.test.ts +++ b/src/lib/sources/__tests__/source-priority-whoop-applehealth.test.ts @@ -150,8 +150,13 @@ describe("WHOOP-written HealthKit records remain source-agnostic", () => { }); it("persists supported export.xml rows as APPLE_HEALTH, never as a raw writer label", () => { + // Spot rows are built as `NewMeasurementRow`s and written through the + // column-array bulk insert; the source is fixed where each row is built. expect(appleImporterSource).toMatch( - /measurement\.createManyAndReturn\([\s\S]*source:\s*"APPLE_HEALTH"/, + /const createData: NewMeasurementRow\[\] = chunk\.map\(\(row\) => \(\{[^}]*source:\s*"APPLE_HEALTH"/, + ); + expect(appleImporterSource).toMatch( + /insertNewMeasurementRows\(prisma, createData\)/, ); expect(appleImporterSource).not.toMatch( /source:\s*attrs\.(?:sourceName|device)/, diff --git a/src/lib/tz/resolver.ts b/src/lib/tz/resolver.ts index 1835432d5..6f3034dc5 100644 --- a/src/lib/tz/resolver.ts +++ b/src/lib/tz/resolver.ts @@ -144,6 +144,21 @@ export async function resolveServerDefaultTimezone(): Promise { return tz; } +/** + * The zone the server actually uses for a stored `User.timezone` value: the + * value itself when it is a usable IANA name, the instance default otherwise. + * + * Every response that reports an account's zone goes through this, so a + * client never buckets days by a value the server itself has set aside. + */ +export async function resolveStoredTimezone( + stored: string | null | undefined, +): Promise { + return stored && isValidTimezone(stored) + ? stored + : resolveServerDefaultTimezone(); +} + /** * Resolve the per-user display timezone. Reads `User.timezone`. Falls * back to the server default and then to "Europe/Berlin". Cached for diff --git a/src/lib/validations/nutrients.ts b/src/lib/validations/nutrients.ts index a48fc76af..e65014ed6 100644 --- a/src/lib/validations/nutrients.ts +++ b/src/lib/validations/nutrients.ts @@ -64,7 +64,7 @@ export const nutrientEntryResultSchema = z .meta({ id: "NutrientEntryResult", description: - "Per-entry ingest outcome. skipped reasons: unit_mismatch | value_out_of_range | day_invalid | upsert_failed. Log and drop a skipped entry — do not retry it.", + "Per-entry ingest outcome. skipped reasons: unit_mismatch | value_out_of_range | day_invalid | upsert_failed. The first three are verdicts on the entry itself and terminal: resending it unchanged is skipped again, so log and drop it. `upsert_failed` is a write failure on the server's side, not a verdict on the entry: nothing was stored for it, and it should be sent again on a later sync. A write failure takes out the whole group it was written in (every new day total, or every existing one), so several entries can carry it at once. A response carrying it is marked `Cache-Control: no-store` and is not kept for `Idempotency-Key` replay, so the retry writes again rather than replaying the failure.", }); export const nutrientBatchResponseSchema = z diff --git a/src/lib/validations/sleep-stage-entry.ts b/src/lib/validations/sleep-stage-entry.ts new file mode 100644 index 000000000..3949c5390 --- /dev/null +++ b/src/lib/validations/sleep-stage-entry.ts @@ -0,0 +1,210 @@ +/** + * One sleep-stage segment, named rather than coded, for `POST /api/measurements`. + * + * Until v1.39.1 a stage could only arrive through the batch route, in the + * HealthKit dialect: `hkIdentifier: HKCategoryTypeIdentifierSleepAnalysis`, a + * `sleepStage` codepoint 0..5 whose meaning is Apple's, and a value the client + * had to work out in minutes. A bridge such as Tasker or Home Assistant knows + * none of that. This shape lets it say what it has: the stage by name, and the + * segment either as its start and end or as its end and length. + * + * The row that lands is the row the batch route would have written for the + * same segment: `SLEEP_DURATION`, value in minutes, `measuredAt` at the + * segment's END (Apple's convention, which every sleep reader assumes), the + * stage in `sleepStage`. Dedupe is the batch route's too: the + * `(type, source, externalId)` identity and the + * `(type, measuredAt, source, sleepStage)` identity, reconciled in one step. + */ +import { z } from "zod/v4"; + +import { validateEntryInstant } from "./entry-instant"; +import { assertStableExternalId } from "@/lib/validations/external-id"; +import { + validateMeasurementRange, + writableMeasurementSourceEnum, +} from "./measurement"; + +/** The stage names the `SleepStage` column stores, as a caller writes them. */ +export const SLEEP_STAGE_NAMES = [ + "IN_BED", + "ASLEEP", + "AWAKE", + "CORE", + "DEEP", + "REM", +] as const; + +export type SleepStageName = (typeof SLEEP_STAGE_NAMES)[number]; + +/** A difference of up to a minute between `value` and the dates is rounding. */ +const VALUE_TOLERANCE_MINUTES = 1; + +const instant = () => + validateEntryInstant( + z.iso.datetime({ offset: true }).transform((s) => new Date(s)), + ); + +export const sleepStageEntrySchema = z + .object({ + type: z.literal("SLEEP_DURATION"), + sleepStage: z.enum(SLEEP_STAGE_NAMES), + startDate: instant().optional(), + endDate: instant().optional(), + /** The segment's END, when it is sent as an end and a length. */ + measuredAt: instant().optional(), + /** Minutes. Required with `measuredAt`; optional with start and end. */ + value: z.number().finite().optional(), + externalId: z.string().min(1).max(120).optional(), + source: writableMeasurementSourceEnum.optional().default("MANUAL"), + deviceType: z.string().min(1).max(32).nullable().optional(), + }) + .superRefine((entry, ctx) => { + assertStableExternalId(entry, ctx); + + const hasDates = + entry.startDate !== undefined || entry.endDate !== undefined; + const hasEndAndLength = entry.measuredAt !== undefined; + if (hasDates && hasEndAndLength) { + ctx.addIssue({ + code: "custom", + path: ["measuredAt"], + message: + "Send the segment either as startDate and endDate, or as measuredAt (its end) and value (its length in minutes), not both", + }); + return; + } + if (hasDates) { + if (entry.startDate === undefined || entry.endDate === undefined) { + ctx.addIssue({ + code: "custom", + path: [entry.startDate === undefined ? "startDate" : "endDate"], + message: "startDate and endDate are sent together", + }); + return; + } + if (entry.endDate < entry.startDate) { + ctx.addIssue({ + code: "custom", + path: ["endDate"], + message: "endDate must not be before startDate", + }); + return; + } + const minutes = + (entry.endDate.getTime() - entry.startDate.getTime()) / 60_000; + if ( + entry.value !== undefined && + Math.abs(entry.value - minutes) > VALUE_TOLERANCE_MINUTES + ) { + ctx.addIssue({ + code: "custom", + path: ["value"], + message: + "value disagrees with startDate and endDate; send the dates alone, or a value that matches them", + }); + return; + } + if (validateMeasurementRange("SLEEP_DURATION", minutes) !== null) { + ctx.addIssue({ + code: "custom", + path: ["endDate"], + message: "A sleep segment lasts between 0 and 1440 minutes", + }); + } + return; + } + if (hasEndAndLength) { + if (entry.value === undefined) { + ctx.addIssue({ + code: "custom", + path: ["value"], + message: + "value (the segment's length in minutes) is required with measuredAt", + }); + return; + } + if (validateMeasurementRange("SLEEP_DURATION", entry.value) !== null) { + ctx.addIssue({ + code: "custom", + path: ["value"], + message: "A sleep segment lasts between 0 and 1440 minutes", + }); + } + return; + } + ctx.addIssue({ + code: "custom", + path: ["startDate"], + message: + "Send the segment as startDate and endDate, or as measuredAt (its end) and value (its length in minutes)", + }); + }); + +export type SleepStageEntry = z.infer; + +/** A parsed entry, reduced to the segment it describes. */ +export interface SleepStageSegment { + stage: SleepStageName; + start: Date; + end: Date; + minutes: number; + /** + * The caller's id, or one derived from the stage and the two instants, so a + * bridge that sends no id still re-posts onto the same row. + */ + externalId: string; +} + +/** + * The id a segment gets when the caller sends none. The stage and both + * instants, so the same segment sent twice is one row, and a segment whose + * start was corrected keeps its end (and so its natural identity) and is + * updated rather than duplicated. + */ +export function derivedSleepStageExternalId( + stage: SleepStageName, + start: Date, + end: Date, +): string { + return `sleep-stage:${stage}:${start.toISOString()}/${end.toISOString()}`; +} + +export function toSleepStageSegment(entry: SleepStageEntry): SleepStageSegment { + let start: Date; + let end: Date; + let minutes: number; + if (entry.startDate && entry.endDate) { + start = entry.startDate; + end = entry.endDate; + minutes = (end.getTime() - start.getTime()) / 60_000; + } else { + // The refinement guarantees the end-and-length form here. + end = entry.measuredAt!; + minutes = entry.value!; + start = new Date(end.getTime() - Math.round(minutes * 60_000)); + } + return { + stage: entry.sleepStage, + start, + end, + minutes, + externalId: + entry.externalId ?? + derivedSleepStageExternalId(entry.sleepStage, start, end), + }; +} + +/** + * Whether a single-object body asks for the named-stage form. Only a string + * `sleepStage` does. The plain create schema never read the field, so a body + * that carried one before was stored without its stage; it is now stored with + * it, or refused with the reason. + */ +export function isNamedSleepStageBody(body: unknown): boolean { + return ( + typeof body === "object" && + body !== null && + !Array.isArray(body) && + typeof (body as { sleepStage?: unknown }).sleepStage === "string" + ); +} diff --git a/tests/integration/measurement-bulk-insert.test.ts b/tests/integration/measurement-bulk-insert.test.ts index c0caae7b1..81379639e 100644 --- a/tests/integration/measurement-bulk-insert.test.ts +++ b/tests/integration/measurement-bulk-insert.test.ts @@ -10,7 +10,10 @@ import { beforeEach, describe, expect, it } from "vitest"; import { getPrismaClient, truncateAllTables } from "./setup"; import { insertMeasurementRows, + insertNewMeasurementRows, + newMeasurementId, type MeasurementInsertRow, + type NewMeasurementRow, } from "@/lib/export/measurement-bulk-insert"; const USER = "user-bulk-insert"; @@ -137,3 +140,164 @@ describe("insertMeasurementRows", () => { ); }); }); + +/** + * `insertNewMeasurementRows` replaced `createManyAndReturn({ skipDuplicates })` + * in the Apple Health export import. Each case below runs the same input + * through both and compares what they return and what they leave in the + * table, so a difference in either direction fails. + */ +describe("insertNewMeasurementRows", () => { + function spot( + externalId: string, + at: string, + overrides: Partial = {}, + ): NewMeasurementRow { + return { + userId: USER, + type: "PULSE", + value: 61, + unit: "bpm", + source: "APPLE_HEALTH", + measuredAt: new Date(at), + externalId, + externalSourceVersion: "17.4", + sleepStage: null, + deviceType: null, + ...overrides, + }; + } + + const SELECT = { + id: true, + type: true, + measuredAt: true, + externalId: true, + } as const; + + /** Run `rows` through Prisma, then through the raw path in a fresh table. */ + async function both(seed: NewMeasurementRow[], rows: NewMeasurementRow[]) { + const prisma = getPrismaClient(); + const run = async ( + write: (r: NewMeasurementRow[]) => Promise< + Array<{ + id: string; + type: string; + measuredAt: Date; + externalId: string | null; + }> + >, + ) => { + await prisma.measurement.deleteMany({ where: { userId: USER } }); + if (seed.length > 0) { + await prisma.measurement.createMany({ data: seed as never }); + } + const returned = await write(rows); + const table = await prisma.measurement.findMany({ + where: { userId: USER }, + orderBy: [{ measuredAt: "asc" }, { externalId: "asc" }], + }); + return { + returned: returned + .map(({ id: _id, ...rest }) => rest) + .sort((a, b) => `${a.externalId}`.localeCompare(`${b.externalId}`)), + table: table.map( + ({ id: _id, createdAt: _c, updatedAt: _u, ...rest }) => rest, + ), + ids: returned.map((r) => r.id), + }; + }; + const viaPrisma = await run((r) => + prisma.measurement.createManyAndReturn({ + data: r as never, + skipDuplicates: true, + select: SELECT, + }), + ); + const viaRaw = await run((r) => insertNewMeasurementRows(prisma, r)); + return { viaPrisma, viaRaw }; + } + + it("returns and stores exactly what createManyAndReturn does", async () => { + const rows = [ + spot("hr-1", "2026-03-01T08:00:00.123Z"), + spot("sleep-1", "2026-03-01T06:00:00.000Z", { + type: "SLEEP_DURATION", + value: 42.5, + unit: "minutes", + sleepStage: "REM", + deviceType: "watch", + }), + spot("spo2-1", "2026-03-01T09:00:00.000Z", { + type: "OXYGEN_SATURATION", + value: 97, + unit: "%", + externalSourceVersion: null, + }), + ]; + const { viaPrisma, viaRaw } = await both([], rows); + expect(viaRaw.returned).toEqual(viaPrisma.returned); + expect(viaRaw.table).toEqual(viaPrisma.table); + expect(viaRaw.returned).toHaveLength(3); + for (const r of viaRaw.returned) expect(r.measuredAt).toBeInstanceOf(Date); + }); + + it("skips a row that meets an existing row on either unique identity", async () => { + const seed = [ + // Same external id as `ext-dup`. + spot("ext-dup", "2026-03-02T08:00:00.000Z", { value: 70 }), + // Same (type, measured_at, source, sleep_stage) as `nat-dup`. + spot("seeded-natural", "2026-03-02T09:00:00.000Z", { value: 71 }), + ]; + const rows = [ + spot("ext-dup", "2026-03-02T10:00:00.000Z", { value: 80 }), + spot("nat-dup", "2026-03-02T09:00:00.000Z", { value: 81 }), + spot("fresh", "2026-03-02T11:00:00.000Z", { value: 82 }), + ]; + const { viaPrisma, viaRaw } = await both(seed, rows); + expect(viaRaw.returned).toEqual(viaPrisma.returned); + expect(viaRaw.table).toEqual(viaPrisma.table); + expect(viaRaw.returned.map((r) => r.externalId)).toEqual(["fresh"]); + }); + + it("keeps the first of two rows in one call that share an identity", async () => { + const rows = [ + spot("twin", "2026-03-03T08:00:00.000Z", { value: 90 }), + spot("twin", "2026-03-03T08:30:00.000Z", { value: 91 }), + spot("other", "2026-03-03T08:00:00.000Z", { value: 92 }), + ]; + const { viaPrisma, viaRaw } = await both([], rows); + expect(viaRaw.returned).toEqual(viaPrisma.returned); + expect(viaRaw.table).toEqual(viaPrisma.table); + expect(viaRaw.returned).toHaveLength(1); + }); + + it("mints ids in the shape the cuid() default produces", async () => { + const prisma = getPrismaClient(); + await prisma.measurement.deleteMany({ where: { userId: USER } }); + const viaPrisma = await prisma.measurement.create({ + data: spot("shape-prisma", "2026-03-04T08:00:00.000Z") as never, + }); + const [viaRaw] = await insertNewMeasurementRows(prisma, [ + spot("shape-raw", "2026-03-04T09:00:00.000Z"), + ]); + const shape = /^c[0-9a-z]{24}$/; + expect(viaPrisma.id).toMatch(shape); + expect(viaRaw!.id).toMatch(shape); + const ids = new Set(Array.from({ length: 10_000 }, newMeasurementId)); + expect(ids.size).toBe(10_000); + const stored = await prisma.measurement.findUniqueOrThrow({ + where: { id: viaRaw!.id }, + }); + expect(stored.syncVersion).toBe(1); + expect(stored.deletedAt).toBeNull(); + expect(Math.abs(Date.now() - stored.createdAt.getTime())).toBeLessThan( + 60_000, + ); + expect(stored.updatedAt.getTime()).toBe(stored.createdAt.getTime()); + }); + + it("answers an empty call without touching the database", async () => { + expect(await insertNewMeasurementRows(getPrismaClient(), [])).toEqual([]); + }); +}); diff --git a/tests/integration/measurements-named-sleep-stage.test.ts b/tests/integration/measurements-named-sleep-stage.test.ts new file mode 100644 index 000000000..809206262 --- /dev/null +++ b/tests/integration/measurements-named-sleep-stage.test.ts @@ -0,0 +1,398 @@ +/** + * `POST /api/measurements` with a named `sleepStage`, against real Postgres. + * + * A bridge (Tasker, Home Assistant) sends one segment with the stage by name + * and its narrow `measurements:write` token. What this pins: + * + * - the row is the row the batch route writes for the same segment: + * SLEEP_DURATION, minutes, `measuredAt` at the segment's end, the stage; + * - re-posting the same segment is one row, whether the bridge sends an id + * or not, and whichever of the two routes it used first; + * - a corrected start updates the row instead of adding a second one; + * - the scoped token's source rule still holds: rows carry EXTERNAL, and a + * body naming any source is refused; + * - a body the shape does not describe is a 422 with every issue listed. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +process.env.API_TOKEN_HMAC_KEY ??= + "test-hmac-key-named-sleep-stage-ingest-32-bytes-min-0123456789"; + +const { hashToken } = await import("@/lib/auth/hmac"); + +const USER_ID = "user-named-sleep-stage"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const { POST } = await import("@/app/api/measurements/route"); +const { POST: POST_BATCH } = await import("@/app/api/measurements/batch/route"); + +let token = ""; + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + cookieJar.clear(); + headerJar.clear(); + await prisma.user.create({ + data: { + id: USER_ID, + username: "named-sleep", + email: "named-sleep@example.test", + timezone: "UTC", + }, + }); + token = `hlk_sleepstage_${"0".repeat(48)}`; + await prisma.apiToken.create({ + data: { + userId: USER_ID, + name: "bridge", + tokenHash: hashToken(token), + permissions: ["measurements:write"], + }, + }); + headerJar.set("authorization", `Bearer ${token}`); +}); + +function post(body: unknown, url = "https://health.example/api/measurements") { + return new NextRequest(url, { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + }, + body: JSON.stringify(body), + }); +} + +const END = new Date(Date.now() - 6 * 3_600_000); +END.setUTCSeconds(0, 0); +const START = new Date(END.getTime() - 42 * 60_000); + +function rows() { + return getPrismaClient().measurement.findMany({ + where: { userId: USER_ID, type: "SLEEP_DURATION" }, + orderBy: { measuredAt: "asc" }, + }); +} + +type Created = { + data: { + id: string; + status: string; + type: string; + value: number; + unit: string; + sleepStage: string | null; + source: string; + measuredAt: string; + externalId: string | null; + }; +}; + +describe("POST /api/measurements — named sleep stage", () => { + it("stores a start/end segment as the batch route would", async () => { + const res = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + expect(res.status).toBe(201); + const body = (await res.json()) as Created; + expect(body.data).toMatchObject({ + status: "inserted", + type: "SLEEP_DURATION", + value: 42, + unit: "minutes", + sleepStage: "REM", + source: "EXTERNAL", + measuredAt: END.toISOString(), + }); + const [row] = await rows(); + expect(row).toMatchObject({ + value: 42, + unit: "minutes", + sleepStage: "REM", + source: "EXTERNAL", + }); + expect(row!.measuredAt.toISOString()).toBe(END.toISOString()); + expect(row!.externalId).toBe( + `sleep-stage:REM:${START.toISOString()}/${END.toISOString()}`, + ); + }); + + it("accepts the end-and-length form and lands on the same row", async () => { + const first = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "DEEP", + measuredAt: END.toISOString(), + value: 42, + }), + ); + expect(first.status).toBe(201); + const second = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "DEEP", + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + expect(second.status).toBe(200); + expect(((await second.json()) as Created).data.status).toBe("duplicate"); + expect(await rows()).toHaveLength(1); + }); + + it("answers a re-post of the same segment with the stored row, not a second one", async () => { + const body = { + type: "SLEEP_DURATION", + sleepStage: "CORE", + startDate: START.toISOString(), + endDate: END.toISOString(), + }; + const first = (await (await POST(post(body))).json()) as Created; + const again = await POST(post(body)); + expect(again.status).toBe(200); + const second = (await again.json()) as Created; + expect(second.data.status).toBe("duplicate"); + expect(second.data.id).toBe(first.data.id); + expect(await rows()).toHaveLength(1); + }); + + it("updates the row when the same segment comes back with a corrected start", async () => { + await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "AWAKE", + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + const corrected = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "AWAKE", + startDate: new Date(START.getTime() + 12 * 60_000).toISOString(), + endDate: END.toISOString(), + }), + ); + expect(corrected.status).toBe(200); + expect(((await corrected.json()) as Created).data.status).toBe("updated"); + const all = await rows(); + expect(all).toHaveLength(1); + expect(all[0]!.value).toBe(30); + }); + + it("keeps two stages that end at the same instant apart", async () => { + for (const sleepStage of ["IN_BED", "ASLEEP"]) { + const res = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage, + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + expect(res.status).toBe(201); + } + expect((await rows()).map((r) => r.sleepStage).sort()).toEqual([ + "ASLEEP", + "IN_BED", + ]); + }); + + it("dedupes against the batch route: same source, same id, one row", async () => { + const externalId = "tasker-night-1-rem"; + const viaBatch = await POST_BATCH( + post( + { + entries: [ + { + hkIdentifier: "HKCategoryTypeIdentifierSleepAnalysis", + value: 42, + unit: "min", + startDate: START.toISOString(), + endDate: END.toISOString(), + sleepStage: 5, + externalId, + }, + ], + }, + "https://health.example/api/measurements/batch", + ), + ); + expect(viaBatch.status).toBe(200); + const viaNamed = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + externalId, + }), + ); + expect(viaNamed.status).toBe(200); + expect(((await viaNamed.json()) as Created).data.status).toBe("duplicate"); + const all = await rows(); + expect(all).toHaveLength(1); + expect(all[0]).toMatchObject({ sleepStage: "REM", source: "EXTERNAL" }); + }); + + it("keeps a bridge segment apart from the same reading from Apple Health", async () => { + // A scoped credential's rows carry their own source (EXTERNAL), so a + // bridge and the phone reporting the same night are two sources, and + // the sleep source priority picks one per night. Only MANUAL and + // APPLE_HEALTH merge as one reading. + await getPrismaClient().measurement.create({ + data: { + userId: USER_ID, + type: "SLEEP_DURATION", + value: 42, + unit: "minutes", + source: "APPLE_HEALTH", + measuredAt: END, + sleepStage: "REM", + externalId: "hk-uuid-1", + }, + }); + const res = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + }), + ); + expect(res.status).toBe(201); + const all = await rows(); + expect(all.map((r) => r.source).sort()).toEqual([ + "APPLE_HEALTH", + "EXTERNAL", + ]); + }); + + it("refuses a body naming MANUAL from the scoped token", async () => { + const res = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + source: "MANUAL", + }), + ); + expect(res.status).toBe(422); + expect(await rows()).toHaveLength(0); + }); + + it("refuses APPLE_HEALTH from the scoped token", async () => { + const res = await POST( + post({ + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + source: "APPLE_HEALTH", + }), + ); + expect(res.status).toBe(422); + const body = (await res.json()) as { meta?: { errorCode?: string } }; + expect(body.meta?.errorCode).toBe( + "measurement.create.source_not_permitted", + ); + expect(await rows()).toHaveLength(0); + }); + + it("refuses an unknown stage, a missing end, a mismatched value and an unstable id, listing each", async () => { + for (const body of [ + { + type: "SLEEP_DURATION", + sleepStage: "LIGHT", + startDate: START.toISOString(), + endDate: END.toISOString(), + }, + { + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + }, + { + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + value: 90, + }, + { + type: "SLEEP_DURATION", + sleepStage: "REM", + startDate: START.toISOString(), + endDate: END.toISOString(), + externalId: "0xdeadbeef", + }, + { + type: "WEIGHT", + sleepStage: "REM", + measuredAt: END.toISOString(), + value: 70, + }, + ]) { + const res = await POST(post(body)); + expect(res.status).toBe(422); + const parsed = (await res.json()) as { + details?: { issues: unknown[] }; + meta?: { errorCode?: string }; + }; + expect(parsed.meta?.errorCode).toBe("measurement.create.invalid"); + expect(parsed.details?.issues.length).toBeGreaterThan(0); + } + expect(await rows()).toHaveLength(0); + }); + + it("leaves the plain single create untouched", async () => { + const res = await POST( + post({ + type: "SLEEP_DURATION", + value: 420, + measuredAt: END.toISOString(), + }), + ); + expect(res.status).toBe(201); + const body = (await res.json()) as { + data: Record; + }; + expect(body.data).not.toHaveProperty("status"); + expect(body.data.sleepStage).toBeNull(); + }); +});