From a0f6b89e92752eebb161dfadf322619275b70ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 16 Oct 2026 00:18:35 +0200 Subject: [PATCH] feat(measurements): take hourly sums of summed types into the intraday profile and a backfill trigger that moves no live stamp; fix(backup): restore qualitative lab markers --- docs/api/openapi.yaml | 456 +++++++++++------- messages/de.json | 3 +- messages/en.json | 3 +- messages/es.json | 3 +- messages/fr.json | 3 +- messages/it.json | 3 +- messages/ko.json | 3 +- messages/pl.json | 3 +- prisma/schema.prisma | 24 +- .../client-threshold-literal-guard.test.ts | 356 ++++++++++++++ .../pulse-day-mean-reader-guard.test.ts | 4 + .../resolved-field-consumer-guard.test.ts | 131 +++++ src/app/api/auth/me/route.ts | 4 +- .../healthkit/__tests__/route.test.ts | 62 +++ src/app/api/measurements/batch/route.ts | 226 ++++++++- .../bulk/__tests__/dose-arrival-scope.test.ts | 35 +- src/app/api/medications/intake/bulk/route.ts | 13 +- .../__tests__/apple-health-card.test.tsx | 25 + .../integrations/apple-health-card.tsx | 24 +- .../__tests__/intraday-profile-backup.test.ts | 64 +++ src/lib/export/intraday-profile-backup.ts | 37 +- .../integrations/connection-health-load.ts | 8 +- src/lib/integrations/healthkit-type-sync.ts | 99 +++- .../__tests__/intraday-hour-buckets.test.ts | 160 ++++++ .../intraday-cumulative-profile.ts | 45 +- src/lib/measurements/intraday-hour-buckets.ts | 271 +++++++++++ src/lib/openapi/routes/cycle.ts | 22 +- src/lib/openapi/routes/measurements.ts | 9 +- src/lib/openapi/routes/medications/paths.ts | 4 +- src/lib/validations/__tests__/backup.test.ts | 63 +++ src/lib/validations/backup.ts | 55 ++- .../backup-qualitative-lab-restore.test.ts | 137 ++++++ tests/integration/backup-round-trip.test.ts | 15 +- .../hour-buckets-and-backfill.test.ts | 384 +++++++++++++++ 34 files changed, 2468 insertions(+), 286 deletions(-) create mode 100644 src/__tests__/client-threshold-literal-guard.test.ts create mode 100644 src/lib/measurements/__tests__/intraday-hour-buckets.test.ts create mode 100644 src/lib/measurements/intraday-hour-buckets.ts create mode 100644 tests/integration/backup-qualitative-lab-restore.test.ts create mode 100644 tests/integration/hour-buckets-and-backfill.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 27982c45f..3c7e6bc22 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -447,17 +447,36 @@ paths: tags: - Cycle summary: Soft-delete a cycle day-log (v1.15.0) - description: Sets `deletedAt` + bumps `syncVersion`; surfaces as a tombstone on the next `/api/sync/changes` page. 204. - Idempotent. + description: "Sets `deletedAt` + bumps `syncVersion`; surfaces as a tombstone on the next `/api/sync/changes` page. A + day that opened a cycle takes that cycle with it, which gives back any start that cycle had folded in. 204, or + 200 with `CycleDeleteOutcome` under `Prefer: return=representation`. Idempotent." parameters: - in: path name: id schema: type: string required: true + - name: Prefer + in: header + required: false + schema: + type: string + example: return=representation + description: "v1.43 — send `return=representation` (RFC 7240) to get 200 with what the delete changed + (`CycleDeleteOutcome`) and a `Preference-Applied: return=representation` header. Without it the route + answers 204 with no body, as before." responses: "200": - description: "v1.43 — with `Prefer: return=representation`: what the delete changed." + description: "v1.43 — with `Prefer: return=representation`: the removed cycle (null when the delete removed none, e.g. a + day that opened no cycle or a repeated delete) and the starts the removal gave back." + headers: + Preference-Applied: + required: true + description: "The route honoured `Prefer: return=representation`." + schema: + type: string + const: return=representation + description: "The route honoured `Prefer: return=representation`." content: application/json: schema: @@ -618,16 +637,35 @@ paths: tags: - Cycle summary: Soft-delete a menstrual cycle (v1.15.0) - description: Sets `deletedAt` + bumps `syncVersion`; tombstones on the next sync page. 204. Idempotent. + description: "Sets `deletedAt` + bumps `syncVersion`; tombstones on the next sync page. Any start this cycle had folded + in stands again. 204, or 200 with `CycleDeleteOutcome` under `Prefer: return=representation`. Idempotent." parameters: - in: path name: id schema: type: string required: true + - name: Prefer + in: header + required: false + schema: + type: string + example: return=representation + description: "v1.43 — send `return=representation` (RFC 7240) to get 200 with what the delete changed + (`CycleDeleteOutcome`) and a `Preference-Applied: return=representation` header. Without it the route + answers 204 with no body, as before." responses: "200": - description: "v1.43 — with `Prefer: return=representation`: what the delete changed." + description: "v1.43 — with `Prefer: return=representation`: the removed cycle (null when the delete removed none, e.g. a + day that opened no cycle or a repeated delete) and the starts the removal gave back." + headers: + Preference-Applied: + required: true + description: "The route honoured `Prefer: return=representation`." + schema: + type: string + const: return=representation + description: "The route honoured `Prefer: return=representation`." content: application/json: schema: @@ -4733,59 +4771,6 @@ paths: "403": *a13 "422": *a4 "429": *a5 - /api/personal-records: - get: - tags: - - Awards - summary: List detected personal bests - description: >- - The record's personal bests, newest achievement first. Read-only — rows come from the detection worker and no - endpoint creates or edits one. - - - Both query parameters are validated and a bad value is a 422. They were forgiving until this release, and in one - direction that was worse than merely quiet: dropping an unrecognised `metricType` WIDENED the read to every - metric, so a typo returned more rows than the caller asked for. The `limit` ceiling refuses rather than clamps - for the same reason. - - - Delegable on a `measurements` grant. - parameters: - - name: metricType - in: query - required: false - schema: - type: string - description: A `MeasurementType` to filter to. A value outside the enum is refused with 422 — it is not dropped, because - dropping it would widen the read rather than narrow it. - - name: limit - in: query - required: false - schema: - type: integer - minimum: 1 - maximum: 500 - default: 100 - description: Page cap, default 100, ceiling 500. A non-integer, a zero, a negative value or anything above 500 is - refused with 422 rather than corrected — serving a different number of rows than were asked for is another - difference nobody is told about. - responses: - "200": - description: The record's personal bests, newest first. - content: - application/json: - schema: - $ref: "#/components/schemas/ListPersonalRecordsEnvelope" - "401": *a1 - "403": *a11 - "422": - description: "`metricType` or `limit` carried an unacceptable value. `meta.errorCode` = - `personal_records.invalid_query`, with the offending path under `details.issues`." - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorEnvelope" - "429": *a5 /api/workouts: get: tags: @@ -22352,6 +22337,59 @@ paths: anyOf: - $ref: "#/components/schemas/HealthAdminResponse" - $ref: "#/components/schemas/HealthPublicResponse" + /api/personal-records: + get: + tags: + - Awards + summary: List detected personal bests + description: >- + The record's personal bests, newest achievement first. Read-only — rows come from the detection worker and no + endpoint creates or edits one. + + + Both query parameters are validated and a bad value is a 422. They were forgiving until this release, and in one + direction that was worse than merely quiet: dropping an unrecognised `metricType` WIDENED the read to every + metric, so a typo returned more rows than the caller asked for. The `limit` ceiling refuses rather than clamps + for the same reason. + + + Delegable on a `measurements` grant. + parameters: + - name: metricType + in: query + required: false + schema: + type: string + description: A `MeasurementType` to filter to. A value outside the enum is refused with 422 — it is not dropped, because + dropping it would widen the read rather than narrow it. + - name: limit + in: query + required: false + schema: + type: integer + minimum: 1 + maximum: 500 + default: 100 + description: Page cap, default 100, ceiling 500. A non-integer, a zero, a negative value or anything above 500 is + refused with 422 rather than corrected — serving a different number of rows than were asked for is another + difference nobody is told about. + responses: + "200": + description: The record's personal bests, newest first. + content: + application/json: + schema: + $ref: "#/components/schemas/ListPersonalRecordsEnvelope" + "401": *a1 + "403": *a11 + "422": + description: "`metricType` or `limit` carried an unacceptable value. `meta.errorCode` = + `personal_records.invalid_query`, with the offending path under `details.issues`." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "429": *a5 /api/gamification/achievements: get: tags: @@ -26382,19 +26420,24 @@ components: $ref: "#/components/schemas/AppleHealthBatchEntry" syncTrigger: description: 'Diagnostic-only. Names what triggered this sync (foreground app open, background refresh, a push wake, or - a user-initiated `manual` "Sync all" run, since v1.42). Recorded on the ingest wide event and kept on the - account as `lastSyncTrigger` — plus `lastBackgroundSyncAt` for a `background` or `push` trigger — both - readable from `GET /api/integrations/healthkit`. Since v1.42 it is also recorded per measurement type the - batch carried (Apple Health rows only), served as `lastReceivedAt` / `lastTrigger` / `lastNewSampleAt` on - each `metricFreshness` entry of that read, plus `lastBackgroundAt` (since v1.42.4) for a `background` - trigger. It does not affect dedup, attribution, or how any sample is stored. Optional and - backward-compatible: pre-#66 clients omit it.' + a user-initiated `manual` "Sync all" run, since v1.42). `backfill` (v1.43, ios #77; send it only when `GET + /api/auth/me` reports `features.syncTriggerBackfill: true`, an older server refuses the whole batch with + 422) names a history catch-up: it moves only `syncProgress` and the per-type `lastBackfillAt` in `GET + /api/integrations/healthkit`, never `lastReceivedAt`, `lastTrigger`, `lastBackgroundAt`, `lastSyncTrigger`, + `lastBackgroundSyncAt` or the sync checkpoint. Recorded on the ingest wide event and kept on the account as + `lastSyncTrigger` — plus `lastBackgroundSyncAt` for a `background` or `push` trigger — both readable from + `GET /api/integrations/healthkit`. Since v1.42 it is also recorded per measurement type the batch carried + (Apple Health rows only), served as `lastReceivedAt` / `lastTrigger` / `lastNewSampleAt` on each + `metricFreshness` entry of that read, plus `lastBackgroundAt` (since v1.42.4) for a `background` trigger. It + does not affect dedup, attribution, or how any sample is stored. Optional and backward-compatible: pre-#66 + clients omit it.' type: string enum: - foreground - background - push - manual + - backfill required: - entries description: Apple Health batch ingest. ≤500 entries per call; idempotent via `Idempotency-Key`. @@ -26459,7 +26502,7 @@ components: minLength: 1 maxLength: 120 description: >- - Dedup key for the `(userId, type, source, externalId)` composite index. Three shapes: + Dedup key for the `(userId, type, source, externalId)` composite index. Four shapes: • `HKSample.uuid` — a per-sample reading. First-write-wins: a re-post returns `duplicate` and the stored row is immutable. @@ -26474,6 +26517,17 @@ components: returns `updated`, so iOS uploads ~144 rows/day instead of one per raw HR sample. A row that targets this prefix with a malformed bucket-start suffix is `skipped` with reason `malformed_hr_bucket_id`. + • `stats::T` — v1.43 (ios #77), only when `GET /api/auth/me` + reports `features.cumulativeHourBuckets: true`: one local hour's SUM of steps, active energy, walking and + running distance or flights climbed, the date and the zero-padded hour being the local hour in the account's + zone (`timezone` on `/api/auth/me`). `value` is the hour's sum in the unit of the day form, `startDate` the + first instant of that local hour, `endDate` one hour later. Such a row is never a measurement and never + changes the day total, which stays the day form: it feeds the intraday profile behind the same-time baseline + (a day with at least four posted hours is built from them). A re-post overwrites (`updated`). Skipped with + `hour_bucket_misaligned` when `startDate` is not the start of that local hour, + `hour_bucket_unsupported_type` for any other identifier or a `MANUAL` / bridge entry, `hour_bucket_too_old` + when the day is more than 400 days back. + The id MUST be stable across app restarts. An object description or memory address (``, `0x12ab34cd`) is a new value on every launch, so it can never match its own earlier row; such an entry is `skipped` with reason `unstable_external_id` while the rest of the batch still lands. @@ -26977,17 +27031,20 @@ components: items: $ref: "#/components/schemas/BulkMedicationIntakeEntry" syncTrigger: - description: 'Diagnostic-only, the same field `POST /api/measurements/batch` takes (since v1.42.4): what woke the client - for this sync. When the batch carries entries with `source: "APPLE_HEALTH"` that land (`inserted`, `updated` - or `duplicate`), it is recorded on the `MEDICATION_DOSE` entry of `metricFreshness` in `GET + description: "Diagnostic-only, the same field `POST /api/measurements/batch` takes (since v1.42.4): what woke the client + for this sync. `backfill` (v1.43, only when `features.syncTriggerBackfill` is `true`) marks a history + catch-up, which moves only the `MEDICATION_DOSE` entry's `lastBackfillAt` and leaves its live stamps and the + sync checkpoint alone. When the batch carries entries with `source: \"APPLE_HEALTH\"` that land (`inserted`, + `updated` or `duplicate`), it is recorded on the `MEDICATION_DOSE` entry of `metricFreshness` in `GET /api/integrations/healthkit` as `lastTrigger`, and a `background` value also dates its `lastBackgroundAt`. - It changes no dose row. Optional; omitted records a null trigger.' + It changes no dose row. Optional; omitted records a null trigger." type: string enum: - foreground - background - push - manual + - backfill required: - entries description: iOS SyncMode bulk intake backfill. 1–500 entries per call; idempotent via `Idempotency-Key` and per-entry @@ -36037,7 +36094,8 @@ components: - type: object properties: restored: - description: v1.43 — starts given back because this edit cleared the flow on a start day. + description: v1.43 — starts given back because this edit cleared the flow on a start day; empty when it gave back none. + Absent on older servers. type: array items: $ref: "#/components/schemas/CycleRestoredStart" @@ -36193,7 +36251,8 @@ components: - $ref: "#/components/schemas/CycleDayLogDTO" - type: "null" absorbed: - description: 'v1.43 — on `action: "start"`: a later start within the period length folded into this one, or null.' + description: 'v1.43 — a later start within the period length that this start folded in (it stops being a start of its + own), or null. Present on every `action: "start"` answer; null on `end`.' anyOf: - $ref: "#/components/schemas/CycleRestoredStart" - type: "null" @@ -39958,7 +40017,6 @@ components: - type: "null" unit: type: string - minLength: 1 referenceLow: anyOf: - type: number @@ -40041,7 +40099,6 @@ components: minLength: 1 unit: type: string - minLength: 1 lowerBound: anyOf: - type: number @@ -41451,6 +41508,24 @@ components: type: integer minimum: -9007199254740991 maximum: 9007199254740991 + hourlyIncrements: + default: [] + type: array + items: + type: number + basis: + default: PER_SAMPLE + type: string + enum: + - PER_SAMPLE + - HOUR_BUCKETS + bucketCount: + default: null + anyOf: + - type: integer + minimum: 0 + maximum: 9007199254740991 + - type: "null" timezone: type: string minLength: 1 @@ -41466,6 +41541,9 @@ components: - hourlyCumulative - dayTotal - sampleCount + - hourlyIncrements + - basis + - bucketCount - timezone additionalProperties: {} healthScoreRecords: @@ -46630,7 +46708,9 @@ components: reason: description: "Why a row was `skipped` or a `duplicate`. `folded_window` (since v1.42): a raw sample of a dense or mean type whose local day lies before the fold boundary and whose hour or day a live `stats:` row of the same - source already covers. Nothing to retry; treat it like any duplicate." + source already covers. Nothing to retry; treat it like any duplicate. `hour_bucket_misaligned`, + `hour_bucket_unsupported_type`, `hour_bucket_too_old` (since v1.43): an hourly + `stats::T` row the server refused, see `externalId`; permanent for the row as sent." type: string convertedValue: description: "Only on a `value_out_of_range` skip (since v1.39.1): the entry's value after conversion into the stored @@ -47240,115 +47320,6 @@ components: description: "Server-authoritative sleep-rhythm read: the outstanding sleep-debt balance over the rolling window + MCTQ chronotype (MSF/MSFsc band, social jetlag). Free vs work nights default to weekend = free in the user's timezone (no work calendar). A view over existing per-stage SLEEP_DURATION rows — no schema, no new type." - ListPersonalRecordsEnvelope: - type: object - properties: - data: - type: array - items: - $ref: "#/components/schemas/PersonalRecord" - error: - type: "null" - meta: - type: object - properties: - requestId: - type: string - additionalProperties: false - required: - - data - - error - additionalProperties: false - PersonalRecord: - type: object - properties: - id: - type: string - userId: - type: string - description: The RECORD's user id — under a delegated read this is the owner, not the caller. - metricType: - $ref: "#/components/schemas/MeasurementType" - metricSlot: - anyOf: - - type: string - - type: "null" - description: Per-sport bucket for workout-driven bests (e.g. "running_5km_time"). Null for a plain measurement-driven - best, where the metric type alone is the dimension. - direction: - type: string - enum: - - MAX - - MIN - description: Which way is better. MAX for steps / VO2 max / HRV / distance; MIN for resting heart rate, body fat, audio - exposure. - value: - type: number - unit: - type: string - achievedAt: - type: string - format: date-time - pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ - sourceMeasurementId: - anyOf: - - type: string - - type: "null" - description: The measurement that achieved the record. Nulled rather than cascaded when that measurement is later - deleted, so the historical fact survives the reading behind it. - source: - $ref: "#/components/schemas/MeasurementSource" - externalId: - anyOf: - - type: string - - type: "null" - createdAt: - type: string - format: date-time - pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ - benchmark: - description: v1.43 — an age- and sex-aware comparison band where a cited norm exists, in the record's unit; null - otherwise. - anyOf: - - $ref: "#/components/schemas/RecordBenchmark" - - type: "null" - required: - - id - - userId - - metricType - - metricSlot - - direction - - value - - unit - - achievedAt - - sourceMeasurementId - - source - - externalId - - createdAt - additionalProperties: false - description: One detected personal best. Rows are written by the detection worker, never by a client — there is no write - endpoint on this resource. - RecordBenchmark: - type: object - properties: - low: - type: number - high: - type: number - unit: - type: string - source: - type: string - sourceLabel: - type: string - required: - - low - - high - - unit - - source - - sourceLabel - additionalProperties: false - description: An age- and sex-aware comparison band from a cited norm, in the record's unit. ListWorkoutsResponse: type: object properties: @@ -63746,9 +63717,11 @@ components: `STRESS_SCORE` / `STRAIN_SCORE` return the `WellnessScoreValue` schema — field-by-field in `components.schemas`, including the `series`, `daysInWindow` and `asOf` a client would otherwise have to discover by inspecting a live payload. `VITALS_BASELINE` returns { type, center, low, high, spread, - sampleDays, k, series }, where `series` is the trailing per-day means for the inline sparkline. The - remaining metrics' value objects are not modelled here yet; their shapes are TypeScript interfaces beside - their engines under `src/lib/insights/derived/`. + sampleDays, k, series, direction }, where `series` is the trailing per-day means for the inline sparkline + and `direction` (v1.43, a `TrendVerdict`) says which way a change of the vital counts for this person, with + `center` placed against their own target or a profile default. The remaining metrics' value objects are not + modelled here yet; their shapes are TypeScript interfaces beside their engines under + `src/lib/insights/derived/`. coverage: $ref: "#/components/schemas/DerivedCoverage" confidence: @@ -82248,6 +82221,115 @@ components: additionalProperties: false description: "What an unauthenticated (or non-admin) caller sees: the aggregate verdict and nothing else. No component detail, so an uptime probe cannot be used to fingerprint which part of a self-hosted instance is unwell." + ListPersonalRecordsEnvelope: + type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/PersonalRecord" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + PersonalRecord: + type: object + properties: + id: + type: string + userId: + type: string + description: The RECORD's user id — under a delegated read this is the owner, not the caller. + metricType: + $ref: "#/components/schemas/MeasurementType" + metricSlot: + anyOf: + - type: string + - type: "null" + description: Per-sport bucket for workout-driven bests (e.g. "running_5km_time"). Null for a plain measurement-driven + best, where the metric type alone is the dimension. + direction: + type: string + enum: + - MAX + - MIN + description: Which way is better. MAX for steps / VO2 max / HRV / distance; MIN for resting heart rate, body fat, audio + exposure. + value: + type: number + unit: + type: string + achievedAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + sourceMeasurementId: + anyOf: + - type: string + - type: "null" + description: The measurement that achieved the record. Nulled rather than cascaded when that measurement is later + deleted, so the historical fact survives the reading behind it. + source: + $ref: "#/components/schemas/MeasurementSource" + externalId: + anyOf: + - type: string + - type: "null" + createdAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + benchmark: + description: v1.43 — an age- and sex-aware comparison band where a cited norm exists, in the record's unit; null + otherwise. + anyOf: + - $ref: "#/components/schemas/RecordBenchmark" + - type: "null" + required: + - id + - userId + - metricType + - metricSlot + - direction + - value + - unit + - achievedAt + - sourceMeasurementId + - source + - externalId + - createdAt + additionalProperties: false + description: One detected personal best. Rows are written by the detection worker, never by a client — there is no write + endpoint on this resource. + RecordBenchmark: + type: object + properties: + low: + type: number + high: + type: number + unit: + type: string + source: + type: string + sourceLabel: + type: string + required: + - low + - high + - unit + - source + - sourceLabel + additionalProperties: false + description: An age- and sex-aware comparison band from a cited norm, in the record's unit. AchievementsEnvelope: type: object properties: diff --git a/messages/de.json b/messages/de.json index f30261f62..4179ec1a9 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7159,7 +7159,8 @@ "lastReceived": "Zuletzt empfangen {relative}", "lastNewValue": "Letzter neuer Wert {relative}", "lastBackground": "Zuletzt im Hintergrund {relative}", - "never": "Noch nicht empfangen" + "never": "Noch nicht empfangen", + "lastBackfill": "Verlauf nachgeladen {relative}" } }, "garminInfo": { diff --git a/messages/en.json b/messages/en.json index 146ad753d..d525113dd 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7159,7 +7159,8 @@ "lastReceived": "Last received {relative}", "lastNewValue": "Last new value {relative}", "lastBackground": "Last in background {relative}", - "never": "Not received yet" + "never": "Not received yet", + "lastBackfill": "History caught up {relative}" } }, "garminInfo": { diff --git a/messages/es.json b/messages/es.json index 8f07b72b5..09e4fef90 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7159,7 +7159,8 @@ "lastReceived": "Última recepción {relative}", "lastNewValue": "Último valor nuevo {relative}", "lastBackground": "Última vez en segundo plano {relative}", - "never": "Aún no recibido" + "never": "Aún no recibido", + "lastBackfill": "Historial recuperado {relative}" } }, "garminInfo": { diff --git a/messages/fr.json b/messages/fr.json index 1eca293b5..5af0ba276 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7159,7 +7159,8 @@ "lastReceived": "Dernière réception {relative}", "lastNewValue": "Dernière nouvelle valeur {relative}", "lastBackground": "Dernière fois en arrière-plan {relative}", - "never": "Pas encore reçu" + "never": "Pas encore reçu", + "lastBackfill": "Historique rattrapé {relative}" } }, "garminInfo": { diff --git a/messages/it.json b/messages/it.json index 9fc11b0b5..2db5b18f7 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7159,7 +7159,8 @@ "lastReceived": "Ultima ricezione {relative}", "lastNewValue": "Ultimo nuovo valore {relative}", "lastBackground": "Ultima volta in background {relative}", - "never": "Non ancora ricevuto" + "never": "Non ancora ricevuto", + "lastBackfill": "Storico recuperato {relative}" } }, "garminInfo": { diff --git a/messages/ko.json b/messages/ko.json index 09171a7a7..ac172d3c7 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7159,7 +7159,8 @@ "lastReceived": "마지막 수신 {relative}", "lastNewValue": "마지막 새 값 {relative}", "lastBackground": "마지막 백그라운드 수신 {relative}", - "never": "아직 수신되지 않음" + "never": "아직 수신되지 않음", + "lastBackfill": "기록 보충 {relative}" } }, "garminInfo": { diff --git a/messages/pl.json b/messages/pl.json index cb5a9a4b5..d35fe1131 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7159,7 +7159,8 @@ "lastReceived": "Ostatnio odebrano {relative}", "lastNewValue": "Ostatnia nowa wartość {relative}", "lastBackground": "Ostatnio w tle {relative}", - "never": "Jeszcze nie odebrano" + "never": "Jeszcze nie odebrano", + "lastBackfill": "Historia uzupełniona {relative}" } }, "garminInfo": { diff --git a/prisma/schema.prisma b/prisma/schema.prisma index f423e3f64..1285da288 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -2334,16 +2334,18 @@ model IntradayCumulativeProfile { /// two rows is not a shape; the baseline reader uses this to refuse such /// a day rather than let it drag the median. sampleCount Int @map("sample_count") - /// @pending-consumer(#v143-B9) v1.43 (#77) — per-hour increments posted as - /// `stats::T` rows, 24 slots in canonical units. Empty - /// for a profile folded from per-sample rows. + /// v1.43 (#77) — per-hour increments posted as + /// `stats::T` rows (`intraday-hour-buckets.ts`), 24 + /// slots in canonical units, -1 for an hour not posted. Empty for a day + /// the phone never posted hours for. hourlyIncrements Float[] @map("hourly_increments") - /// @pending-consumer(#v143-B9) What `hourlyCumulative` was built from: - /// `PER_SAMPLE` (the fold of raw rows) or `HOUR_BUCKETS` (the posted hour - /// rows). CHECK in migration 0408. + /// What `hourlyCumulative` was built from: `PER_SAMPLE` (the fold of raw + /// rows) or `HOUR_BUCKETS` (the posted hours, once a day holds four). + /// CHECK in migration 0408. Read by the profile reader and the drain's + /// merge, which leaves a bucket-built day alone. basis String @default("PER_SAMPLE") - /// @pending-consumer(#v143-B9) How many hour buckets backed a - /// `HOUR_BUCKETS` profile; NULL for `PER_SAMPLE`. + /// How many hours the phone posted for the day; NULL when it posted none. + /// Below four the day keeps the per-sample fold (or no curve yet). bucketCount Int? @map("bucket_count") /// The IANA zone the day was bucketed in. A user who later moves zones /// leaves behind profiles cut on the old clock; carrying the zone lets @@ -4044,7 +4046,11 @@ model HealthKitTypeSync { /// A `MeasurementType` name for rows the measurement batch writes, or /// `MEDICATION_DOSE` for Apple Health doses the intake bulk route /// receives. Text, not the enum, since v1.42.4 (migration 0401) because a - /// dose is not a measurement type. + /// dose is not a measurement type. Since v1.43 (ios #77) a + /// `backfill:` row records when a `syncTrigger: "backfill"` batch + /// last carried the type (its `lastReceivedAt`, every other column NULL), + /// so a catch-up leaves the live row of the type untouched; live readers + /// skip the prefix (`healthkit-type-sync.ts`). type String /// Last batch that carried this type. lastReceivedAt DateTime @map("last_received_at") diff --git a/src/__tests__/client-threshold-literal-guard.test.ts b/src/__tests__/client-threshold-literal-guard.test.ts new file mode 100644 index 000000000..3ac062f5e --- /dev/null +++ b/src/__tests__/client-threshold-literal-guard.test.ts @@ -0,0 +1,356 @@ +/** + * Client code holds no threshold or polarity table of its own (v1.43, ios + * #116 items 3 and 4). + * + * Which way a change counts and where a chart's reference lines sit depend on + * the person: their own target, their age, their sex, their glucose unit. The + * server resolves both (`src/lib/targets/trend-verdict.ts`, + * `src/lib/measurements/series-reference-lines.ts`) and publishes them on the + * dashboard snapshot (`tiles.directions`), the vitals baseline (`direction`), + * the summary and target reads and the series read. A client that keeps its + * own copy answers for an imaginary average person: a rising blood pressure + * painted as a setback for someone below their target, a 140 mmHg line on a + * chart whose person has a band of 115 to 125. + * + * Four shapes are refused in client code (`src/components`, `src/hooks`, and + * every `"use client"` module under `src/app` outside the API): + * + * 1. a `directionSentiment` given as a string literal: the tile must read + * the resolved verdict; + * 2. a `ReferenceLine` / `ReferenceArea` drawn at a number literal: a line + * has to come from data the server resolved; + * 3. a polarity table, i.e. a map or set of metric names to `up-good` / + * `up-bad`, or a `…Sentiment(type)` function deciding one; + * 4. an import of the server's resolvers into the browser (a type import + * is fine). + * + * The guard cannot pass by reading nothing: the walk carries a floor, every + * rule's matcher is proven against a sample of the shape it exists to catch, + * every allowlist entry must still match with its exact count, and the rules + * must see at least one real match across the tree. + * + * Its limit: it reads the source, not the running page. A polarity computed + * from a variable that happens to hold a literal is not seen; review catches + * that. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import ts from "typescript"; +import { describe, expect, it } from "vitest"; + +import { stripComments, walkSourceFiles } from "./helpers/source-files"; + +const ROOT = process.cwd(); + +interface ClientFile { + rel: string; + source: string; +} + +function isClientModule(rel: string, source: string): boolean { + if (rel.startsWith("src/components/") || rel.startsWith("src/hooks/")) { + return true; + } + return /^\s*["']use client["']/.test(stripComments(source).trimStart()); +} + +function clientFiles(): ClientFile[] { + const roots: Array<{ dir: string; floor: number }> = [ + { dir: "src/components", floor: 900 }, + { dir: "src/hooks", floor: 50 }, + { dir: "src/app", floor: 900 }, + ]; + const out: ClientFile[] = []; + for (const { dir, floor } of roots) { + for (const file of walkSourceFiles(join(ROOT, dir), { floor })) { + const rel = `${dir}/${file}`; + if (rel.startsWith("src/app/api/")) continue; + if (rel.includes("/__tests__/") || /\.test\.tsx?$/.test(rel)) continue; + const source = readFileSync(join(ROOT, rel), "utf8"); + if (!isClientModule(rel, source)) continue; + out.push({ rel, source }); + } + } + return out; +} + +function parse(name: string, source: string): ts.SourceFile { + return ts.createSourceFile( + name, + source, + ts.ScriptTarget.Latest, + true, + name.endsWith("x") ? ts.ScriptKind.TSX : ts.ScriptKind.TS, + ); +} + +/** The literal text a JSX attribute initialiser carries, if it is one. */ +function literalOf(init: ts.JsxAttributeValue | undefined): string | null { + if (!init) return null; + if (ts.isStringLiteral(init)) return init.text; + if (ts.isJsxExpression(init) && init.expression) { + const e = init.expression; + if (ts.isStringLiteral(e) || ts.isNoSubstitutionTemplateLiteral(e)) { + return e.text; + } + if (ts.isNumericLiteral(e)) return e.text; + if ( + ts.isPrefixUnaryExpression(e) && + e.operator === ts.SyntaxKind.MinusToken && + ts.isNumericLiteral(e.operand) + ) { + return `-${e.operand.text}`; + } + } + return null; +} + +function jsxAttributes( + file: ts.SourceFile, + visit: ( + tag: string, + name: string, + init: ts.JsxAttributeValue | undefined, + node: ts.Node, + ) => void, +): void { + const walk = (node: ts.Node) => { + if (ts.isJsxSelfClosingElement(node) || ts.isJsxOpeningElement(node)) { + const tag = node.tagName.getText(file); + for (const attr of node.attributes.properties) { + if (!ts.isJsxAttribute(attr)) continue; + visit(tag, attr.name.getText(file), attr.initializer, attr); + } + } + ts.forEachChild(node, walk); + }; + walk(file); +} + +function lineOf(file: ts.SourceFile, node: ts.Node): number { + return file.getLineAndCharacterOfPosition(node.getStart(file)).line + 1; +} + +/** Rule 1 — `directionSentiment` given as a literal. */ +function literalDirections(name: string, source: string): string[] { + const file = parse(name, source); + const hits: string[] = []; + jsxAttributes(file, (tag, attr, init, node) => { + if (attr !== "directionSentiment") return; + const literal = literalOf(init); + if (literal !== null) { + hits.push( + `${name}:${lineOf(file, node)}: <${tag} directionSentiment="${literal}">`, + ); + } + }); + return hits; +} + +const LINE_TAGS = new Set(["ReferenceLine", "ReferenceArea"]); +const LINE_ATTRS = new Set(["y", "y1", "y2"]); + +/** Rule 2 — a chart reference drawn at a number literal. */ +function literalLines(name: string, source: string): string[] { + const file = parse(name, source); + const hits: string[] = []; + jsxAttributes(file, (tag, attr, init, node) => { + if (!LINE_TAGS.has(tag) || !LINE_ATTRS.has(attr)) return; + const literal = literalOf(init); + if (literal !== null && /^-?\d/.test(literal)) { + hits.push(`${name}:${lineOf(file, node)}: <${tag} ${attr}={${literal}}>`); + } + }); + return hits; +} + +/** Rule 3 — a polarity table: line-shaped, over comment-stripped code. */ +const POLARITY_PATTERNS: RegExp[] = [ + // `RESTING_HEART_RATE: "up-bad"` / `["HRV", "up-good"]` + /\b[A-Z][A-Z0-9_]{2,}["']?\s*[:,]\s*["']up-(good|bad)["']/, + // `const UP_BAD_VITALS = new Set([...])` + /\b(UP_GOOD|UP_BAD)_[A-Z0-9_]+\s*=/, + // `function vitalSentiment(type: string)` + /function\s+\w*Sentiment\s*\(\s*(type|metric|kind)\b/, +]; +const POLARITY_SAMPLES = [ + 'const POLARITY = { RESTING_HEART_RATE: "up-bad" };', + 'new Map([["HEART_RATE_VARIABILITY", "up-good"]])', + "const UP_BAD_VITALS = new Set(['RESTING_HEART_RATE']);", + "function vitalSentiment(type: string): TrendDirectionSentiment {", +]; + +function polarityTables(name: string, source: string): string[] { + const hits: string[] = []; + stripComments(source) + .split("\n") + .forEach((line, index) => { + if (POLARITY_PATTERNS.some((p) => p.test(line))) { + hits.push(`${name}:${index + 1}: ${line.trim()}`); + } + }); + return hits; +} + +/** Rule 4 — the server's resolvers, imported into the browser. */ +const SERVER_RESOLVERS = [ + "@/lib/targets/trend-verdict", + "@/lib/measurements/series-reference-lines", + "@/lib/measurements/series-change", + "@/lib/personal-records/record-benchmark", + "@/lib/insights/mood-pattern-cards", +]; + +function resolverImports(name: string, source: string): string[] { + const file = parse(name, source); + const hits: string[] = []; + for (const statement of file.statements) { + if (!ts.isImportDeclaration(statement)) continue; + const from = (statement.moduleSpecifier as ts.StringLiteral).text; + if (!SERVER_RESOLVERS.includes(from)) continue; + if (statement.importClause?.isTypeOnly) continue; + hits.push(`${name}:${lineOf(file, statement)}: import from "${from}"`); + } + return hits; +} + +interface Rule { + name: string; + find: (name: string, source: string) => string[]; + /** Sources that must produce at least one hit, proving the matcher. */ + samples: Array<{ name: string; source: string }>; + /** File → exact hit count it may have, with the reason. */ + allow: Record; +} + +const RULES: Rule[] = [ + { + name: "a literal directionSentiment", + find: literalDirections, + samples: [ + { + name: "sample.tsx", + source: 'const a = ;', + }, + { + name: "sample.tsx", + source: 'const a = ;', + }, + ], + allow: { + "src/components/insights/derived/vitals-dashboard.tsx": { + count: 6, + reason: + "the six derived re-frame tiles (same-time steps, six-minute walk, fitness age, vascular age, HRV balance, BMI) colour a figure whose good direction is its definition (a longer walk, a younger vascular age, no judgement on BMI), not a person's target; the per-vital baseline tile reads the server's `direction`", + }, + }, + }, + { + name: "a reference line at a literal", + find: literalLines, + samples: [ + { + name: "sample.tsx", + source: 'const a = ;', + }, + { + name: "sample.tsx", + source: "const a = ;", + }, + ], + allow: { + "src/components/charts/medication-compliance-chart.tsx": { + count: 2, + reason: + "the opt-in adherence overlay: an 80 % minimum and the 100 % goal are properties of a share of doses, the same for every person, not a clinical threshold the person's profile moves", + }, + "src/components/insights/mood/mood-time-of-day-chart.tsx": { + count: 1, + reason: + "the 3.5 line is the green edge of the 1 to 5 mood scale itself (`MOOD_GREEN_MIN`), the same for every person", + }, + "src/components/insights/mood/mood-weekday-chart.tsx": { + count: 1, + reason: + "the 3.5 line is the green edge of the 1 to 5 mood scale itself (`MOOD_GREEN_MIN`), the same for every person", + }, + }, + }, + { + name: "a polarity table", + find: polarityTables, + samples: POLARITY_SAMPLES.map((source) => ({ name: "sample.ts", source })), + allow: {}, + }, + { + name: "a server resolver imported into the browser", + find: resolverImports, + samples: [ + { + name: "sample.ts", + source: + 'import { resolveTrendVerdict } from "@/lib/targets/trend-verdict";', + }, + ], + allow: {}, + }, +]; + +const FILES = clientFiles(); + +function hitsFor(rule: Rule): Map { + const out = new Map(); + for (const { rel, source } of FILES) { + const hits = rule.find(rel, source); + if (hits.length > 0) out.set(rel, hits); + } + return out; +} + +describe("client threshold literal guard", () => { + it("walks a client tree of the size it expects", () => { + expect(FILES.length).toBeGreaterThan(700); + }); + + for (const rule of RULES) { + describe(rule.name, () => { + it("still catches the shape it exists to catch", () => { + for (const sample of rule.samples) { + expect( + rule.find(sample.name, sample.source).length, + `no hit on: ${sample.source}`, + ).toBeGreaterThan(0); + } + }); + + it("finds none outside its allowlist, and the allowlist exactly", () => { + const hits = hitsFor(rule); + const offenders = [...hits.entries()] + .filter(([rel]) => !(rel in rule.allow)) + .flatMap(([, lines]) => lines); + expect( + offenders, + "client code decides a threshold or a polarity itself. Read the " + + "server's resolved field instead (`tiles.directions`, `direction`, " + + "`referenceLines`), or allowlist the file with its reason.", + ).toEqual([]); + for (const [rel, { count }] of Object.entries(rule.allow)) { + expect( + hits.get(rel)?.length ?? 0, + `${rel} is allowlisted for exactly ${count} hits`, + ).toBe(count); + } + }); + }); + } + + it("reads a non-empty set of matches across the tree", () => { + const total = RULES.reduce( + (sum, rule) => + sum + [...hitsFor(rule).values()].reduce((n, l) => n + l.length, 0), + 0, + ); + expect(total).toBeGreaterThan(0); + }); +}); diff --git a/src/__tests__/pulse-day-mean-reader-guard.test.ts b/src/__tests__/pulse-day-mean-reader-guard.test.ts index 1e418fc3a..df329bd84 100644 --- a/src/__tests__/pulse-day-mean-reader-guard.test.ts +++ b/src/__tests__/pulse-day-mean-reader-guard.test.ts @@ -125,6 +125,10 @@ const TS_READERS: Record = { "lib/analytics/correlations-fast-path.ts": { exempt: "mood mean; pulse enters as the resting series, never a day mean", }, + "app/api/dashboard/summary/route.ts": { + exempt: + "places a card's seven-day sparkline mean against its band for the trend verdict; each sparkline point is already one rollup day value, so the mean weighs every day once", + }, "lib/dashboard/snapshot.ts": { exempt: "summarises water-intake days and glucose contexts", }, diff --git a/src/__tests__/resolved-field-consumer-guard.test.ts b/src/__tests__/resolved-field-consumer-guard.test.ts index d63da7a57..7caab3bd6 100644 --- a/src/__tests__/resolved-field-consumer-guard.test.ts +++ b/src/__tests__/resolved-field-consumer-guard.test.ts @@ -289,3 +289,134 @@ describe("resolved-field consumer guard", () => { expect(total).toBeGreaterThanOrEqual(6); }); }); + +/** + * v1.43 (ios #116 items 3, 4, 5, 6, 9; #77) — the fields the server resolves + * for the native client reach the web as well, so the two clients render the + * same answer. Each field names the web module that reads it, by a pattern + * over that module's code. A field the web has no screen for names the web + * twin it is resolved alongside (the same resolver, published where the web + * reads it) and why; the twin's reader is checked the same way. + * + * The rebuild side of the same rule (a client deciding a polarity or drawing + * a threshold of its own) is `client-threshold-literal-guard.test.ts`. + */ +interface WebReader { + file: string; + pattern: RegExp; +} + +interface ResolvedFieldReader { + field: string; + readers: WebReader[]; + /** Set when the web reads a twin rather than this route's field. */ + twin?: string; +} + +const V143_FIELDS: ResolvedFieldReader[] = [ + { + field: "GET /api/dashboard/snapshot tiles.directions", + readers: [ + { file: "src/app/page-client.tsx", pattern: /tiles\.directions/ }, + ], + }, + { + field: + "GET /api/dashboard/summary DashboardMetricCard.direction / trendSentiment", + twin: "the summary route is the native client's dashboard; the web dashboard reads the same `resolveTrendVerdict` answer from the snapshot's `tiles.directions`", + readers: [{ file: "src/app/page-client.tsx", pattern: /directionOf\(/ }], + }, + { + field: "GET /api/insights/targets TargetItem.direction / trendSentiment", + twin: "the web target panels show the band, not a change; the change they would colour is the dashboard tile's, read from `tiles.directions`", + readers: [ + { file: "src/app/page-client.tsx", pattern: /tileDirections\?\.\[key\]/ }, + ], + }, + { + field: "GET /api/insights/derived VITALS_BASELINE value.direction", + readers: [ + { + file: "src/components/insights/derived/vitals-dashboard.tsx", + pattern: /v\.direction\?\.direction/, + }, + ], + }, + { + field: "GET /api/measurements/series direction / change / referenceLines", + twin: "the web charts read the series-batch and draw the person's bands from the snapshot's `targetBands`; the series read is the native chart source", + readers: [{ file: "src/app/page-client.tsx", pattern: /targetBands/ }], + }, + { + field: "GET /api/personal-records benchmark", + readers: [ + { + file: "src/components/insights/personal-record-badge.tsx", + pattern: /fresh\.benchmark/, + }, + ], + }, + { + field: "GET /api/mood/insights patterns", + readers: [ + { + file: "src/components/insights/mood/mood-insights-breakdowns.tsx", + pattern: /data\.patterns/, + }, + { + file: "src/components/insights/mood/mood-patterns.tsx", + pattern: /pattern\.values\.subject/, + }, + ], + }, + { + field: "POST /api/cycle/period absorbed", + readers: [ + { + file: "src/components/cycle/use-cycle.ts", + pattern: /data\?\.absorbed/, + }, + ], + }, + { + field: "DELETE /api/cycle/day-logs/{id} (Prefer) and PATCH restored", + readers: [ + { + file: "src/components/cycle/use-cycle.ts", + pattern: /Prefer: "return=representation"/, + }, + { + file: "src/components/cycle/use-cycle.ts", + pattern: /data\?\.restored/, + }, + ], + }, + { + field: "GET /api/integrations/healthkit metricFreshness[].lastBackfillAt", + readers: [ + { + file: "src/components/settings/integrations/apple-health-card.tsx", + pattern: /entry\.lastBackfillAt/, + }, + ], + }, +]; + +describe("v1.43 resolved fields reach the web", () => { + for (const entry of V143_FIELDS) { + it(`${entry.field} has a web reader${entry.twin ? " through its twin" : ""}`, () => { + expect(entry.readers.length).toBeGreaterThan(0); + if (entry.twin !== undefined) + expect(entry.twin.length).toBeGreaterThan(40); + for (const reader of entry.readers) { + const code = stripComments( + readFileSync(join(ROOT, reader.file), "utf8"), + ); + expect( + reader.pattern.test(code), + `${reader.file} no longer reads ${entry.field} (${reader.pattern})`, + ).toBe(true); + } + }); + } +}); diff --git a/src/app/api/auth/me/route.ts b/src/app/api/auth/me/route.ts index 590730f6c..c354df00a 100644 --- a/src/app/api/auth/me/route.ts +++ b/src/app/api/auth/me/route.ts @@ -87,8 +87,8 @@ const SERVER_FEATURES = { trackIntake: true, // v1.43 — each turns true in the change that makes its route or request // form work; until then a client must not send it (contract §0.6). - cumulativeHourBuckets: false, - syncTriggerBackfill: false, + cumulativeHourBuckets: true, + syncTriggerBackfill: true, biomarkerMerge: true, lifeCircumstances: false, } as const; diff --git a/src/app/api/integrations/healthkit/__tests__/route.test.ts b/src/app/api/integrations/healthkit/__tests__/route.test.ts index deaedee17..4dfb5976e 100644 --- a/src/app/api/integrations/healthkit/__tests__/route.test.ts +++ b/src/app/api/integrations/healthkit/__tests__/route.test.ts @@ -533,10 +533,72 @@ describe("GET /api/integrations/healthkit — sync health", () => { lastTrigger: "foreground", lastNewSampleAt: null, lastBackgroundAt: backgroundAt.toISOString(), + lastBackfillAt: null, }); expect(byType.PULSE).toMatchObject({ lastBackgroundAt: null }); }); + // v1.43 (ios #77) — a backfill's own ledger row reads as `lastBackfillAt` + // of its type and moves none of the live facts. + it("serves lastBackfillAt from the backfill row, apart from the live arrival", async () => { + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + healthKitConfigJson: null, + healthKitLastSyncedAt: ago(60 * 60 * 1000), + } as never); + vi.mocked(listLiveMeasurementTypes).mockResolvedValue([ + "PULSE", + "ACTIVITY_STEPS", + ]); + mockProbes([ + { type: "PULSE", last_seen: ago(60 * 60 * 1000) }, + { type: "ACTIVITY_STEPS", last_seen: ago(60 * 60 * 1000) }, + ]); + const receivedAt = ago(5 * 60 * 60 * 1000); + const backfillAt = ago(60 * 1000); + vi.mocked(prisma.healthKitTypeSync.findMany).mockResolvedValue([ + { + type: "PULSE", + lastReceivedAt: receivedAt, + lastTrigger: "foreground", + lastNewSampleAt: receivedAt, + lastBackgroundAt: null, + }, + { + type: "backfill:PULSE", + lastReceivedAt: backfillAt, + lastTrigger: null, + lastNewSampleAt: null, + lastBackgroundAt: null, + }, + { + type: "backfill:ACTIVITY_STEPS", + lastReceivedAt: backfillAt, + lastTrigger: null, + lastNewSampleAt: null, + lastBackgroundAt: null, + }, + ] as never); + + const body = await read(); + const byType = Object.fromEntries( + body.metricFreshness.map((entry) => [entry.type, entry]), + ); + expect(byType.PULSE).toMatchObject({ + lastReceivedAt: receivedAt.toISOString(), + lastTrigger: "foreground", + lastBackfillAt: backfillAt.toISOString(), + }); + // A type that only ever arrived through a backfill has no live arrival. + expect(byType.ACTIVITY_STEPS).toMatchObject({ + lastReceivedAt: null, + lastTrigger: null, + lastBackfillAt: backfillAt.toISOString(), + }); + expect(body.metricFreshness.map((e) => e.type)).not.toContain( + "backfill:PULSE", + ); + }); + it("leaves the dose entry out when there is no Apple Health dose", async () => { vi.mocked(prisma.user.findUnique).mockResolvedValue({ healthKitConfigJson: null, diff --git a/src/app/api/measurements/batch/route.ts b/src/app/api/measurements/batch/route.ts index 299b0ff1b..097011380 100644 --- a/src/app/api/measurements/batch/route.ts +++ b/src/app/api/measurements/batch/route.ts @@ -73,10 +73,19 @@ import { findFoldedWindowDuplicates, } from "@/lib/measurements/folded-window"; import { - HEALTHKIT_SYNC_TRIGGERS, + BACKFILL_SYNC_TRIGGER, + HEALTHKIT_REQUEST_SYNC_TRIGGERS, recordHealthKitTypeArrivals, type HealthKitTypeArrival, } from "@/lib/integrations/healthkit-type-sync"; +import { + classifyHourBucket, + HOUR_BUCKET_SKIP_REASONS, + parseHourBucketId, + writeHourBuckets, + type HourBucketWrite, +} from "@/lib/measurements/intraday-hour-buckets"; +import { resolveStoredTimezone } from "@/lib/tz/resolver"; import { isPlausibleEntryInstant } from "@/lib/validations/entry-instant"; import { classifyExternalId, @@ -199,7 +208,9 @@ const batchEntrySchema = z.object({ // pre-#66 caller stays byte-for-byte unchanged. // v1.42 (#1173) — `manual` names a user-initiated "Sync all" run, as opposed to // the app simply being open (`foreground`). -const syncTriggerEnum = z.enum(HEALTHKIT_SYNC_TRIGGERS); +// v1.43 (ios #77) — `backfill` names a history catch-up. It moves only the +// per-type `lastBackfillAt`, never the live sync stamps (see below). +const syncTriggerEnum = z.enum(HEALTHKIT_REQUEST_SYNC_TRIGGERS); const batchPayloadSchema = z.object({ entries: z.array(batchEntrySchema).min(1), @@ -380,6 +391,11 @@ async function postBatch(request: NextRequest): Promise { const prepared: Prepared[] = []; const results: EntryResult[] = new Array(entries.length); const unstableShapes: UnstableExternalIdShape[] = []; + // v1.43 — hourly sums for the intraday profile, and the zone they are cut in + // (the account's, the one `/api/auth/me` publishes as `timezone`). + const hourWrites: HourBucketWrite[] = []; + let accountTz: string | undefined; + const batchNow = new Date(); for (let index = 0; index < entries.length; index++) { const entry = entries[index]; @@ -402,6 +418,75 @@ async function postBatch(request: NextRequest): Promise { continue; } + // v1.43 (ios #77) — an hourly sum of a summed type + // (`stats::T`) is never a measurement: it lands in the + // intraday profile behind the same-time baseline, and the day's total + // stays the day row. Validated here, written after the measurements. + const hourId = parseHourBucketId(entry.externalId); + if (hourId) { + const skip = (reason: string) => { + results[index] = { index, status: "skipped", reason }; + }; + // The profile has no source dimension and is the phone's: a bridge or a + // hand-entered row cannot post hours into it. + if (scoped || (entry.source ?? "APPLE_HEALTH") !== "APPLE_HEALTH") { + skip(HOUR_BUCKET_SKIP_REASONS.unsupportedType); + continue; + } + accountTz ??= await resolveStoredTimezone(user.timezone); + const startDate = new Date(entry.startDate); + const verdict = classifyHourBucket( + hourId, + startDate, + accountTz, + batchNow, + ); + if (!verdict.ok) { + skip(verdict.reason); + continue; + } + if (!isPlausibleEntryInstant(startDate)) { + skip("implausible_timestamp"); + continue; + } + const hourMapped = mapAppleHealthEntry({ + hkIdentifier: entry.hkIdentifier, + value: entry.value, + unit: entry.unit, + startDate: entry.startDate, + endDate: entry.endDate, + }); + if (!hourMapped || hourMapped.type !== verdict.type) { + // The entry's identifier must be the one its id names. + skip( + hourMapped + ? HOUR_BUCKET_SKIP_REASONS.unsupportedType + : "unmappable_identifier", + ); + continue; + } + if ( + hourMapped.value < 0 || + validateMeasurementRange(hourMapped.type, hourMapped.value) !== null + ) { + results[index] = outOfRangeResult( + index, + hourMapped.type, + hourMapped.value, + ); + continue; + } + hourWrites.push({ + index, + type: verdict.type, + hkIdentifier: hourId.hkIdentifier, + dateKey: hourId.dateKey, + hour: hourId.hour, + value: hourMapped.value, + }); + continue; + } + const mapped = mapAppleHealthEntry({ hkIdentifier: entry.hkIdentifier, value: entry.value, @@ -898,15 +983,79 @@ async function postBatch(request: NextRequest): Promise { } } + // v1.43 — the hours, after the measurements. The last entry for one slot + // wins, as for the day rows; an earlier one in the same batch is a + // superseded duplicate. + let hourInsertedCount = 0; + let hourUpdatedCount = 0; + const hourLanded: HourBucketWrite[] = []; + if (hourWrites.length > 0 && accountTz) { + const lastBySlot = new Map(); + for (const w of hourWrites) { + lastBySlot.set(`${w.type}::${w.dateKey}::${w.hour}`, w); + } + for (const w of hourWrites) { + if (lastBySlot.get(`${w.type}::${w.dateKey}::${w.hour}`) !== w) { + results[w.index] = { + index: w.index, + status: "duplicate", + reason: "superseded_in_batch", + }; + duplicateCount++; + } + } + const toWrite = [...lastBySlot.values()]; + try { + const statuses = await writeHourBuckets( + prisma, + user.id, + accountTz, + toWrite, + ); + for (const w of toWrite) { + const status = statuses.get(w.index) ?? "inserted"; + results[w.index] = { index: w.index, status }; + if (status === "inserted") hourInsertedCount++; + else hourUpdatedCount++; + hourLanded.push(w); + } + } catch (err) { + for (const w of toWrite) { + results[w.index] = { + index: w.index, + status: "failed", + reason: "write_failed", + }; + failedCount++; + } + annotate({ + action: { name: "measurement.batch.hour-bucket.failed" }, + meta: { error: err instanceof Error ? err.message : String(err) }, + }); + } + annotate({ + action: { name: "measurement.batch.hour-bucket" }, + meta: { + attempted: hourWrites.length, + inserted: hourInsertedCount, + updated: hourUpdatedCount, + processed: entries.length, + }, + }); + } + const healthKitSyncSucceeded = failedCount === 0 && - prepared.some((p) => { - if ((p.entry.source ?? "APPLE_HEALTH") !== "APPLE_HEALTH") return false; - const status = results[p.index]?.status; - return ( - status === "inserted" || status === "updated" || status === "duplicate" - ); - }); + (hourLanded.length > 0 || + prepared.some((p) => { + if ((p.entry.source ?? "APPLE_HEALTH") !== "APPLE_HEALTH") return false; + const status = results[p.index]?.status; + return ( + status === "inserted" || + status === "updated" || + status === "duplicate" + ); + })); // This batch route is the native ingestion boundary, so it is also where the // SyncMode checkpoint (`User.lastSyncedAt`, the same column @@ -938,6 +1087,11 @@ async function postBatch(request: NextRequest): Promise { // uses to size its undelivered window, so letting a scale advance it can make // the phone skip a window of its own samples — a silent data loss whose cause // would be nowhere near the code that caused it. + // + // v1.43 (ios #77) — a `backfill` batch is a catch-up, not a sync. It records + // its types under `lastBackfillAt` only and moves none of the live stamps: + // thirty days of history must not read as thirty healthy daily syncs. + const backfill = syncTrigger === BACKFILL_SYNC_TRIGGER; if (failedCount === 0 && !scoped) { const syncedAt = new Date(); const deliveredWithoutTheApp = @@ -962,6 +1116,13 @@ async function postBatch(request: NextRequest): Promise { if (status !== "duplicate") entry.newSamples += 1; arrivals.set(type, entry); } + // The hours the phone posted are an arrival of their type as well. + for (const w of hourLanded) { + const entry = arrivals.get(w.type) ?? { accepted: 0, newSamples: 0 }; + entry.accepted += 1; + entry.newSamples += 1; + arrivals.set(w.type, entry); + } if (arrivals.size > 0) { await recordHealthKitTypeArrivals(prisma, { userId: user.id, @@ -970,21 +1131,23 @@ async function postBatch(request: NextRequest): Promise { at: syncedAt, }); } - await prisma.user.update({ - where: { id: user.id }, - data: { - lastSyncedAt: syncedAt, - ...(healthKitSyncSucceeded - ? { - healthKitLastSyncedAt: syncedAt, - healthKitLastSyncTrigger: syncTrigger ?? null, - ...(deliveredWithoutTheApp - ? { healthKitLastBackgroundSyncAt: syncedAt } - : {}), - } - : {}), - }, - }); + if (!backfill) { + await prisma.user.update({ + where: { id: user.id }, + data: { + lastSyncedAt: syncedAt, + ...(healthKitSyncSucceeded + ? { + healthKitLastSyncedAt: syncedAt, + healthKitLastSyncTrigger: syncTrigger ?? null, + ...(deliveredWithoutTheApp + ? { healthKitLastBackgroundSyncAt: syncedAt } + : {}), + } + : {}), + }, + }); + } } const freshlyInsertedRows = insertedPrepared.map((p) => ({ @@ -1251,10 +1414,21 @@ async function postBatch(request: NextRequest): Promise { } } + // v1.43 — the hours are not measurements, so they touched none of the + // measurement consumers above; only the caches that read the intraday + // profile (the same-time baseline) are marked stale. + if ( + hourInsertedCount + hourUpdatedCount > 0 && + insertedCount === 0 && + updatedCount === 0 + ) { + invalidateUserMeasurements(user.id); + } + const response = apiSuccess({ processed: entries.length, - inserted: insertedCount, - updated: updatedCount, + inserted: insertedCount + hourInsertedCount, + updated: updatedCount + hourUpdatedCount, duplicates: duplicateCount, failed: failedCount, skipped, diff --git a/src/app/api/medications/intake/bulk/__tests__/dose-arrival-scope.test.ts b/src/app/api/medications/intake/bulk/__tests__/dose-arrival-scope.test.ts index 3c8e9c0aa..18232033f 100644 --- a/src/app/api/medications/intake/bulk/__tests__/dose-arrival-scope.test.ts +++ b/src/app/api/medications/intake/bulk/__tests__/dose-arrival-scope.test.ts @@ -80,7 +80,9 @@ const USER = { }; /** A replayed Apple dose: the cheapest batch that still counts as an arrival. */ -async function postReplayedDose(): Promise { +async function postReplayedDose( + syncTrigger: string = "background", +): Promise { vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValueOnce([ { id: "row-existing", externalId: "hk-dose-1" }, ] as never); @@ -89,7 +91,7 @@ async function postReplayedDose(): Promise { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ - syncTrigger: "background", + syncTrigger, entries: [ { medicationId: "med-apple", @@ -134,4 +136,33 @@ describe("the dose arrival record and the credential that sent the batch", () => expect(res.status).toBe(200); expect(recordHealthKitTypeArrivals).not.toHaveBeenCalled(); }); + + // v1.43 (ios #77) — a history catch-up records the dose arrival under + // `lastBackfillAt` only, and does not move the sync checkpoint. + it("passes a backfill through and leaves the checkpoint where it was", async () => { + vi.mocked(requireAuth).mockResolvedValue({ + session: { id: "s" }, + user: USER, + } as never); + const res = await postReplayedDose("backfill"); + expect(res.status).toBe(200); + expect(recordHealthKitTypeArrivals).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ trigger: "backfill" }), + ); + expect(prisma.user.update).not.toHaveBeenCalled(); + }); + + it("moves the checkpoint for a live sync", async () => { + vi.mocked(requireAuth).mockResolvedValue({ + session: { id: "s" }, + user: USER, + } as never); + await postReplayedDose("background"); + expect(prisma.user.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ lastSyncedAt: expect.any(Date) }), + }), + ); + }); }); diff --git a/src/app/api/medications/intake/bulk/route.ts b/src/app/api/medications/intake/bulk/route.ts index 00bfc9d5d..085322060 100644 --- a/src/app/api/medications/intake/bulk/route.ts +++ b/src/app/api/medications/intake/bulk/route.ts @@ -58,7 +58,8 @@ import { withIdempotency } from "@/lib/idempotency"; import { checkRateLimit } from "@/lib/rate-limit"; import { markSyncCheckpoint } from "@/lib/sync/checkpoint"; import { - HEALTHKIT_SYNC_TRIGGERS, + BACKFILL_SYNC_TRIGGER, + HEALTHKIT_REQUEST_SYNC_TRIGGERS, MEDICATION_DOSE_ARRIVAL_TYPE, recordHealthKitTypeArrivals, } from "@/lib/integrations/healthkit-type-sync"; @@ -176,7 +177,9 @@ const bulkPayloadSchema = z.object({ // v1.42.4 — what woke the phone for this sync, the same diagnostic-only // field `POST /api/measurements/batch` takes. It changes no row; it only // feeds the `MEDICATION_DOSE` arrival record below. - syncTrigger: z.enum(HEALTHKIT_SYNC_TRIGGERS).optional(), + // v1.43 (ios #77) — `backfill` for a history catch-up: it moves only the + // dose entry's `lastBackfillAt`, never the live stamps or the checkpoint. + syncTrigger: z.enum(HEALTHKIT_REQUEST_SYNC_TRIGGERS).optional(), }); // v1.8.2 — `updated` joins the per-entry status vocabulary: a write that @@ -1038,8 +1041,10 @@ async function postBulk(request: NextRequest): Promise { // The SyncMode checkpoint belongs to the push that moved the data, not to // the `/api/sync/state` handshake that reports it. This endpoint is one of // the three domains that handshake summarises, so an intake drain advances - // it. - await markSyncCheckpoint(user.id); + // it. A `backfill` catch-up does not: it is history, not a sync. + if (syncTrigger !== BACKFILL_SYNC_TRIGGER) { + await markSyncCheckpoint(user.id); + } return apiSuccess({ processed: entries.length, diff --git a/src/components/settings/integrations/__tests__/apple-health-card.test.tsx b/src/components/settings/integrations/__tests__/apple-health-card.test.tsx index 2cee22c27..fac15f2be 100644 --- a/src/components/settings/integrations/__tests__/apple-health-card.test.tsx +++ b/src/components/settings/integrations/__tests__/apple-health-card.test.tsx @@ -455,6 +455,31 @@ describe(" — per-type arrivals (#1173)", () => { expect(out).not.toContain("Last in background"); }); + // v1.43 (ios #77) — a history catch-up is named apart, and a type that + // only ever arrived through one still reads as not received live. + it("names the last backfill apart from the live arrival", () => { + const live = html({ + ...base, + lastReceivedAt: hoursAgo(1), + lastTrigger: "foreground", + lastNewSampleAt: hoursAgo(1), + lastBackfillAt: hoursAgo(3), + }); + expect(live).toContain('data-state="received"'); + expect(live).toContain("History caught up 3 h ago"); + const backfillOnly = html({ + ...base, + lastReceivedAt: null, + lastTrigger: null, + lastNewSampleAt: null, + lastBackfillAt: hoursAgo(2), + }); + expect(backfillOnly).toContain('data-state="never"'); + expect(backfillOnly).toContain( + "Not received yet, History caught up 2 h ago", + ); + }); + it("names the medication dose row in the freshness list", () => { statusPayload = { lastSyncedAt: hoursAgo(1), diff --git a/src/components/settings/integrations/apple-health-card.tsx b/src/components/settings/integrations/apple-health-card.tsx index ee0431664..c2f909c11 100644 --- a/src/components/settings/integrations/apple-health-card.tsx +++ b/src/components/settings/integrations/apple-health-card.tsx @@ -148,16 +148,25 @@ export function ArrivalDetail({ now?: Date; }) { if (entry.lastReceivedAt === undefined) return null; - if (entry.lastReceivedAt === null) { - return ( - - {t("settings.appleHealth.freshness.never")} - - ); - } const reference = (now ?? new Date()).getTime(); const relative = (iso: string) => formatRelative(Math.max(0, reference - new Date(iso).getTime()), t); + // v1.43 (ios #77) — a history catch-up is named apart from the live + // arrivals, and never counts as one. + const backfill = entry.lastBackfillAt + ? t("settings.appleHealth.freshness.lastBackfill", { + relative: relative(entry.lastBackfillAt), + }) + : null; + if (entry.lastReceivedAt === null) { + return ( + + {[t("settings.appleHealth.freshness.never"), backfill] + .filter((part): part is string => part !== null) + .join(", ")} + + ); + } const trigger = entry.lastTrigger ?? null; const parts = [ t("settings.appleHealth.freshness.lastReceived", { @@ -189,6 +198,7 @@ export function ArrivalDetail({ }), ); } + if (backfill) parts.push(backfill); return ( { ).rejects.toThrow(/23 hourly slots, not 24/); expect(created, "a row every reader drops is worse than none").toEqual([]); }); + + // v1.43 (ios #77) — the posted hours travel with the day. + it("carries the posted hours, the basis and the hour count both ways", async () => { + const hours = Array.from({ length: 24 }, (_, h) => (h < 8 ? -1 : 100)); + const built = await buildIntradayProfileBackupSection( + { + intradayCumulativeProfile: { + findMany: vi.fn().mockResolvedValue([ + { + ...sourceRow(), + hourlyIncrements: hours, + basis: "HOUR_BUCKETS", + bucketCount: 16, + }, + ]), + }, + } as never, + "user-A", + ); + expect(built.intradayProfiles[0]).toMatchObject({ + hourlyIncrements: hours, + basis: "HOUR_BUCKETS", + bucketCount: 16, + }); + + const created: Array> = []; + await restoreIntradayProfileData(tx(created), "user-B", { + intradayProfiles: [built.intradayProfiles[0]], + }); + expect(created[0]).toMatchObject({ + hourlyIncrements: hours, + basis: "HOUR_BUCKETS", + bucketCount: 16, + }); + }); + + it("restores a day that so far holds only a few posted hours", async () => { + const hours = Array.from({ length: 24 }, (_, h) => (h === 9 ? 120 : -1)); + const created: Array> = []; + await restoreIntradayProfileData(tx(created), "user-B", { + intradayProfiles: [ + entry({ + hourlyCumulative: [], + dayTotal: 0, + sampleCount: 0, + hourlyIncrements: hours, + bucketCount: 1, + }), + ], + }); + expect(created[0]).toMatchObject({ hourlyCumulative: [], bucketCount: 1 }); + + await expect( + restoreIntradayProfileData(tx([]), "user-B", { + intradayProfiles: [entry({ hourlyIncrements: hours.slice(0, 3) })], + }), + ).rejects.toThrow(/3 posted-hour slots/); + }); }); diff --git a/src/lib/export/intraday-profile-backup.ts b/src/lib/export/intraday-profile-backup.ts index 3ce7caf8f..1be5b099e 100644 --- a/src/lib/export/intraday-profile-backup.ts +++ b/src/lib/export/intraday-profile-backup.ts @@ -59,6 +59,18 @@ export interface IntradayProfileBackupEntry { hourlyCumulative: number[]; dayTotal: number; sampleCount: number; + /** + * v1.43 (ios #77) — the hours the phone posted for the day, 24 slots with + * -1 for an hour not posted, or empty for a day folded from single samples + * only. A day with fewer than four posted hours keeps its posted hours here + * while its curve is still the fold's, or empty when the drain never saw + * the day. + */ + hourlyIncrements: number[]; + /** What `hourlyCumulative` was built from. */ + basis: "PER_SAMPLE" | "HOUR_BUCKETS"; + /** Posted hours behind the day; null when none were posted. */ + bucketCount: number | null; /** * The IANA zone the day was cut on. Carried because the reader compares it * against the account's current zone and drops the mismatched days; a @@ -113,6 +125,9 @@ export async function buildIntradayProfileBackupSection( hourlyCumulative: row.hourlyCumulative, dayTotal: row.dayTotal, sampleCount: row.sampleCount, + hourlyIncrements: row.hourlyIncrements, + basis: row.basis === "HOUR_BUCKETS" ? "HOUR_BUCKETS" : "PER_SAMPLE", + bucketCount: row.bucketCount, timezone: row.timezone, })), }; @@ -180,7 +195,24 @@ export async function restoreIntradayProfileData( } seen.add(key); - if (profile.hourlyCumulative.length !== PROFILE_BACKUP_HOURS) { + const increments = profile.hourlyIncrements ?? []; + if (increments.length !== 0 && increments.length !== PROFILE_BACKUP_HOURS) { + throw new Error( + `Intraday profile for ${profile.type} on ${profile.dateKey} carries ` + + `${increments.length} posted-hour slots, not ${PROFILE_BACKUP_HOURS} or none.`, + ); + } + // A day that so far holds only a few posted hours has no curve yet; the + // reader skips it until it does, and the hours are what it is restored + // for. Every other row must carry a full day. + const hoursOnly = + profile.hourlyCumulative.length === 0 && + increments.length === PROFILE_BACKUP_HOURS && + (profile.basis ?? "PER_SAMPLE") === "PER_SAMPLE"; + if ( + !hoursOnly && + profile.hourlyCumulative.length !== PROFILE_BACKUP_HOURS + ) { throw new Error( `Intraday profile for ${profile.type} on ${profile.dateKey} carries ` + `${profile.hourlyCumulative.length} hourly slots, not ${PROFILE_BACKUP_HOURS}. ` + @@ -200,6 +232,9 @@ export async function restoreIntradayProfileData( hourlyCumulative: profile.hourlyCumulative, dayTotal: profile.dayTotal, sampleCount: profile.sampleCount, + hourlyIncrements: profile.hourlyIncrements ?? [], + basis: profile.basis ?? "PER_SAMPLE", + bucketCount: profile.bucketCount ?? null, timezone: profile.timezone, ...(profile.createdAt ? { createdAt: new Date(profile.createdAt) } diff --git a/src/lib/integrations/connection-health-load.ts b/src/lib/integrations/connection-health-load.ts index a309f8253..5ab603ad5 100644 --- a/src/lib/integrations/connection-health-load.ts +++ b/src/lib/integrations/connection-health-load.ts @@ -15,6 +15,7 @@ * * Read-only. A connection that does not exist resolves to null. */ +import { BACKFILL_LEDGER_PREFIX } from "./healthkit-type-sync"; import { prisma } from "@/lib/db"; import type { MeasurementSource } from "@/generated/prisma/client"; import type { IntegrationKey } from "./status"; @@ -194,7 +195,12 @@ export async function loadConnectionFacts( : null, wanted.has("apple-health") ? prisma.healthKitTypeSync.findMany({ - where: { userId }, + // A backfill's own ledger row is a catch-up, not a live arrival, + // and must not make the connection look alive (ios #77). + where: { + userId, + NOT: { type: { startsWith: BACKFILL_LEDGER_PREFIX } }, + }, select: { type: true, lastReceivedAt: true, lastNewSampleAt: true }, orderBy: { lastReceivedAt: "desc" }, }) diff --git a/src/lib/integrations/healthkit-type-sync.ts b/src/lib/integrations/healthkit-type-sync.ts index 4c456242a..53f46e753 100644 --- a/src/lib/integrations/healthkit-type-sync.ts +++ b/src/lib/integrations/healthkit-type-sync.ts @@ -25,7 +25,10 @@ import { type MetricFreshnessEntry, } from "./sync-verdict"; -/** The `syncTrigger` values the Apple Health ingest routes accept. */ +/** + * The `syncTrigger` values that describe a live sync: the ones a batch can + * leave behind as a type's `lastTrigger`. + */ export const HEALTHKIT_SYNC_TRIGGERS = [ "foreground", "background", @@ -35,8 +38,39 @@ export const HEALTHKIT_SYNC_TRIGGERS = [ export type HealthKitSyncTrigger = (typeof HEALTHKIT_SYNC_TRIGGERS)[number]; +/** + * v1.43 (ios #77) — a history catch-up (the 30-day baseline backfill, a + * "sync everything" walk). Accepted on the two ingest routes when the server + * says so (`features.syncTriggerBackfill`). It is not a live sync, so it is + * never stored as `lastTrigger` and moves none of the live stamps; it moves + * only `lastBackfillAt`. + */ +export const BACKFILL_SYNC_TRIGGER = "backfill" as const; + +/** Every `syncTrigger` value the Apple Health ingest routes accept. */ +export const HEALTHKIT_REQUEST_SYNC_TRIGGERS = [ + ...HEALTHKIT_SYNC_TRIGGERS, + BACKFILL_SYNC_TRIGGER, +] as const; + +export type HealthKitRequestSyncTrigger = + (typeof HEALTHKIT_REQUEST_SYNC_TRIGGERS)[number]; + const TRIGGERS: ReadonlySet = new Set(HEALTHKIT_SYNC_TRIGGERS); +/** + * A backfill arrival is kept as its own ledger row, keyed `backfill:`, + * beside the live row of the same type. Its `last_received_at` is the + * backfill's time; it carries no trigger and no sample stamps. Keeping it in + * a row of its own is what lets a catch-up leave every live column of the type + * exactly as it was, and every live reader skips the prefix. + */ +export const BACKFILL_LEDGER_PREFIX = "backfill:"; + +export function isBackfillLedgerKey(type: string): boolean { + return type.startsWith(BACKFILL_LEDGER_PREFIX); +} + /** * The ledger key for Apple Health medication doses, which are not a * `MeasurementType`. Also the `type` of their `metricFreshness` entry. @@ -72,7 +106,7 @@ export async function recordHealthKitTypeArrivals( input: { userId: string; counts: ReadonlyMap; - trigger: HealthKitSyncTrigger | null; + trigger: HealthKitRequestSyncTrigger | null; at?: Date; }, ): Promise { @@ -88,6 +122,27 @@ export async function recordHealthKitTypeArrivals( // serialised in the process's local zone and these are zone-less UTC // columns (same reasoning as the access-token sunset in refresh-token.ts). const at = (input.at ?? new Date()).toISOString(); + if (input.trigger === BACKFILL_SYNC_TRIGGER) { + // v1.43 — a catch-up moves only its own row; the live row of each type + // keeps its arrival, trigger and sample stamps untouched. + const keys = types.map((type) => `${BACKFILL_LEDGER_PREFIX}${type}`); + try { + await tx.$executeRaw` + INSERT INTO "healthkit_type_syncs" + ("user_id", "type", "last_received_at") + SELECT ${input.userId}, u.t, ${at}::timestamptz AT TIME ZONE 'UTC' + FROM unnest(${keys}::text[]) AS u(t) + ON CONFLICT ("user_id", "type") DO UPDATE SET + "last_received_at" = EXCLUDED."last_received_at" + `; + } catch (error) { + logCaught("healthkit.type_sync.record_failed", error, { + types: types.length, + backfill: true, + }); + } + return; + } const background = input.trigger === "background"; try { await tx.$executeRaw` @@ -120,10 +175,13 @@ export async function recordHealthKitTypeArrivals( /** One type's arrival facts, as the Apple Health status read serves them. */ export interface HealthKitTypeArrivalRecord { - lastReceivedAt: string; + /** Null for a type that so far arrived only through a backfill. */ + lastReceivedAt: string | null; lastTrigger: HealthKitSyncTrigger | null; lastNewSampleAt: string | null; lastBackgroundAt: string | null; + /** v1.43 — when a `backfill` batch last carried the type; null when none has. */ + lastBackfillAt: string | null; } /** @@ -145,16 +203,30 @@ export async function getHealthKitTypeArrivals( }, }); const out = new Map(); + const empty = (): HealthKitTypeArrivalRecord => ({ + lastReceivedAt: null, + lastTrigger: null, + lastNewSampleAt: null, + lastBackgroundAt: null, + lastBackfillAt: null, + }); for (const row of rows) { - out.set(row.type, { - lastReceivedAt: row.lastReceivedAt.toISOString(), - lastTrigger: - row.lastTrigger && TRIGGERS.has(row.lastTrigger) - ? (row.lastTrigger as HealthKitSyncTrigger) - : null, - lastNewSampleAt: row.lastNewSampleAt?.toISOString() ?? null, - lastBackgroundAt: row.lastBackgroundAt?.toISOString() ?? null, - }); + if (isBackfillLedgerKey(row.type)) { + const type = row.type.slice(BACKFILL_LEDGER_PREFIX.length); + const record = out.get(type) ?? empty(); + record.lastBackfillAt = row.lastReceivedAt.toISOString(); + out.set(type, record); + continue; + } + const record = out.get(row.type) ?? empty(); + record.lastReceivedAt = row.lastReceivedAt.toISOString(); + record.lastTrigger = + row.lastTrigger && TRIGGERS.has(row.lastTrigger) + ? (row.lastTrigger as HealthKitSyncTrigger) + : null; + record.lastNewSampleAt = row.lastNewSampleAt?.toISOString() ?? null; + record.lastBackgroundAt = row.lastBackgroundAt?.toISOString() ?? null; + out.set(row.type, record); } return out; } @@ -195,6 +267,8 @@ export interface HealthKitFreshnessEntry extends MetricFreshnessEntry { lastNewSampleAt?: string | null; /** When a `background` sync last carried it; a later foreground one keeps it. */ lastBackgroundAt?: string | null; + /** v1.43 — when a `backfill` batch last carried it. */ + lastBackfillAt?: string | null; } /** @@ -220,6 +294,7 @@ export function withHealthKitArrivals( lastTrigger: arrival?.lastTrigger ?? null, lastNewSampleAt: arrival?.lastNewSampleAt ?? null, lastBackgroundAt: arrival?.lastBackgroundAt ?? null, + lastBackfillAt: arrival?.lastBackfillAt ?? null, }; }); } diff --git a/src/lib/measurements/__tests__/intraday-hour-buckets.test.ts b/src/lib/measurements/__tests__/intraday-hour-buckets.test.ts new file mode 100644 index 000000000..e7008f973 --- /dev/null +++ b/src/lib/measurements/__tests__/intraday-hour-buckets.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, it } from "vitest"; + +import { + classifyHourBucket, + cumulativeFromIncrements, + HOUR_NOT_POSTED, + isStartOfLocalHour, + parseHourBucketId, + postedHours, +} from "../intraday-hour-buckets"; + +const BERLIN = "Europe/Berlin"; +const NOW = new Date("2026-10-15T12:00:00.000Z"); + +describe("parseHourBucketId", () => { + it("reads the hour form", () => { + expect( + parseHourBucketId( + "stats:HKQuantityTypeIdentifierStepCount:2026-10-14T09", + ), + ).toEqual({ + hkIdentifier: "HKQuantityTypeIdentifierStepCount", + dateKey: "2026-10-14", + hour: 9, + }); + }); + + it.each([ + "stats:HKQuantityTypeIdentifierStepCount:2026-10-14", + "stats:HKQuantityTypeIdentifierHeartRate:2026-10-14T09", + "stats:HKQuantityTypeIdentifierHeartRate:2026-10-14T09:10:00.000Z", + "stats:HKQuantityTypeIdentifierStepCount:2026-10-14T24", + "stats:HKQuantityTypeIdentifierStepCount:2026-02-30T09", + "stats:HKQuantityTypeIdentifierStepCount:2026-10-14T9", + "uuid-1234", + ])("leaves %s to its own path", (id) => { + expect(parseHourBucketId(id)).toBeNull(); + }); +}); + +describe("isStartOfLocalHour", () => { + const id = { hkIdentifier: "x", dateKey: "2026-10-14", hour: 9 }; + it("accepts the first instant of the local hour only", () => { + // 09:00 in Berlin (CEST, UTC+2) is 07:00Z. + expect( + isStartOfLocalHour(new Date("2026-10-14T07:00:00.000Z"), id, BERLIN), + ).toBe(true); + expect( + isStartOfLocalHour(new Date("2026-10-14T09:00:00.000Z"), id, BERLIN), + ).toBe(false); + expect( + isStartOfLocalHour(new Date("2026-10-14T07:00:00.001Z"), id, BERLIN), + ).toBe(false); + expect( + isStartOfLocalHour(new Date("2026-10-14T07:30:00.000Z"), id, BERLIN), + ).toBe(false); + }); + + it("accepts either start of a repeated autumn hour and no start of a skipped spring hour", () => { + const repeated = { hkIdentifier: "x", dateKey: "2026-10-25", hour: 2 }; + expect( + isStartOfLocalHour( + new Date("2026-10-25T00:00:00.000Z"), + repeated, + BERLIN, + ), + ).toBe(true); + expect( + isStartOfLocalHour( + new Date("2026-10-25T01:00:00.000Z"), + repeated, + BERLIN, + ), + ).toBe(true); + const skipped = { hkIdentifier: "x", dateKey: "2026-03-29", hour: 2 }; + expect( + isStartOfLocalHour(new Date("2026-03-29T01:00:00.000Z"), skipped, BERLIN), + ).toBe(false); + }); +}); + +describe("classifyHourBucket", () => { + const steps = (dateKey: string, hour: number) => ({ + hkIdentifier: "HKQuantityTypeIdentifierStepCount", + dateKey, + hour, + }); + + it("maps the four summed identifiers", () => { + expect( + classifyHourBucket( + steps("2026-10-14", 9), + new Date("2026-10-14T07:00:00Z"), + BERLIN, + NOW, + ), + ).toMatchObject({ ok: true, type: "ACTIVITY_STEPS" }); + }); + + it("refuses another identifier", () => { + expect( + classifyHourBucket( + { + hkIdentifier: "HKQuantityTypeIdentifierBodyMass", + dateKey: "2026-10-14", + hour: 9, + }, + new Date("2026-10-14T07:00:00Z"), + BERLIN, + NOW, + ), + ).toEqual({ ok: false, reason: "hour_bucket_unsupported_type" }); + }); + + it("refuses a start that is not the local hour's start", () => { + // 09:00Z is 11:00 in Berlin, not 09:00. + expect( + classifyHourBucket( + steps("2026-10-14", 9), + new Date("2026-10-14T09:00:00Z"), + BERLIN, + NOW, + ), + ).toEqual({ ok: false, reason: "hour_bucket_misaligned" }); + }); + + it("refuses a day more than 400 days back", () => { + expect( + classifyHourBucket( + steps("2025-09-01", 9), + new Date("2025-09-01T07:00:00Z"), + BERLIN, + NOW, + ), + ).toEqual({ ok: false, reason: "hour_bucket_too_old" }); + expect( + classifyHourBucket( + steps("2025-09-11", 9), + new Date("2025-09-11T07:00:00Z"), + BERLIN, + NOW, + ), + ).toMatchObject({ ok: true }); + }); +}); + +describe("cumulativeFromIncrements", () => { + it("adds only posted hours and keeps the running total flat across gaps", () => { + const increments = new Array(24).fill(HOUR_NOT_POSTED); + increments[8] = 100; + increments[9] = 0; + increments[12] = 50; + const cumulative = cumulativeFromIncrements(increments); + expect(cumulative[7]).toBe(0); + expect(cumulative[8]).toBe(100); + expect(cumulative[11]).toBe(100); + expect(cumulative[23]).toBe(150); + expect(postedHours(increments)).toBe(3); + }); +}); diff --git a/src/lib/measurements/intraday-cumulative-profile.ts b/src/lib/measurements/intraday-cumulative-profile.ts index f05e95cd0..a37cc6cbd 100644 --- a/src/lib/measurements/intraday-cumulative-profile.ts +++ b/src/lib/measurements/intraday-cumulative-profile.ts @@ -110,6 +110,14 @@ export interface CumulativeDayFold { export interface CumulativeDayProfile extends CumulativeDayFold { /** Whether this came off the table or was folded live from raw rows. */ origin: "stored" | "live"; + /** + * v1.43 (ios #77) — what the running totals were built from: the fold of + * single samples, or the hours the phone posted (`intraday-hour-buckets`). + * `sampleCount` is the number of posted hours for the latter. + */ + basis: "PER_SAMPLE" | "HOUR_BUCKETS"; + /** Posted hours behind the day; null for a fold of single samples. */ + bucketCount: number | null; } /** @@ -207,9 +215,26 @@ export async function mergeIntradayCumulativeProfile( const existing = await tx.intradayCumulativeProfile.findUnique({ where: { userId_type_dateKey: key }, - select: { hourlyCumulative: true, sampleCount: true, timezone: true }, + select: { + hourlyCumulative: true, + sampleCount: true, + timezone: true, + basis: true, + }, }); + // v1.43 (ios #77) — a day built from the hours the phone posted keeps that + // shape: the posted hours already account for every sample, so adding a + // fold of the same samples would count the day twice. The fold is dropped, + // exactly as the samples it came from are about to be. + if ( + existing !== null && + existing.basis === "HOUR_BUCKETS" && + existing.timezone === increment.timezone + ) { + return; + } + const reusable = existing !== null && existing.timezone === increment.timezone && @@ -227,6 +252,11 @@ export async function mergeIntradayCumulativeProfile( dayTotal: hourlyCumulative[PROFILE_HOURS - 1], sampleCount, timezone: increment.timezone, + // A profile cut on another clock is replaced whole, its posted hours + // included (`basis` stays the default for a day under the threshold). + ...(existing !== null && existing.timezone !== increment.timezone + ? { hourlyIncrements: [], bucketCount: null, basis: "PER_SAMPLE" } + : {}), }; await tx.intradayCumulativeProfile.upsert({ @@ -300,6 +330,8 @@ export async function readCumulativeDayProfiles(args: { hourlyCumulative: true, dayTotal: true, sampleCount: true, + basis: true, + bucketCount: true, }, }); @@ -316,6 +348,8 @@ export async function readCumulativeDayProfiles(args: { dayTotal: row.dayTotal, sampleCount: row.sampleCount, origin: "stored", + basis: row.basis === "HOUR_BUCKETS" ? "HOUR_BUCKETS" : "PER_SAMPLE", + bucketCount: row.bucketCount, }); } @@ -326,7 +360,14 @@ export async function readCumulativeDayProfiles(args: { if (dateKey < startDateKey) break; if (byDay.has(dateKey)) continue; const folded = await foldLiveDay(db, userId, type, dateKey, timezone); - if (folded) byDay.set(dateKey, { ...folded, origin: "live" }); + if (folded) { + byDay.set(dateKey, { + ...folded, + origin: "live", + basis: "PER_SAMPLE", + bucketCount: null, + }); + } } return [...byDay.values()].sort((a, b) => (a.dateKey < b.dateKey ? 1 : -1)); diff --git a/src/lib/measurements/intraday-hour-buckets.ts b/src/lib/measurements/intraday-hour-buckets.ts new file mode 100644 index 000000000..0c9940b29 --- /dev/null +++ b/src/lib/measurements/intraday-hour-buckets.ts @@ -0,0 +1,271 @@ +/** + * Hour buckets for the summed intraday types (v1.43, ios #77). + * + * The phone can hand over a day of steps, active energy, walking distance or + * flights as twenty-four hourly sums (`stats::T`, the + * local hour in the account's zone) instead of the minute-level samples the + * nightly drain folds. That matters for history: a thirty-day backfill of + * hourly sums makes the same-time baseline usable on day one, where the + * drain only ever sees samples that are still on the phone's recent window. + * + * Such a row is NEVER a measurement. The day's total stays the + * `stats::` row the phone keeps posting; an hourly row stored + * as a measurement would be summed into that day a second time. Each hour + * lands as one slot of `IntradayCumulativeProfile.hourlyIncrements` instead, + * and once a day holds `HOUR_BUCKET_MIN_FOR_PROFILE` hours its running totals + * (`hourlyCumulative`) are rebuilt from the posted hours and the profile's + * `basis` reads `HOUR_BUCKETS`. Below that a day keeps the per-sample fold, + * if it has one. + * + * An hour not posted holds `HOUR_NOT_POSTED` (-1) in its slot; every value a + * posted hour can carry is zero or more, so the sentinel cannot collide with + * a reading, and `bucketCount` is the number of slots at zero or above. + */ +import type { + MeasurementType, + Prisma, + PrismaClient, +} from "@/generated/prisma/client"; +import { shiftDateKey, userDayKey } from "@/lib/tz/format"; +import { wallClockInTz } from "@/lib/tz/wall-clock"; + +import { + CUMULATIVE_PROFILE_RETENTION_DAYS, + PROFILE_HOURS, +} from "./intraday-cumulative-profile"; + +/** The HealthKit identifiers the hour form is defined for, and their types. */ +export const HOUR_BUCKET_TYPES: Readonly> = { + HKQuantityTypeIdentifierStepCount: "ACTIVITY_STEPS", + HKQuantityTypeIdentifierActiveEnergyBurned: "ACTIVE_ENERGY_BURNED", + HKQuantityTypeIdentifierDistanceWalkingRunning: "WALKING_RUNNING_DISTANCE", + HKQuantityTypeIdentifierFlightsClimbed: "FLIGHTS_CLIMBED", +}; + +/** Hours a day needs before its profile is built from buckets. */ +export const HOUR_BUCKET_MIN_FOR_PROFILE = 4; + +/** The oldest hour accepted: the profile's own retention bound. */ +export const HOUR_BUCKET_MAX_AGE_DAYS = CUMULATIVE_PROFILE_RETENTION_DAYS; + +/** The slot value of an hour the phone has not posted. */ +export const HOUR_NOT_POSTED = -1; + +export const HOUR_BUCKET_SKIP_REASONS = { + misaligned: "hour_bucket_misaligned", + unsupportedType: "hour_bucket_unsupported_type", + tooOld: "hour_bucket_too_old", +} as const; + +export type HourBucketSkipReason = + (typeof HOUR_BUCKET_SKIP_REASONS)[keyof typeof HOUR_BUCKET_SKIP_REASONS]; + +/** + * The heart-rate identifier keeps its own bucket contract (ten-minute ISO + * buckets, `malformed_hr_bucket_id`), so the hour form is not claimed for it. + */ +const HEART_RATE_HK = "HKQuantityTypeIdentifierHeartRate"; + +const HOUR_FORM = /^stats:([A-Za-z0-9]+):(\d{4}-\d{2}-\d{2})T(\d{2})$/; + +export interface HourBucketId { + hkIdentifier: string; + dateKey: string; + hour: number; +} + +/** + * Parse `stats::T`. Null for anything else, the day form + * and the heart-rate buckets included, so every other external id keeps the + * path it had. + */ +export function parseHourBucketId(externalId: string): HourBucketId | null { + const match = HOUR_FORM.exec(externalId); + if (!match) return null; + const [, hkIdentifier, dateKey, hh] = match; + if (hkIdentifier === HEART_RATE_HK) return null; + const hour = Number(hh); + if (!Number.isInteger(hour) || hour < 0 || hour >= PROFILE_HOURS) return null; + const [y, m, d] = dateKey.split("-").map(Number); + const probe = new Date(Date.UTC(y, m - 1, d)); + if ( + probe.getUTCFullYear() !== y || + probe.getUTCMonth() !== m - 1 || + probe.getUTCDate() !== d + ) { + return null; + } + return { hkIdentifier, dateKey, hour }; +} + +/** + * Whether `startDate` is the first instant of that local hour in the + * account's zone. Read through the zone rather than computed from it, so a + * repeated autumn hour accepts either of its two starts and a spring hour + * that does not exist accepts none. + */ +export function isStartOfLocalHour( + startDate: Date, + id: HourBucketId, + timezone: string, +): boolean { + if (startDate.getUTCMilliseconds() !== 0) return false; + const wall = wallClockInTz(startDate, timezone); + const key = `${String(wall.year).padStart(4, "0")}-${String(wall.month).padStart(2, "0")}-${String(wall.day).padStart(2, "0")}`; + return ( + key === id.dateKey && + wall.hour === id.hour && + wall.minute === 0 && + wall.second === 0 + ); +} + +export type HourBucketVerdict = + | { ok: true; type: MeasurementType; id: HourBucketId } + | { ok: false; reason: HourBucketSkipReason }; + +/** Decide whether one parsed hour row may be stored, and as which type. */ +export function classifyHourBucket( + id: HourBucketId, + startDate: Date, + timezone: string, + now: Date, +): HourBucketVerdict { + const type = HOUR_BUCKET_TYPES[id.hkIdentifier]; + if (!type) + return { ok: false, reason: HOUR_BUCKET_SKIP_REASONS.unsupportedType }; + const oldest = shiftDateKey( + userDayKey(now, timezone), + -HOUR_BUCKET_MAX_AGE_DAYS, + ); + if (id.dateKey < oldest) { + return { ok: false, reason: HOUR_BUCKET_SKIP_REASONS.tooOld }; + } + if (!isStartOfLocalHour(startDate, id, timezone)) { + return { ok: false, reason: HOUR_BUCKET_SKIP_REASONS.misaligned }; + } + return { ok: true, type, id }; +} + +/** Running totals from posted hours; an hour not posted adds nothing. */ +export function cumulativeFromIncrements( + increments: readonly number[], +): number[] { + const out = new Array(PROFILE_HOURS); + let running = 0; + for (let h = 0; h < PROFILE_HOURS; h += 1) { + const v = increments[h]; + if (v !== undefined && v >= 0) running += v; + out[h] = running; + } + return out; +} + +export function postedHours(increments: readonly number[]): number { + return increments.filter((v) => v >= 0).length; +} + +/** One accepted hour, as the batch hands it over. */ +export interface HourBucketWrite { + /** The entry's index in the batch, for its per-entry status. */ + index: number; + type: MeasurementType; + hkIdentifier: string; + dateKey: string; + hour: number; + /** The hour's sum in the canonical unit of `type`. */ + value: number; +} + +/** + * Store a batch's hours, one transaction per (type, day). Returns the + * per-entry status: `inserted` for the first value of a slot, `updated` when + * it replaced one. A later entry for the same slot in one batch wins, and the + * earlier ones read `updated` too, since the slot already held a value when + * they were applied in order. + * + * Serialised against the nightly drain's merge for the same day by the + * drain's own advisory lock (keyed on the day's `stats:` id), so a bucket + * write and a per-sample fold never interleave. + */ +export async function writeHourBuckets( + db: PrismaClient, + userId: string, + timezone: string, + writes: readonly HourBucketWrite[], +): Promise> { + const statuses = new Map(); + const groups = new Map(); + for (const w of writes) { + const key = `${w.type}::${w.dateKey}`; + const list = groups.get(key); + if (list) list.push(w); + else groups.set(key, [w]); + } + + for (const group of groups.values()) { + const { type, dateKey, hkIdentifier } = group[0]; + await db.$transaction(async (tx) => { + await tx.$queryRaw` + SELECT 1 AS locked + FROM pg_advisory_xact_lock(hashtextextended('drain-cumulative:' || ${userId} || ':' || ${`stats:${hkIdentifier}:${dateKey}`}, 0)) + `; + const key = { userId, type, dateKey }; + const existing = await tx.intradayCumulativeProfile.findUnique({ + where: { userId_type_dateKey: key }, + select: { + hourlyIncrements: true, + hourlyCumulative: true, + dayTotal: true, + sampleCount: true, + timezone: true, + basis: true, + }, + }); + // Hours cut on another clock mean other hours: start the day over. + const sameZone = existing !== null && existing.timezone === timezone; + const increments = + sameZone && existing.hourlyIncrements.length === PROFILE_HOURS + ? [...existing.hourlyIncrements] + : new Array(PROFILE_HOURS).fill(HOUR_NOT_POSTED); + for (const w of group) { + statuses.set(w.index, increments[w.hour] >= 0 ? "updated" : "inserted"); + increments[w.hour] = w.value; + } + const bucketCount = postedHours(increments); + const fromBuckets = bucketCount >= HOUR_BUCKET_MIN_FOR_PROFILE; + const cumulative = cumulativeFromIncrements(increments); + // Below the threshold the day keeps whatever the per-sample fold left + // (nothing, for a day the drain never saw): the reader skips a profile + // without 24 running totals. + const keepFold = sameZone && !fromBuckets; + const data = { + hourlyIncrements: increments, + bucketCount, + basis: fromBuckets ? "HOUR_BUCKETS" : "PER_SAMPLE", + timezone, + hourlyCumulative: fromBuckets + ? cumulative + : keepFold + ? existing.hourlyCumulative + : [], + dayTotal: fromBuckets + ? cumulative[PROFILE_HOURS - 1] + : keepFold + ? existing.dayTotal + : 0, + sampleCount: fromBuckets + ? bucketCount + : keepFold + ? existing.sampleCount + : 0, + } satisfies Prisma.IntradayCumulativeProfileUncheckedUpdateInput; + await tx.intradayCumulativeProfile.upsert({ + where: { userId_type_dateKey: key }, + create: { ...key, ...data }, + update: data, + }); + }); + } + return statuses; +} diff --git a/src/lib/openapi/routes/cycle.ts b/src/lib/openapi/routes/cycle.ts index d60b379f1..27b0d342c 100644 --- a/src/lib/openapi/routes/cycle.ts +++ b/src/lib/openapi/routes/cycle.ts @@ -333,18 +333,22 @@ const cycleDeleteOutcomeEnvelope = dataEnvelope( "CycleDeleteOutcomeEnvelope", ); -/** `Prefer: return=representation` on the two deletes that can remove a start. */ -const preferReturnRepresentationParameter = { +/** + * `Prefer: return=representation` on the two deletes that can remove a start. + * A factory, so each operation carries its own copy: a shared object becomes a + * YAML anchor and renumbers every other anchor in the document. + */ +const preferReturnRepresentationParameter = () => ({ name: "Prefer", in: "header" as const, required: false, schema: { type: "string" as const, example: "return=representation" }, description: "v1.43 — send `return=representation` (RFC 7240) to get 200 with what the delete changed (`CycleDeleteOutcome`) and a `Preference-Applied: return=representation` header. Without it the route answers 204 with no body, as before.", -}; +}); /** The 200 a delete answers with when the request preferred a representation. */ -const cycleDeleteOutcomeResponse = { +const cycleDeleteOutcomeResponse = () => ({ description: "v1.43 — with `Prefer: return=representation`: the removed cycle (null when the delete removed none, e.g. a day that opened no cycle or a repeated delete) and the starts the removal gave back.", content: { @@ -355,7 +359,7 @@ const cycleDeleteOutcomeResponse = { description: "The route honoured `Prefer: return=representation`.", }), }), -}; +}); const cyclePeriodResponse = z.object({ cycle: menstrualCycleDto.nullable(), @@ -834,9 +838,9 @@ export const cyclePaths: NonNullable = { description: "Sets `deletedAt` + bumps `syncVersion`; surfaces as a tombstone on the next `/api/sync/changes` page. A day that opened a cycle takes that cycle with it, which gives back any start that cycle had folded in. 204, or 200 with `CycleDeleteOutcome` under `Prefer: return=representation`. Idempotent.", requestParams: { path: z.object({ id: z.string() }) }, - parameters: [preferReturnRepresentationParameter], + parameters: [preferReturnRepresentationParameter()], responses: { - "200": cycleDeleteOutcomeResponse, + "200": cycleDeleteOutcomeResponse(), "204": { description: "Soft-deleted (no body)." }, "404": { description: "Day-log not found / not owned.", @@ -991,9 +995,9 @@ export const cyclePaths: NonNullable = { description: "Sets `deletedAt` + bumps `syncVersion`; tombstones on the next sync page. Any start this cycle had folded in stands again. 204, or 200 with `CycleDeleteOutcome` under `Prefer: return=representation`. Idempotent.", requestParams: { path: z.object({ id: z.string() }) }, - parameters: [preferReturnRepresentationParameter], + parameters: [preferReturnRepresentationParameter()], responses: { - "200": cycleDeleteOutcomeResponse, + "200": cycleDeleteOutcomeResponse(), "204": { description: "Soft-deleted (no body)." }, "404": { description: "Cycle not found / not owned.", diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index d145a24bf..ee7b945b2 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -92,10 +92,11 @@ const batchEntrySchema = z .min(1) .max(120) .describe( - "Dedup key for the `(userId, type, source, externalId)` composite index. Three shapes:\n" + + "Dedup key for the `(userId, type, source, externalId)` composite index. Four shapes:\n" + "• `HKSample.uuid` — a per-sample reading. First-write-wins: a re-post returns `duplicate` and the stored row is immutable.\n" + "• `stats::` — a per-day cumulative total (Steps, Active Energy, Sleep Duration, Walking/Running Distance, Flights Climbed). A re-post OVERWRITES the row and returns `updated`.\n" + "• `stats:HKQuantityTypeIdentifierHeartRate:` — v1.19.0 (iOS #34), refined to 10-minute granularity in v1.30.7, heart-rate bucket carrying the bucket's AVERAGE bpm as one PULSE row. `` is the ISO-8601 UTC 10-minute boundary — zeroed seconds/millis, minutes ∈ {00,10,20,30,40,50}, trailing `Z` (e.g. `2026-06-21T14:10:00.000Z`). A re-post OVERWRITES the row and returns `updated`, so iOS uploads ~144 rows/day instead of one per raw HR sample. A row that targets this prefix with a malformed bucket-start suffix is `skipped` with reason `malformed_hr_bucket_id`.\n" + + "• `stats::T` — v1.43 (ios #77), only when `GET /api/auth/me` reports `features.cumulativeHourBuckets: true`: one local hour's SUM of steps, active energy, walking and running distance or flights climbed, the date and the zero-padded hour being the local hour in the account's zone (`timezone` on `/api/auth/me`). `value` is the hour's sum in the unit of the day form, `startDate` the first instant of that local hour, `endDate` one hour later. Such a row is never a measurement and never changes the day total, which stays the day form: it feeds the intraday profile behind the same-time baseline (a day with at least four posted hours is built from them). A re-post overwrites (`updated`). Skipped with `hour_bucket_misaligned` when `startDate` is not the start of that local hour, `hour_bucket_unsupported_type` for any other identifier or a `MANUAL` / bridge entry, `hour_bucket_too_old` when the day is more than 400 days back.\n" + "The id MUST be stable across app restarts. An object description or memory address (``, `0x12ab34cd`) is a new value on every launch, so it can never match its own earlier row; such an entry is `skipped` with reason `unstable_external_id` while the rest of the batch still lands.", ), externalSourceVersion: z.string().min(1).max(120).optional(), @@ -132,10 +133,10 @@ const batchPayloadSchema = z // used in attribution — sending it or omitting it produces byte-identical // row outcomes. Omit on any client that does not track its wake reason. syncTrigger: z - .enum(["foreground", "background", "push", "manual"]) + .enum(["foreground", "background", "push", "manual", "backfill"]) .optional() .describe( - 'Diagnostic-only. Names what triggered this sync (foreground app open, background refresh, a push wake, or a user-initiated `manual` "Sync all" run, since v1.42). Recorded on the ingest wide event and kept on the account as `lastSyncTrigger` — plus `lastBackgroundSyncAt` for a `background` or `push` trigger — both readable from `GET /api/integrations/healthkit`. Since v1.42 it is also recorded per measurement type the batch carried (Apple Health rows only), served as `lastReceivedAt` / `lastTrigger` / `lastNewSampleAt` on each `metricFreshness` entry of that read, plus `lastBackgroundAt` (since v1.42.4) for a `background` trigger. It does not affect dedup, attribution, or how any sample is stored. Optional and backward-compatible: pre-#66 clients omit it.', + 'Diagnostic-only. Names what triggered this sync (foreground app open, background refresh, a push wake, or a user-initiated `manual` "Sync all" run, since v1.42). `backfill` (v1.43, ios #77; send it only when `GET /api/auth/me` reports `features.syncTriggerBackfill: true`, an older server refuses the whole batch with 422) names a history catch-up: it moves only `syncProgress` and the per-type `lastBackfillAt` in `GET /api/integrations/healthkit`, never `lastReceivedAt`, `lastTrigger`, `lastBackgroundAt`, `lastSyncTrigger`, `lastBackgroundSyncAt` or the sync checkpoint. Recorded on the ingest wide event and kept on the account as `lastSyncTrigger` — plus `lastBackgroundSyncAt` for a `background` or `push` trigger — both readable from `GET /api/integrations/healthkit`. Since v1.42 it is also recorded per measurement type the batch carried (Apple Health rows only), served as `lastReceivedAt` / `lastTrigger` / `lastNewSampleAt` on each `metricFreshness` entry of that read, plus `lastBackgroundAt` (since v1.42.4) for a `background` trigger. It does not affect dedup, attribution, or how any sample is stored. Optional and backward-compatible: pre-#66 clients omit it.', ), }) .meta({ @@ -156,7 +157,7 @@ const batchEntryResult = z .string() .optional() .describe( - "Why a row was `skipped` or a `duplicate`. `folded_window` (since v1.42): a raw sample of a dense or mean type whose local day lies before the fold boundary and whose hour or day a live `stats:` row of the same source already covers. Nothing to retry; treat it like any duplicate.", + "Why a row was `skipped` or a `duplicate`. `folded_window` (since v1.42): a raw sample of a dense or mean type whose local day lies before the fold boundary and whose hour or day a live `stats:` row of the same source already covers. Nothing to retry; treat it like any duplicate. `hour_bucket_misaligned`, `hour_bucket_unsupported_type`, `hour_bucket_too_old` (since v1.43): an hourly `stats::T` row the server refused, see `externalId`; permanent for the row as sent.", ), convertedValue: z .number() diff --git a/src/lib/openapi/routes/medications/paths.ts b/src/lib/openapi/routes/medications/paths.ts index ac6ecd4e5..7ac5aa6dd 100644 --- a/src/lib/openapi/routes/medications/paths.ts +++ b/src/lib/openapi/routes/medications/paths.ts @@ -199,10 +199,10 @@ const bulkIntakePayload = z .object({ entries: z.array(bulkIntakeEntry).min(1).max(500), syncTrigger: z - .enum(["foreground", "background", "push", "manual"]) + .enum(["foreground", "background", "push", "manual", "backfill"]) .optional() .describe( - 'Diagnostic-only, the same field `POST /api/measurements/batch` takes (since v1.42.4): what woke the client for this sync. When the batch carries entries with `source: "APPLE_HEALTH"` that land (`inserted`, `updated` or `duplicate`), it is recorded on the `MEDICATION_DOSE` entry of `metricFreshness` in `GET /api/integrations/healthkit` as `lastTrigger`, and a `background` value also dates its `lastBackgroundAt`. It changes no dose row. Optional; omitted records a null trigger.', + 'Diagnostic-only, the same field `POST /api/measurements/batch` takes (since v1.42.4): what woke the client for this sync. `backfill` (v1.43, only when `features.syncTriggerBackfill` is `true`) marks a history catch-up, which moves only the `MEDICATION_DOSE` entry\'s `lastBackfillAt` and leaves its live stamps and the sync checkpoint alone. When the batch carries entries with `source: "APPLE_HEALTH"` that land (`inserted`, `updated` or `duplicate`), it is recorded on the `MEDICATION_DOSE` entry of `metricFreshness` in `GET /api/integrations/healthkit` as `lastTrigger`, and a `background` value also dates its `lastBackgroundAt`. It changes no dose row. Optional; omitted records a null trigger.', ), }) .meta({ diff --git a/src/lib/validations/__tests__/backup.test.ts b/src/lib/validations/__tests__/backup.test.ts index 42d16f7af..9a965226c 100644 --- a/src/lib/validations/__tests__/backup.test.ts +++ b/src/lib/validations/__tests__/backup.test.ts @@ -210,6 +210,69 @@ describe("backupPayloadSchema — v1.28 backup-completeness domains", () => { } }); + // A qualitative marker is minted with an empty unit and its readings are + // stored under it; the file the account's own backup wrote must parse. + describe("units of qualitative lab markers", () => { + const base = { + schemaVersion: "1", + exportedAt: "2026-05-08T07:00:00.000Z", + userId: "u1", + }; + const qualitativeMarker = { id: "bm-1", name: "Urine glucose", unit: "" }; + const qualitativeReading = { + analyte: "Urine glucose", + value: null, + valueText: "negative", + unit: "", + takenAt: "2026-04-01T09:00:00.000Z", + source: "MANUAL", + biomarkerId: "bm-1", + biomarkerName: "Urine glucose", + }; + + it("accepts an empty unit on a qualitative marker and its reading", () => { + const r = backupPayloadSchema.safeParse({ + ...base, + biomarkers: [qualitativeMarker], + labResults: [qualitativeReading], + }); + expect(r.success).toBe(true); + }); + + it("still needs the unit on a numeric reading", () => { + const r = backupPayloadSchema.safeParse({ + ...base, + labResults: [ + { + ...qualitativeReading, + value: 5, + valueText: null, + biomarkerId: null, + }, + ], + }); + expect(r.success).toBe(false); + expect(r.error?.issues.map((i) => i.path.join("."))).toEqual([ + "labResults.0.unit", + ]); + }); + + it("refuses an empty unit on a marker a numeric reading is filed under", () => { + const r = backupPayloadSchema.safeParse({ + ...base, + biomarkers: [qualitativeMarker], + labResults: [ + qualitativeReading, + { ...qualitativeReading, value: 5, valueText: null, unit: "mg/dL" }, + ], + }); + expect(r.success).toBe(false); + expect(r.error?.issues.map((i) => i.path.join("."))).toEqual([ + "biomarkers.0.unit", + ]); + }); + }); + it("round-trips an illness episode with a flare referencing its parent + nested day-logs", () => { const r = backupPayloadSchema.safeParse({ schemaVersion: "1", diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index 3cdb869bb..71a813486 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -971,6 +971,12 @@ const intradayProfileBackupSchema = z hourlyCumulative: z.array(z.number()), dayTotal: z.number(), sampleCount: z.number().int(), + // v1.43 (ios #77) — the hours the phone posted, what the curve was built + // from, and how many hours there were. Defaulted so a file written before + // the hours existed restores as a fold of single samples, which it was. + hourlyIncrements: z.array(z.number()).default([]), + basis: z.enum(["PER_SAMPLE", "HOUR_BUCKETS"]).default("PER_SAMPLE"), + bucketCount: z.number().int().nonnegative().nullable().default(null), // The zone the day was cut on. The reader compares it against the // account's current zone and drops the mismatched days, so a restore that // dropped this column would make old curves look comparable when their @@ -1098,7 +1104,10 @@ const labResultBackupSchema = z analyte: z.string().min(1), value: z.number().nullable().optional(), valueText: z.string().nullable().optional(), - unit: z.string().min(1), + // A qualitative reading ("positive", "trace") has no unit and is stored + // under its marker's empty one (`POST /api/labs`); the refine below keeps + // the unit required for every numeric reading. + unit: z.string(), referenceLow: z.number().nullable().optional(), referenceHigh: z.number().nullable().optional(), // Optional so a backup written before these columns existed still @@ -1120,13 +1129,29 @@ const labResultBackupSchema = z deletedAt: isoDateTime.nullable().optional(), updatedAt: isoDateTime.optional(), }) - .passthrough(); + .passthrough() + .superRefine((reading, ctx) => { + const qualitative = + (reading.value === null || reading.value === undefined) && + typeof reading.valueText === "string" && + reading.valueText.length > 0; + if (reading.unit.length === 0 && !qualitative) { + ctx.addIssue({ + code: "custom", + path: ["unit"], + message: "A numeric lab reading needs its unit", + }); + } + }); const biomarkerBackupSchema = z .object({ id: z.string().min(1).optional(), name: z.string().min(1), - unit: z.string().min(1), + // Empty for a qualitative marker, minted from a reading with no unit. The + // payload refine below refuses an empty unit on a marker a numeric + // reading is filed under. + unit: z.string(), lowerBound: z.number().nullable().optional(), upperBound: z.number().nullable().optional(), panel: z.string().nullable().optional(), @@ -2224,6 +2249,30 @@ export const backupPayloadSchema = z }) .passthrough() .superRefine((payload, ctx) => { + // A marker without a unit is a qualitative one. A numeric reading filed + // under it would restore as a number with no unit, so the file is refused + // naming the marker rather than restored into that. + const numericByMarker = new Set(); + for (const reading of payload.labResults) { + if (reading.value === null || reading.value === undefined) continue; + if (reading.biomarkerId) numericByMarker.add(`id:${reading.biomarkerId}`); + if (reading.biomarkerName) { + numericByMarker.add(`name:${reading.biomarkerName}`); + } + } + payload.biomarkers.forEach((marker, index) => { + if (marker.unit.length > 0) return; + if ( + (marker.id && numericByMarker.has(`id:${marker.id}`)) || + numericByMarker.has(`name:${marker.name}`) + ) { + ctx.addIssue({ + code: "custom", + path: ["biomarkers", index, "unit"], + message: `Biomarker "${marker.name}" has numeric readings and needs its unit`, + }); + } + }); if (payload.schemaVersion !== BACKUP_SCHEMA_VERSION) return; payload.measurements.forEach((measurement, index) => { if (!measurement.id) { diff --git a/tests/integration/backup-qualitative-lab-restore.test.ts b/tests/integration/backup-qualitative-lab-restore.test.ts new file mode 100644 index 000000000..c0f0dc071 --- /dev/null +++ b/tests/integration/backup-qualitative-lab-restore.test.ts @@ -0,0 +1,137 @@ +/** + * A backup holding a qualitative lab marker restores. + * + * A qualitative reading ("positive", "trace") has no unit, so its marker is + * minted with an empty unit and the reading stored under it + * (`POST /api/labs`, `isQualitative`). The backup schema demanded a non-empty + * unit on both rows, so the file the account's own backup wrote could not be + * read back: the parse refused it and the restore never started. Asserted on + * rows through the real writer, the real wire format, the real parser and the + * real restore, with the marker removed first so the file is the only copy. + * + * The empty unit is accepted for a qualitative reading and a marker none of + * whose readings is numeric; a numeric reading, or a marker a numeric reading + * is filed under, still needs its unit. + */ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { encrypt } from "@/lib/crypto"; +import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { parseBackupPayload } from "@/lib/validations/backup"; +import { POST as RESTORE } from "./restore-job-driver"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/cache/invalidate", async (importOriginal) => { + const actual = + await importOriginal(); + return { ...actual, invalidateUserData: vi.fn() }; +}); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +describe("a backup with a qualitative lab marker", () => { + it("restores the marker and its reading", async () => { + const prisma = getPrismaClient(); + const owner = await prisma.user.create({ + data: { + username: "qualitative-lab-owner", + email: "qualitative-lab-owner@example.test", + role: "ADMIN", + }, + }); + const session = await prisma.session.create({ + data: { userId: owner.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + + // The real write path mints the marker with an empty unit. + const { POST: CREATE_LAB } = await import("@/app/api/labs/route"); + const created = await CREATE_LAB( + new NextRequest("http://localhost/api/labs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + analyte: "Urine glucose", + valueText: "negative", + takenAt: "2026-07-01T08:00:00.000Z", + }), + }), + ); + expect(created.status).toBe(201); + const marker = await prisma.biomarker.findFirstOrThrow({ + where: { userId: owner.id }, + }); + expect(marker.unit).toBe(""); + + const { payload: built } = await buildFullBackupPayload(prisma, owner.id, { + purpose: "disaster-recovery", + }); + // The parse is where the file used to be refused. + const parsed = parseBackupPayload(JSON.stringify(built)); + const backup = await prisma.dataBackup.create({ + data: { + userId: owner.id, + type: "QUALITATIVE_LAB_ROUND_TRIP", + data: encrypt(JSON.stringify(parsed)), + }, + }); + + await prisma.labResult.deleteMany({ where: { userId: owner.id } }); + await prisma.biomarker.deleteMany({ where: { userId: owner.id } }); + + const response = await RESTORE( + new Request(`http://localhost/api/admin/backups/${backup.id}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }) as unknown as Parameters[0], + { params: Promise.resolve({ id: backup.id }) }, + ); + expect(response.status).toBe(200); + + const restoredMarker = await prisma.biomarker.findFirstOrThrow({ + where: { userId: owner.id, name: "Urine glucose" }, + }); + expect(restoredMarker.unit).toBe(""); + const restoredReading = await prisma.labResult.findFirstOrThrow({ + where: { userId: owner.id }, + }); + expect(restoredReading).toMatchObject({ + biomarkerId: restoredMarker.id, + value: null, + valueText: "negative", + unit: "", + }); + }); +}); diff --git a/tests/integration/backup-round-trip.test.ts b/tests/integration/backup-round-trip.test.ts index 1f2497960..a47ed5359 100644 --- a/tests/integration/backup-round-trip.test.ts +++ b/tests/integration/backup-round-trip.test.ts @@ -3644,14 +3644,7 @@ const COLUMN_PRESENCE_ONLY: Readonly> = { * starts coming back, so the entry has to go in the change that carries it. * The list is empty again before the release is tagged. */ -const RELEASE_PENDING_COLUMNS: Readonly> = { - "IntradayCumulativeProfile.hourlyIncrements": - "hour buckets for summed types (ios #77): the intraday section carries it with the hour-bucket ingest", - "IntradayCumulativeProfile.basis": - "hour buckets for summed types (ios #77): the intraday section carries it with the hour-bucket ingest", - "IntradayCumulativeProfile.bucketCount": - "hour buckets for summed types (ios #77): the intraday section carries it with the hour-bucket ingest", -}; +const RELEASE_PENDING_COLUMNS: Readonly> = {}; /** * Columns every reader parses through one function before using, compared @@ -3680,6 +3673,12 @@ const FILL_OVERRIDES: Readonly> = { "CorrelationPattern.dismissedEvidenceHash": "ab".repeat(32), // CHECK in migration 0408: the profile is built from samples or from hour buckets. "IntradayCumulativeProfile.basis": "HOUR_BUCKETS", + // The posted hours are a whole day of slots (-1 for an hour not posted); + // the restore refuses any other length, as it does for the curve. + "IntradayCumulativeProfile.hourlyIncrements": Array.from( + { length: 24 }, + (_, hour) => (hour < 6 ? -1 : 350), + ), }; /** diff --git a/tests/integration/hour-buckets-and-backfill.test.ts b/tests/integration/hour-buckets-and-backfill.test.ts new file mode 100644 index 000000000..299b76dfa --- /dev/null +++ b/tests/integration/hour-buckets-and-backfill.test.ts @@ -0,0 +1,384 @@ +/** + * Hour buckets for summed types and the `backfill` trigger (ios #77), against + * real Postgres. + * + * An hourly `stats::T` row must never become a + * measurement: the day total stays the day row, and an hour stored as a + * measurement would be counted into that day a second time. The hours land in + * the intraday profile instead, which is what makes the same-time baseline + * usable on day one after a thirty-day backfill. And a backfill is a + * catch-up, so it must not move the live sync stamps. + */ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +process.env.API_TOKEN_HMAC_KEY ??= + "test-hmac-key-hour-buckets-backfill-32-bytes-minimum-1234"; + +const USER_ID = "user-hour-buckets"; +const STEPS = "HKQuantityTypeIdentifierStepCount"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + await getPrismaClient().user.create({ + data: { + id: USER_ID, + username: "hour-buckets", + email: "hour-buckets@example.test", + timezone: "UTC", + }, + }); + const session = await getPrismaClient().session.create({ + data: { userId: USER_ID, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.set("healthlog_session", session.id); +}); + +async function postBatch(body: Record) { + const { POST } = await import("@/app/api/measurements/batch/route"); + const res = await POST( + new NextRequest("http://localhost/api/measurements/batch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ); + expect(res.status).toBe(200); + return (await res.json()).data as { + inserted: number; + updated: number; + entries: Array<{ index: number; status: string; reason?: string }>; + }; +} + +function dayKeyBack(days: number): string { + return new Date(Date.now() - days * 86_400_000).toISOString().slice(0, 10); +} + +function hourEntry(dateKey: string, hour: number, value: number) { + const hh = String(hour).padStart(2, "0"); + const start = `${dateKey}T${hh}:00:00.000Z`; + return { + hkIdentifier: STEPS, + value, + unit: "count", + startDate: start, + endDate: new Date(Date.parse(start) + 3_600_000).toISOString(), + externalId: `stats:${STEPS}:${dateKey}T${hh}`, + }; +} + +function dayEntry(dateKey: string, value: number) { + return { + hkIdentifier: STEPS, + value, + unit: "count", + startDate: `${dateKey}T00:00:00.000Z`, + endDate: `${dateKey}T12:00:00.000Z`, + externalId: `stats:${STEPS}:${dateKey}`, + }; +} + +describe("hour buckets for summed types", () => { + it("keeps the day total as the only measurement and builds the profile from the hours", async () => { + const day = dayKeyBack(2); + const hours = Array.from({ length: 24 }, (_, h) => + hourEntry(day, h, h * 10), + ); + const total = hours.reduce((s, e) => s + e.value, 0); + + const first = await postBatch({ + entries: [dayEntry(day, total), ...hours], + }); + expect(first.entries.slice(1).every((e) => e.status === "inserted")).toBe( + true, + ); + + const rows = await getPrismaClient().measurement.findMany({ + where: { userId: USER_ID, type: "ACTIVITY_STEPS" }, + select: { value: true, externalId: true }, + }); + // The guard: no hour of a summed type ever reaches `measurements`. + expect(rows).toEqual([ + { value: total, externalId: `stats:${STEPS}:${day}` }, + ]); + + const profile = + await getPrismaClient().intradayCumulativeProfile.findUniqueOrThrow({ + where: { + userId_type_dateKey: { + userId: USER_ID, + type: "ACTIVITY_STEPS", + dateKey: day, + }, + }, + }); + expect(profile.basis).toBe("HOUR_BUCKETS"); + expect(profile.bucketCount).toBe(24); + expect(profile.hourlyIncrements).toEqual(hours.map((e) => e.value)); + expect(profile.hourlyCumulative[23]).toBe(total); + expect(profile.dayTotal).toBe(total); + expect(profile.timezone).toBe("UTC"); + + // A re-post of one hour overwrites it. + const again = await postBatch({ entries: [hourEntry(day, 5, 999)] }); + expect(again.entries[0]).toMatchObject({ status: "updated" }); + expect(again.updated).toBe(1); + const after = + await getPrismaClient().intradayCumulativeProfile.findFirstOrThrow({ + where: { userId: USER_ID, dateKey: day }, + }); + expect(after.hourlyIncrements[5]).toBe(999); + expect(after.dayTotal).toBe(total - 50 + 999); + // And the day row did not move. + const dayRow = await getPrismaClient().measurement.findFirstOrThrow({ + where: { userId: USER_ID, type: "ACTIVITY_STEPS" }, + }); + expect(dayRow.value).toBe(total); + }); + + it("keeps a day under four hours on its per-sample shape", async () => { + const day = dayKeyBack(3); + await postBatch({ + entries: [hourEntry(day, 8, 100), hourEntry(day, 9, 200)], + }); + const profile = + await getPrismaClient().intradayCumulativeProfile.findFirstOrThrow({ + where: { userId: USER_ID, dateKey: day }, + }); + expect(profile.basis).toBe("PER_SAMPLE"); + expect(profile.bucketCount).toBe(2); + expect(profile.hourlyCumulative).toEqual([]); + + // The drain's fold then fills the curve; the posted hours stay put. + const { mergeIntradayCumulativeProfile } = + await import("@/lib/measurements/intraday-cumulative-profile"); + await getPrismaClient().$transaction((tx) => + mergeIntradayCumulativeProfile(tx, { + userId: USER_ID, + type: "ACTIVITY_STEPS", + increment: { + dateKey: day, + timezone: "UTC", + hourlyCumulative: Array.from({ length: 24 }, (_, h) => + h >= 8 ? 300 : 0, + ), + dayTotal: 300, + sampleCount: 30, + }, + }), + ); + const folded = + await getPrismaClient().intradayCumulativeProfile.findFirstOrThrow({ + where: { userId: USER_ID, dateKey: day }, + }); + expect(folded.dayTotal).toBe(300); + expect(folded.hourlyIncrements[9]).toBe(200); + }); + + it("does not let the per-sample fold count a bucket-built day twice", async () => { + const day = dayKeyBack(4); + await postBatch({ + entries: Array.from({ length: 6 }, (_, h) => hourEntry(day, h + 8, 100)), + }); + const { mergeIntradayCumulativeProfile } = + await import("@/lib/measurements/intraday-cumulative-profile"); + await getPrismaClient().$transaction((tx) => + mergeIntradayCumulativeProfile(tx, { + userId: USER_ID, + type: "ACTIVITY_STEPS", + increment: { + dateKey: day, + timezone: "UTC", + hourlyCumulative: new Array(24).fill(600), + dayTotal: 600, + sampleCount: 60, + }, + }), + ); + const profile = + await getPrismaClient().intradayCumulativeProfile.findFirstOrThrow({ + where: { userId: USER_ID, dateKey: day }, + }); + expect(profile.basis).toBe("HOUR_BUCKETS"); + expect(profile.dayTotal).toBe(600); + expect(profile.sampleCount).toBe(6); + }); + + it("names every refusal", async () => { + const day = dayKeyBack(2); + const misaligned = { + ...hourEntry(day, 9, 10), + startDate: `${day}T09:30:00.000Z`, + }; + const unsupported = { + ...hourEntry(day, 9, 10), + hkIdentifier: "HKQuantityTypeIdentifierBodyMass", + unit: "kg", + externalId: `stats:HKQuantityTypeIdentifierBodyMass:${day}T09`, + }; + const old = hourEntry("2024-01-02", 9, 10); + const result = await postBatch({ entries: [misaligned, unsupported, old] }); + expect(result.entries.map((e) => e.reason)).toEqual([ + "hour_bucket_misaligned", + "hour_bucket_unsupported_type", + "hour_bucket_too_old", + ]); + expect( + await getPrismaClient().intradayCumulativeProfile.count({ + where: { userId: USER_ID }, + }), + ).toBe(0); + }); + + it("makes the same-time baseline usable on day one after a thirty-day backfill", async () => { + const entries = []; + for (let back = 31; back >= 1; back -= 1) { + const day = dayKeyBack(back); + for (let h = 0; h < 24; h += 1) entries.push(hourEntry(day, h, 50 + h)); + } + for (let i = 0; i < entries.length; i += 400) { + await postBatch({ + syncTrigger: "backfill", + entries: entries.slice(i, i + 400), + }); + } + const { computeSameTimeBaseline } = + await import("@/lib/insights/derived/same-time-baseline"); + // "Today" is the last backfilled day, at 20:30. + const now = new Date(`${dayKeyBack(1)}T20:30:00.000Z`); + const derived = await computeSameTimeBaseline(USER_ID, null, { + type: "ACTIVITY_STEPS", + now, + }); + expect(derived.status).toBe("ok"); + }); +}); + +describe('syncTrigger "backfill"', () => { + it("records lastBackfillAt and moves none of the live stamps", async () => { + const live = await postBatch({ + syncTrigger: "foreground", + entries: [ + { + hkIdentifier: "HKQuantityTypeIdentifierHeartRate", + value: 61, + unit: "count/min", + startDate: "2026-10-01T06:00:00.000Z", + endDate: "2026-10-01T06:00:00.000Z", + externalId: "uuid-live-1", + }, + ], + }); + expect(live.inserted).toBe(1); + const prisma = getPrismaClient(); + const userBefore = await prisma.user.findUniqueOrThrow({ + where: { id: USER_ID }, + select: { + lastSyncedAt: true, + healthKitLastSyncedAt: true, + healthKitLastSyncTrigger: true, + healthKitLastBackgroundSyncAt: true, + }, + }); + const ledgerBefore = await prisma.healthKitTypeSync.findUniqueOrThrow({ + where: { userId_type: { userId: USER_ID, type: "PULSE" } }, + }); + + const backfill = await postBatch({ + syncTrigger: "backfill", + entries: [ + { + hkIdentifier: "HKQuantityTypeIdentifierHeartRate", + value: 63, + unit: "count/min", + startDate: "2026-09-01T06:00:00.000Z", + endDate: "2026-09-01T06:00:00.000Z", + externalId: "uuid-backfill-1", + }, + ], + }); + expect(backfill.inserted).toBe(1); + + expect( + await prisma.user.findUniqueOrThrow({ + where: { id: USER_ID }, + select: { + lastSyncedAt: true, + healthKitLastSyncedAt: true, + healthKitLastSyncTrigger: true, + healthKitLastBackgroundSyncAt: true, + }, + }), + ).toEqual(userBefore); + expect( + await prisma.healthKitTypeSync.findUniqueOrThrow({ + where: { userId_type: { userId: USER_ID, type: "PULSE" } }, + }), + ).toEqual(ledgerBefore); + const backfillRow = await prisma.healthKitTypeSync.findUniqueOrThrow({ + where: { userId_type: { userId: USER_ID, type: "backfill:PULSE" } }, + }); + expect(backfillRow.lastTrigger).toBeNull(); + + // The status read serves it, and the live facts are the foreground ones. + const { GET } = await import("@/app/api/integrations/healthkit/route"); + const res = await (GET as () => Promise)(); + const body = (await res.json()).data as { + metricFreshness: Array>; + syncProgress: { recordsAccepted: number } | null; + }; + const pulse = body.metricFreshness.find((e) => e.type === "PULSE"); + expect(pulse).toMatchObject({ + lastReceivedAt: ledgerBefore.lastReceivedAt.toISOString(), + lastTrigger: "foreground", + lastBackfillAt: backfillRow.lastReceivedAt.toISOString(), + }); + // syncProgress counts what arrived, the backfill included. + expect(body.syncProgress?.recordsAccepted).toBe(2); + expect( + body.metricFreshness.some((e) => String(e.type).startsWith("backfill:")), + ).toBe(false); + }); + + it("is refused by name for anything outside the enum", async () => { + const { POST } = await import("@/app/api/measurements/batch/route"); + const res = await POST( + new NextRequest("http://localhost/api/measurements/batch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + syncTrigger: "history", + entries: [dayEntry("2026-10-01", 1)], + }), + }), + ); + expect(res.status).toBe(422); + }); +});