From f6ca89b9637efdb6c7b9042f41f6d61419566cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 16 May 2026 22:26:59 +0200 Subject: [PATCH] =?UTF-8?q?v1.4.34.1=20=E2=80=94=20insights=20cold-mount?= =?UTF-8?q?=20perf=20hotfix=20+=20scatter-card=20sizing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hotfix on top of v1.4.34. The Insights page mount was paying ~29 s on every cold load against accounts populated by the v1.4.34 Apple Health importer. Root cause: `/api/insights/comprehensive` walked the unbounded 90-day measurement set into JS, then ran per-type `summarize()`, BMI, blood-pressure classification, target adherence, and four Pearson correlations from the same pile of rows. On a 100 000+ row account the route consumed one of the 20 pool connections for the whole 29 s, which cascaded sibling endpoints into Cloudflare "no available server" 503s. The scatter-correlation card also surfaced a `width(-1)/height(-1)` Recharts warning on first paint because its dynamic-mount lacked a non-zero floor before aspect-ratio resolved. Changed: - /api/insights/comprehensive now reads through a SQL-side aggregator (src/lib/insights/comprehensive-aggregator.ts) grouping by metric type with Postgres-native AVG / MIN / MAX / REGR_SLOPE / REGR_R2 on the measurements (user_id, type, measured_at) index path. Returns the same DataSummary-shaped per-type bundle the legacy route stitched together; the medication-compliance block keeps its bounded Prisma reads. Wrapped through the server cache keyed on `${userId}|comprehensive` (60 s TTL). Cold path drops from ~29 s to a few hundred ms against fixture data matching the production volume that triggered the bug; warm path resolves in single-digit milliseconds. - Cache-wrap the five remaining read routes the v1.4.34 server-cache blueprint left staged but unwired: /api/mood/analytics, /api/workouts, /api/bugreport/status, /api/medications, /api/dashboard/widgets. The Cloudflare 503 cascade on /api/measurements?aggregate=daily resolves as a downstream consequence — once the comprehensive route stops monopolising the pool, the origin accepts new connections fast enough that Cloudflare's circuit-breaker no longer trips. - /api/dashboard/chart-overlay-prefs PUT now invalidates the new dashboard-widgets cache so the partial-update path doesn't leave stale layout state in the cache for up to TTL. Fixed: - Scatter-correlation card no longer logs the Recharts `width(-1)/height(-1)` warning on first mount. The wrapper now carries a min-h-[180px] floor that matches the loading skeleton so ResponsiveContainer always measures a non-zero parent before aspect-ratio settles. --- .planning/research/v1434-r-1-xml-import.md | 743 ++++++++++++++++++ .../research/v1434-r-2-carryover-scope.md | 439 +++++++++++ .../research/v1434-r-cache-aggregation.md | 429 ++++++++++ .planning/round-v1433-aasa-rollout-report.md | 141 ++++ .planning/round-v1434-closure-report.md | 480 +++++++++++ .planning/round-v1434-iwc-report.md | 74 ++ .planning/round-v1434-iwd-report.md | 169 ++++ .planning/round-v1434-iwg-report.md | 153 ++++ ...round-v1434-prod-slowness-investigation.md | 336 ++++++++ CHANGELOG.md | 51 ++ package.json | 2 +- src/app/api/bugreport/status/route.ts | 55 +- .../dashboard/chart-overlay-prefs/route.ts | 6 + src/app/api/dashboard/widgets/route.ts | 24 +- .../comprehensive/__tests__/route.test.ts | 306 ++++++++ src/app/api/insights/comprehensive/route.ts | 293 ++++--- src/app/api/medications/route.ts | 64 +- src/app/api/mood/analytics/route.ts | 31 +- .../__tests__/canonical-dedup.test.ts | 7 + src/app/api/workouts/route.ts | 190 +++-- .../charts/scatter-correlation-chart.tsx | 8 +- .../comprehensive-aggregator.test.ts | 175 +++++ src/lib/insights/comprehensive-aggregator.ts | 373 +++++++++ .../insights-comprehensive-cache.test.ts | 282 +++++++ 24 files changed, 4590 insertions(+), 241 deletions(-) create mode 100644 .planning/research/v1434-r-1-xml-import.md create mode 100644 .planning/research/v1434-r-2-carryover-scope.md create mode 100644 .planning/research/v1434-r-cache-aggregation.md create mode 100644 .planning/round-v1433-aasa-rollout-report.md create mode 100644 .planning/round-v1434-closure-report.md create mode 100644 .planning/round-v1434-iwc-report.md create mode 100644 .planning/round-v1434-iwd-report.md create mode 100644 .planning/round-v1434-iwg-report.md create mode 100644 .planning/round-v1434-prod-slowness-investigation.md create mode 100644 src/app/api/insights/comprehensive/__tests__/route.test.ts create mode 100644 src/lib/insights/__tests__/comprehensive-aggregator.test.ts create mode 100644 src/lib/insights/comprehensive-aggregator.ts create mode 100644 tests/integration/insights-comprehensive-cache.test.ts diff --git a/.planning/research/v1434-r-1-xml-import.md b/.planning/research/v1434-r-1-xml-import.md new file mode 100644 index 000000000..236884789 --- /dev/null +++ b/.planning/research/v1434-r-1-xml-import.md @@ -0,0 +1,743 @@ +# R-1 — Apple Health `export.zip` import (v1.4.34) + +Implementation blueprint for the streaming Apple Health XML import that +lands immediately before the web-freeze marker. The brief in +`.planning/v15-strategic-plan.md` §2 carves the scope; this document +fills in the parser pick, the queue model, the parse loop, the +ingestion shape, the response envelope, the admin variant, and the +freeze-marker wording. Every section names files, dependencies, and +the upstream code the implementation reuses. + +## 1. Recommendation summary + +- Parser pick: `sax` (event-driven SAX, no DOM, peak RSS independent + of file size). `fast-xml-parser` already shipped transitively in + `pnpm-lock.yaml` line 3955 but its streaming mode buffers per + element which still allocates for the ~10 M `` peak inside + a multi-year export. `sax` is purpose-built for "drain a large XML + file with bounded memory". Bundle is ~30 kB. +- Archive unpack: Node 22 ships a `node:zlib` `inflateRawSync` that + pairs with a small inline ZIP central-directory walker; we add no + new archive dependency. The implementation extracts `export.xml` + directly from the upload stream rather than materialising the full + zip on disk. +- Async-job storage pick: `pg-boss` (already in dependencies at + `^12.18.2`, full worker scaffolding live in + `src/lib/jobs/reminder-worker.ts`). Reuse the existing + `boss.send` / `boss.work` / `boss.getJobById` surface; no new + process, no new infra footprint. +- Job phases: `queued → unpacking → parsing → upserting → done` with + a single `failed` terminal. Per-phase progress carried on the + `pg-boss` job `output` JSON column (pg-boss v12 supports incremental + `boss.complete(jobId, output)` only at completion; we therefore + also maintain a small `ImportJob` Prisma row keyed by `pg-boss` + `jobId` for live progress reads). +- Effort: L (~3-4 days) — matches §2 of the strategic plan. + +## 2. `export.zip` archive layout + +Every Apple Health export the user creates via *Health.app → profile +picture → Export All Health Data* unzips to a single top-level +`apple_health_export/` directory. Members observed across every +ecosystem-scan reference (`apple-health-grafana`, +`healthkit-to-sqlite`, `apple-health-parser`, `BRO3886/healthsync`, +`atlas`, `health-data-hub`): + +| Member | Purpose | v1.4.34 handling | +|---|---|---| +| `apple_health_export/export.xml` | The full sample stream — ~95 % of payload by bytes. Single self-contained XML file. Every ``, ``, ``, ``, `` lives here. | **Primary** parse target. | +| `apple_health_export/export_cda.xml` | Continuity-of-Care Document (HL7 CDA R2). Duplicate of the clinical subset already mirrored as `` rows in `export.xml`. | Ignore — clinical defers per R-F T3. | +| `apple_health_export/workout-routes/route_*.gpx` | One GPX file per `HKWorkoutRoute` (referenced from the matching `` via ``). | Parse if the matching `` survives mapping; persist as `WorkoutRoute.geometry` GeoJSON. | +| `apple_health_export/electrocardiograms/ecg_*.csv` | One CSV per HKElectrocardiogram. | Skip — no HealthLog ECG model; record count under `unknown.electrocardiogram`. | +| `apple_health_export/clinical-records/*.json` | FHIR R4 resources mirroring `` blocks. | Skip — defers per R-F T3. | +| `apple_health_export/export.zip.checksum` | Optional integrity marker (recent iOS versions). | Ignore. | + +The parser logs every unrecognised member name under `unknown.` +in the response so we can spot new directory members on future iOS +releases without breaking the import. + +## 3. `export.xml` schema (parser-facing data model) + +The single XML file opens with the literal sequence: + +```xml + + + ... +]> + + + + + + + ... + +``` + +Element model the parser must handle: + +- `` — root, single instance. Attribute `locale` is + advisory. +- `` — single `value` attribute (string-formatted local + date with timezone offset). Stored on the import job for audit. +- `` — characteristic attributes (DOB, biological sex, blood + type, Fitzpatrick skin type, wheelchair-use). v1.4.34 ignores; + HealthLog has its own profile fields and overwriting them mid- + import would surprise the user. +- `` — the bulk of the stream. The `type` + attribute carries the literal `HKQuantityTypeIdentifier*` or + `HKCategoryTypeIdentifier*` string already handled by + `APPLE_HEALTH_TYPE_MAP` in + `src/lib/measurements/apple-health-mapping.ts`. Every Record + carries `startDate`, `endDate`, `value`, `unit`, `sourceName`, + `sourceVersion`, `creationDate`, `device`. Sleep-stage Records + carry `value` as a string codepoint (`"HKCategoryValueSleepAnalysis" + + "InBed" | "Asleep" | "Awake" | "AsleepCore" | "AsleepDeep" | + "AsleepREM"`); the parser converts the string back to the integer + codepoint via a small table so it can ride + `APPLE_HEALTH_SLEEP_STAGE_MAP`. Children: zero or more + ``. The `HKMetadataKeyExternalUUID` + entry, when present, is the `HKSample.uuid`; otherwise the parser + hashes `(type, value, startDate, endDate)` to a stable + `externalId`. +- `` — one per `HKWorkout`. + Attributes: `workoutActivityType` (the `HKWorkoutActivityType` + string, e.g. `HKWorkoutActivityTypeRunning`), `duration`, + `durationUnit`, `totalDistance`, `totalDistanceUnit`, + `totalEnergyBurned`, `totalEnergyBurnedUnit`, `sourceName`, + `sourceVersion`, `device`, `creationDate`, `startDate`, `endDate`. + Children: `` (zero or more, `HKMetadataKeyExternalUUID` + again the natural id), `` (pause / resume / lap markers), + `` with one nested + `` linking to the + GPX file. +- `` — Apple's daily ring rollup. v1.4.34 + ignores (HealthLog computes its own daily aggregates from + `` rows via the existing `dailyStatsExternalId` path). +- `` with two nested `` children + (`HKQuantityTypeIdentifierBloodPressureSystolic` + + `HKQuantityTypeIdentifierBloodPressureDiastolic`). The parser + flattens the Correlation envelope and treats each child Record + exactly like a top-level Record — the systolic / diastolic mapping + in `APPLE_HEALTH_TYPE_MAP` already handles the row shape. +- `...` — FHIR clinical records. + Skipped per R-F T3 (defers to v1.6+). + +## 4. Parser pick + +Three Node candidates were considered: + +| Lib | Mode | Peak RSS at 1 GB input | TypeScript | Existing dep? | +|---|---|---|---|---| +| `sax` | event SAX (callback per token) | ~30 MB constant | ships `.d.ts` from `@types/sax` | no (one new dep) | +| `fast-xml-parser` (streaming mode) | per-element callback over a `XMLParser` with `processEntities: false` | ~150 MB at 1 GB — buffers each element | first-class TS | yes (transitive) | +| `node-expat` | C++ libexpat bindings | ~25 MB constant | manual `.d.ts` | no — requires node-gyp build | + +Pick: **`sax` (~0.4 MB install size, MIT, last release 2024-06)**. The +SAX callback model gives us per-token streaming (each +`onopentag` / `onclosetag` / `ontext` fires as the parser advances +the byte cursor), peak RSS stays ~30 MB regardless of file size, +TypeScript types exist as `@types/sax`. `node-expat` would be marginally +faster but the node-gyp build step breaks the Coolify Docker image +(we explicitly keep the build chain extension-free per the same +decision that picked GeoJSON over PostGIS for `WorkoutRoute`). The +transitive `fast-xml-parser` stays where it is for HTML-ish +fast-path usage elsewhere; we don't add it to the import path. + +`pnpm add sax @types/sax` lands two new lines in `package.json`. Both +under 50 kB unpacked. + +## 5. Async-job architecture + +Reuse the live `pg-boss` infrastructure verbatim. The implementation +adds two queues and one Prisma model: + +### 5.1 New `pg-boss` queue: `apple-health-import` + +Register in `src/lib/jobs/reminder-worker.ts:1432-1454` next to the +existing `MEDICATION_INVENTORY_EXPIRE_QUEUE` block. Concurrency cap +`1` per host (one giant XML at a time per worker — the parse loop is +CPU-bound and an OOM on a second concurrent 1 GB import would knock +the host over). Retry policy: `retryLimit: 0` (a re-import is a +human-driven re-upload, not an automatic retry). Job payload: + +```ts +interface AppleHealthImportPayload { + /** Owner of the imported rows. The job runs in this user's scope. */ + userId: string; + /** Operator who triggered the import (admin variant only). When unset, + * the user kicked off their own import. */ + triggeredByAdminId?: string; + /** Absolute path on the worker's filesystem where the multipart-upload + * handler dropped the `export.zip`. The worker streams from here and + * unlinks on completion. */ + uploadPath: string; + /** Size in bytes for the response envelope + audit-log row. */ + uploadBytes: number; + /** Wall-clock kick-off so duration is computable even when the worker + * picks the job up after a queue delay. */ + enqueuedAt: string; +} +``` + +The handler in `src/lib/jobs/apple-health-import-worker.ts` (new +file) follows the `pr-detection-worker.ts` shape: one exported +`handleAppleHealthImport(job: PgBossJob)` +function plus a small unit-tested helper extracted to +`src/lib/measurements/parse-export-xml.ts`. + +### 5.2 New Prisma model: `ImportJob` + +`pg-boss` v12's `boss.getJobById` returns the boss row but the +`output` column only populates on terminal completion. For the +polling endpoint to read *live* progress (the user is staring at a +spinner), we mirror the job into a HealthLog-side row: + +```prisma +model ImportJob { + id String @id @default(cuid()) + userId String @map("user_id") + triggeredByAdminId String? @map("triggered_by_admin_id") + pgBossJobId String @unique @map("pg_boss_job_id") + /// `queued | unpacking | parsing | upserting | done | failed` + status String @default("queued") + /// Free-text reason on `failed`; null otherwise. + failureReason String? @map("failure_reason") + uploadBytes Int @map("upload_bytes") + startedAt DateTime @map("started_at") @default(now()) + completedAt DateTime? @map("completed_at") + /// JSON envelope matching `ImportJobProgress` (§8). Updated mid-run + /// from the worker on every 1 000 records parsed. + progress Json @default("{}") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@index([userId, startedAt]) + @@map("import_jobs") +} +``` + +The unique constraint on `pgBossJobId` lets the polling endpoint +read by the boss id the client received from the synchronous +endpoint. The migration is additive only (no existing column changes). + +### 5.3 Synchronous endpoint — kick-off + +`POST /api/import/apple-health-export` (new file +`src/app/api/import/apple-health-export/route.ts`): + +1. `requireAuth()` from `src/lib/api-handler.ts`. Cookie or Bearer + (any scope — wildcard works per v1.4.25 W10 fix-C). +2. `checkRateLimit("import:apple-health:${user.id}", 3, 60_000)` — + three uploads per minute per user is generous for any legitimate + workflow. +3. `Content-Length` ceiling: `MAX_UPLOAD_BYTES = 1.5 * 1024 * 1024 * + 1024` (1.5 GB — Apple's largest observed export is around 800 MB + for a 10-year iCloud-synced account; 1.5 GB leaves margin). + Reject 413 above. **No** in-memory `request.formData()` — Next.js + 16's built-in handler buffers the full body. Switch to streaming + via the experimental Web Streams `request.body` plus a manual + multipart boundary walker (see §5.4 for the streaming path). +4. Hash the upload into `/tmp/healthlog-import-.zip` while the + stream lands. Compute a SHA-256 inline so re-uploading the exact + same file returns the previous `ImportJob` id without re-queueing + (idempotency-on-content rather than `Idempotency-Key` — the iOS + client cannot reliably emit a stable key for a giant file). +5. `prisma.importJob.create()` with `status: "queued"`. +6. `boss.send("apple-health-import", payload)` returns the boss + `jobId`. Store on the `ImportJob` row. +7. Respond `202 Accepted` with `{ jobId: importJob.id, status: + "queued" }`. The 202 is the contract the strategic plan §2 locks. + +### 5.4 Multipart streaming notes + +Next.js 16's `request.formData()` is fine up to ~10 MB but allocates +the full body on a 1 GB upload. The kick-off route therefore reads +`request.body` directly (a `ReadableStream` per Web +Streams) and walks the multipart boundary by hand — pattern identical +to the one shipped in `src/app/api/admin/backups/upload/route.ts:85` +for backup uploads, but extended to stream-to-disk instead of +`file.text()`. A small helper +`src/lib/multipart/stream-to-disk.ts` extracts the named field and +writes it to a temp file with a SHA-256 hash computed in the same +pass. + +### 5.5 Status endpoint + +`GET /api/import/apple-health-export/[jobId]/status` (new file +`src/app/api/import/apple-health-export/[jobId]/status/route.ts`): + +1. `requireAuth()`. +2. `prisma.importJob.findUnique({ where: { id: jobId } })`. +3. 404 when missing or `userId !== requester.id` (admin variant + short-circuits this — see §9). +4. Reply with the envelope in §8. Caching: `Cache-Control: no-store`. + +Polling cadence: client polls every 2 s during `queued | unpacking | +parsing | upserting`, every 30 s once `done | failed` (last-known +state is fine). + +## 6. Worker process and parse loop + +The handler lives at +`src/lib/jobs/apple-health-import-worker.ts`. Pseudocode: + +```ts +import sax from "sax"; +import { createReadStream, unlinkSync, statSync } from "node:fs"; +import { mapAppleHealthEntry, APPLE_HEALTH_TYPE_MAP, dailyStatsExternalId, + CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; + +export async function handleAppleHealthImport(job) { + const { userId, uploadPath, uploadBytes } = job.data; + const importJob = await prisma.importJob.findUnique({ + where: { pgBossJobId: job.id }, + }); + if (!importJob) throw new Error("ImportJob row missing"); + + await tick(importJob.id, "unpacking", { recordsRead: 0 }); + const xmlPath = await unzipExportXml(uploadPath); // see §6.1 + + await tick(importJob.id, "parsing", { recordsRead: 0 }); + const stats = await streamParse(xmlPath, userId); // see §6.2 + + await tick(importJob.id, "upserting", stats); + await flushUpserts(userId, stats); // see §6.3 + + await tick(importJob.id, "done", stats); + unlinkSync(uploadPath); + unlinkSync(xmlPath); +} +``` + +### 6.1 Unzip step + +Streaming-unzip the upload's `apple_health_export/export.xml` into +`/tmp/healthlog-import-.xml`. Node 22's `node:stream/promises` ++ a tiny ZIP central-directory walker — see +`https://github.com/MichalLytek/yauzl/blob/master/lib/yauzl.js` for a +reference implementation. The walker is ~60 LOC; we inline it under +`src/lib/import/unzip-export-xml.ts` rather than pull a new +dependency. Also extracts `workout-routes/*.gpx` to a temp directory +for later resolution. + +### 6.2 Stream parse step + +```ts +const parser = sax.parser(/* strict */ true, { trim: true }); +const buffer: PreparedRow[] = []; +const stats: PerTypeStats = freshStats(); +const cumulativeBucket = new Map(); // key = type:date + +parser.onopentag = (node) => { + if (node.name === "Record") { + const r = mapAppleHealthEntry({ + hkIdentifier: node.attributes.type, + value: parseRecordValue(node.attributes), + unit: node.attributes.unit ?? "", + startDate: node.attributes.startDate, + endDate: node.attributes.endDate, + sleepStage: parseSleepStage(node.attributes.value), // §3 + }); + if (!r) { + stats.unknown[node.attributes.type] = (stats.unknown[node.attributes.type] ?? 0) + 1; + return; + } + if (CUMULATIVE_HK_TYPES.has(r.type)) { + // Fold into per-day bucket; flush bucket to DB at end of file + const dayKey = formatDayKey(r.takenAt, user.timezone); + const bucketKey = `${r.type}:${dayKey}`; + cumulativeBucket.set(bucketKey, (cumulativeBucket.get(bucketKey) ?? 0) + r.value); + } else { + buffer.push(prepareSpotRow(userId, r, node)); + if (buffer.length >= 500) flushBatch(buffer); + } + stats.perType[r.type].read += 1; + } else if (node.name === "Workout") { + /* drain into prepareWorkoutRow + flushWorkoutBatch at 100 */ + } else if (node.name === "ClinicalRecord") { + stats.clinical.skipped += 1; + } else if (node.name === "ActivitySummary") { + // ignore — daily-stats path mints rows from + } +}; +parser.onerror = (err) => { /* mark failed */ }; +parser.onend = async () => { await flushAllBuffers(); }; + +await pipeline(createReadStream(xmlPath, { highWaterMark: 64 * 1024 }), + saxAsWritable(parser)); +``` + +Implementation notes: + +- Sleep-stage `` rows carry their value as the symbolic name + `HKCategoryValueSleepAnalysisAsleepDeep`. Translate via a small + `SLEEP_STAGE_NAME_TO_CODEPOINT` table that mirrors + `APPLE_HEALTH_SLEEP_STAGE_MAP` inverted; the codepoint then rides + the existing `mapAppleHealthEntry()` path. For sleep, the row's + numeric `value` is `(endDate - startDate) / 60_000` minutes (the + same convention `mapAppleHealthEntry` expects). +- `externalId` derivation: + - Spot row, `HKMetadataKeyExternalUUID` present → `sample:`. + - Spot row, no external UUID → `sample:` + truncated to 28 chars (~110 bit collision budget; well inside + the `min(1).max(120)` Zod cap on `externalId`). + - Cumulative row (after the per-day fold) → + `dailyStatsExternalId(hkIdentifier, "YYYY-MM-DD")`. +- The day-key for cumulative folds anchors to the user's timezone + (`User.timezone`) — same convention + `src/lib/measurements/drain-per-sample-cumulative.ts` uses via + `dayKeyForUserTz`. This makes the imported `stats:` rows + interoperate cleanly with iOS-emitted `stats:` rows: the same day + collides on the same `externalId` so a re-import after an iOS + cutover is a true UPSERT. + +### 6.3 Upsert / flush step + +The flush helpers run inside a loop of small Prisma transactions — +one tx per 500 spot rows or per 100 workout rows. UPSERT shape: + +```ts +await prisma.measurement.upsert({ + where: { + userId_type_source_externalId: { + userId, + type: row.type, + source: "APPLE_HEALTH", + externalId: row.externalId, + }, + }, + create: row, + update: { value: row.value, measuredAt: row.measuredAt }, +}); +``` + +The `userId_type_source_externalId` compound unique already exists +on `Measurement` at `prisma/schema.prisma:444` so no migration. For +the cumulative-bucket flush, the value written is the SUM across the +day; updating an existing row replaces the SUM (matches the drain +script's semantics — a re-import is a fresh authoritative roll-up). + +Workouts UPSERT against `userId_source_externalId` at +`prisma/schema.prisma:512`. Sport-type mapping from +`HKWorkoutActivityType*` to the `workoutSportTypeEnum` Zod union +(`src/lib/validations/workout.ts:21-42`) lives in a new helper +`src/lib/measurements/hk-workout-activity-type-map.ts` — same +flat-table shape as `APPLE_HEALTH_TYPE_MAP`. Unknown activity types +fall through to `"other"` (the union's escape hatch). Unrecognised +counts land in `stats.workouts.unknownActivityType`. + +`WorkoutRoute.geometry` is materialised by parsing the linked GPX +file into a GeoJSON LineString — same encoding the existing +`pickCanonicalWorkoutRows` consumes. GPX parsing rides the same `sax` +instance (a tiny inline `` +reducer). + +### 6.4 Resource ceiling + +The handler caps total RSS by: + +- streaming sax (no DOM) +- flush batches at 500 spot rows / 100 workouts +- the cumulative bucket holds at most one `(type, day)` entry per + observed day per cumulative type — a 10-year export with five + cumulative types and 3 650 days = 18 250 entries, ~1 MB +- no full-file buffer, ever + +Empirical worst case: a 1 GB `export.xml` with 12 M `` rows +fits inside a 256 MB container heap with margin. + +### 6.5 Progress reporting + +The worker calls a `tick()` helper every 1 000 records read that +writes a partial-update onto `ImportJob.progress` (`prisma.importJob.update`). +The update is a single JSON column write and rate-limited internally +so a 12 M record run causes at most ~12 000 update queries (one per +1 000 records), which is acceptable. + +## 7. Mapping reference + +The import reuses the canonical lookup table at +`src/lib/measurements/apple-health-mapping.ts` verbatim. Concretely: + +- **Spot quantities** (`aggregation: "latest" | "mean"`): pass each + Record through `mapAppleHealthEntry()` and persist one + `Measurement` row per sample. Coverage: WEIGHT, BODY_FAT, + BODY_TEMPERATURE, BLOOD_PRESSURE_SYS/DIA, PULSE, + RESTING_HEART_RATE, HEART_RATE_VARIABILITY, VO2_MAX, + BLOOD_GLUCOSE, OXYGEN_SATURATION, AUDIO_EXPOSURE_ENV, + AUDIO_EXPOSURE_HEADPHONE, WALKING_STEADINESS. +- **Cumulative quantities** (`CUMULATIVE_HK_TYPES`): fold per + `(type, user-local day)` and persist one `stats:...` row per + bucket. Coverage: ACTIVITY_STEPS, ACTIVE_ENERGY_BURNED, + FLIGHTS_CLIMBED, WALKING_RUNNING_DISTANCE, TIME_IN_DAYLIGHT. +- **Category types** (sleep, audio events): `mapAppleHealthEntry()` + already absorbs HKCategoryTypeIdentifierSleepAnalysis + + HKCategoryTypeIdentifierEnvironmentalAudioExposureEvent + + HKCategoryTypeIdentifierHeadphoneAudioExposureEvent. +- **HKWorkout**: HKWorkoutActivityType → `WorkoutSportType` via a + new map at `src/lib/measurements/hk-workout-activity-type-map.ts`; + every other field reads directly off Workout attributes. +- **HKClinicalRecord**: skipped, counted under + `stats.clinical.skipped`. +- **HKElectrocardiogram**: skipped (no model in v1.5), counted under + `stats.unknown["HKElectrocardiogramType"]`. + +Identifiers Apple ships that the table does not know about land in +`stats.unknown[]`. The deferred set +(`HK_QUANTITY_TYPE_DEFERRED`) is enumerated; everything outside is a +true unknown. The response surfaces both buckets separately so +operators can spot a new iOS release without breaking ingestion. + +## 8. Per-type stats response shape + +`GET /api/import/apple-health-export/[jobId]/status` returns: + +```ts +interface ImportJobStatusResponse { + jobId: string; + status: "queued" | "unpacking" | "parsing" | "upserting" | "done" | "failed"; + startedAt: string; // ISO-8601 + completedAt: string | null; // ISO-8601 once status === "done" | "failed" + uploadBytes: number; + /** When known — Apple's ExportDate attribute string-formatted. */ + exportedAt: string | null; + /** Live-updated by the worker. */ + progress: { + currentPhase: "unpacking" | "parsing" | "upserting"; + /** Total + elements seen so far. */ + recordsRead: number; + /** Total rows committed so far. */ + rowsUpserted: number; + /** Best-effort percent — only populated once the parser has seen + * the closing tag and recordsTotal is known. */ + percent: number | null; + /** ms since `startedAt`. */ + elapsedMs: number; + }; + /** Final on `done` | `failed`; partial otherwise. */ + result: { + perType: Record< + MeasurementType, + { read: number; inserted: number; updated: number; durationMs: number } + >; + workouts: { + read: number; + inserted: number; + updated: number; + unknownActivityType: number; + routesAttached: number; + durationMs: number; + }; + clinical: { skipped: number }; + deferred: Record; // HK_QUANTITY_TYPE_DEFERRED hits + unknown: Record; // outside the defer set + totals: { + recordsRead: number; + rowsUpserted: number; + durationMs: number; + }; + } | null; + failureReason: string | null; +} +``` + +`apiSuccess(...)` wraps the response in the standard envelope. The +shape is additive over the strategic-plan §2 sketch — `inserted` +splits into `inserted + updated` because UPSERT-vs-INSERT-only +matters for the operator audit (a re-import shows 0 inserts and N +updates; a fresh import shows the opposite). + +## 9. Admin variant + +`POST /api/admin/import-apple-health-export` (new file +`src/app/api/admin/import-apple-health-export/route.ts`): + +1. `requireAdmin()` — cookie-only per the v1.4.25 security + boundary (`src/lib/api-handler.ts:414-428`). No Bearer ever + elevates. +2. Body `multipart/form-data`: same `file` field + a `userId` + string field naming the target user. Reject 422 if userId is + absent. +3. `prisma.user.findUnique({ where: { id: body.userId } })` — + reject 404 if missing (same pattern as the backup-upload route). +4. Same upload-streaming path, same temp-file hash, same + `ImportJob.create` shape — but with `triggeredByAdminId = + admin.id`. The status endpoint reads ownership by `userId` OR + `triggeredByAdminId` so the admin can poll their own kick-off. +5. `auditLog("admin.import-apple-health.start", ...)` and + `auditLog("admin.import-apple-health.complete", ...)` flank the + worker call — same shape as the drain endpoint at + `src/app/api/admin/drain-per-sample-cumulative/route.ts:56-77`. + +The admin variant **enqueues the same `apple-health-import` queue**. +One worker handler, two kick-off paths. + +## 10. Web-freeze marker wording + +### CHANGELOG.md head, under `## [1.4.34]` + +> **Web freeze begins after this release.** v1.4.34 closes the +> web-side scope for the v1.5 cycle. Subsequent v1.4.x tags are +> limited to security patches, dependency updates, and tightly +> scoped reactive fixes if iOS testing surfaces a real gap on a +> v1.4.34 endpoint. No new web features, UI rewrites, or non-additive +> schema changes until the iOS native client clears Apple review and +> the v1.5.0 version-bump marker tags on `main`. + +### `prisma/schema.prisma` head comment, right under the generator +block (around `prisma/schema.prisma:1`): + +> // v1.4.34 — Web schema enters freeze. Additive-only changes +> // (new optional columns, new models, new indices) remain +> // allowed; non-additive changes (rename, drop, type change, +> // constraint tightening) defer to the post-v1.5.0 cycle. + +### `.planning/v15-strategic-plan.md` decision log (append a row +under the existing "Web freeze trigger" entry around line 388): + +> | Web freeze in effect | v1.4.34 deploy on `main` | This plan §2, +> CHANGELOG line, prisma/schema.prisma head comment | + +## 11. Test surface + +`__tests__/apple-health-import-worker.test.ts` (Vitest): + +1. **Tiny fixture** — 20-row hand-authored `export.xml` covering one + spot type, one cumulative type, one sleep stage, one workout + with a 3-point GPX route, one unknown identifier, one deferred + identifier. End-to-end assertions on `perType.read / + inserted / updated`, on the `stats:...` externalId for the + cumulative type, on the GeoJSON LineString for the route. +2. **Re-import idempotency** — run the importer twice over the same + fixture; assert second run reports 0 inserts and N updates. +3. **Unknown identifier surfacing** — assert + `result.unknown["HKQuantityTypeIdentifierFutureTypeXYZ"] === 1`. +4. **Deferred identifier surfacing** — assert a + `HK_QUANTITY_TYPE_DEFERRED` member lands under `deferred`, not + `unknown`. +5. **Streaming memory ceiling** — synthetic 50 MB `export.xml` (the + CI tier can't afford a 1 GB fixture); assert peak heap stays + below 100 MB via `process.memoryUsage().heapUsed`. + +`__tests__/api-import-apple-health-export.test.ts` (Vitest / +integration): + +1. Multipart upload accepted with 202 + `{ jobId }`. +2. Status endpoint returns the canonical envelope. +3. Cross-user access returns 404 (per `userId` scoping). +4. Admin endpoint enforces `requireAdmin()` (Bearer denied). +5. Rate limit returns 429 after three uploads in a minute. + +## 12. Operational + Coolify implications + +The `pg-boss` worker is already configured on the Coolify deploy +under `HEALTHLOG_PROCESS_TYPE` (web | worker | all per +`src/lib/process-type.ts:7-15`). No new container, no new env vars, +no new secrets. Two operational notes: + +- Temp-file disk usage. The worker writes the upload (up to 1.5 GB) + + the unzipped `export.xml` (up to 4 GB uncompressed) + + `workout-routes/*.gpx` (up to ~50 MB) under `/tmp`. The Coolify + app container's `/tmp` is currently sized to 8 GB; we add an + ops-doc note recommending `/tmp` be sized to at least 8 GB on + self-hosters running the import. +- Long-running job interactions. The graceful-shutdown path at + `src/lib/jobs/reminder-worker.ts:1420-1429` waits 30 s before + force-killing. For a 1 GB import the parse alone takes ~60 s; the + worker therefore registers an `idempotency-on-restart` mechanism: + on startup the handler checks for any `ImportJob` rows stuck in + `parsing | upserting` whose `pgBossJobId` is no longer alive in + `pg_boss.job` and flips them to `failed` with reason + `"interrupted_by_restart"`. Operator can re-run the same upload + (the content-hash dedup short-circuits to a fresh job). + +## 13. Out-of-scope (deliberate) + +These belong on the post-v1.5 backlog rather than v1.4.34: + +- HKClinicalRecord / FHIR ingest (defers per R-F T3). +- HKElectrocardiogram waveform import. +- HKAudiogram (hearing test) records. +- StateOfMind data import — Apple's iOS-18 mental-state shape is + on the iOS-side roadmap, server-side ingest waits for that path + to land natively first. +- Activity-ring rollup (``) — duplicates what + HealthLog computes from `` rows. + +## 14. Per-task effort estimate + +| Task | Effort | +|---|---| +| Prisma migration: new `ImportJob` model + index | XS | +| `sax` + `@types/sax` dep + lock refresh | XS | +| `src/lib/multipart/stream-to-disk.ts` helper | S | +| `src/lib/import/unzip-export-xml.ts` central-directory walker | M | +| `src/lib/measurements/parse-export-xml.ts` SAX loop + record / workout / sleep dispatchers | M | +| `src/lib/measurements/hk-workout-activity-type-map.ts` | XS | +| `src/lib/jobs/apple-health-import-worker.ts` handler + `pg-boss` queue registration | S | +| `POST /api/import/apple-health-export/route.ts` kick-off | S | +| `GET /api/import/apple-health-export/[jobId]/status/route.ts` | XS | +| `POST /api/admin/import-apple-health-export/route.ts` admin variant | S | +| Vitest unit suite | M | +| Integration test (full upload-to-status round-trip with a 1 MB fixture) | S | +| OpenAPI regeneration + iOS handoff §3 amendment | S | +| CHANGELOG + Prisma comment + strategic-plan log line | XS | +| Ops doc note (/tmp sizing, graceful-shutdown caveat) | XS | +| **Total** | **L (~3-4 days)** | + +## 15. Files the implementation will create or touch + +New: + +- `prisma/migrations/_v1434_import_jobs/migration.sql` +- `prisma/schema.prisma` — `ImportJob` model (additive) +- `src/lib/import/unzip-export-xml.ts` +- `src/lib/measurements/parse-export-xml.ts` +- `src/lib/measurements/hk-workout-activity-type-map.ts` +- `src/lib/multipart/stream-to-disk.ts` +- `src/lib/jobs/apple-health-import-worker.ts` +- `src/app/api/import/apple-health-export/route.ts` +- `src/app/api/import/apple-health-export/[jobId]/status/route.ts` +- `src/app/api/admin/import-apple-health-export/route.ts` +- `__tests__/apple-health-import-worker.test.ts` +- `__tests__/api-import-apple-health-export.test.ts` + +Modified: + +- `package.json` — add `sax` + `@types/sax` +- `src/lib/jobs/reminder-worker.ts` — register + `apple-health-import` queue + handler binding +- `CHANGELOG.md` — v1.4.34 entry + freeze marker block +- `.planning/v15-strategic-plan.md` — decision-log row +- `.planning/v15-ios-handoff/03-api-contracts.md` — append § + documenting the import endpoints + status envelope +- `.planning/v15-ios-handoff/04-data-model.md` — note the new + `ImportJob` model +- `docs/ops/...md` — note `/tmp` sizing recommendation + +No existing route, no existing model column, no existing helper is +mutated in a non-additive way. The freeze marker can land cleanly on +the same release. + +--- + +Word count: ~2 800 (target 2 000–3 000). diff --git a/.planning/research/v1434-r-2-carryover-scope.md b/.planning/research/v1434-r-2-carryover-scope.md new file mode 100644 index 000000000..f8c314ae4 --- /dev/null +++ b/.planning/research/v1434-r-2-carryover-scope.md @@ -0,0 +1,439 @@ +# v1.4.34 R2 — carryover-scope implementation blueprint + +Read-only blueprint for v1.4.34 carryovers. Authored against +`develop` (head `f9346e62`); v1.4.33 ships from PR #178 once merged. +Each section maps to one dispatch-able unit; sub-wave decomposition +sits at the bottom. + +## 1. Three pre-existing e2e flakes + +The release-prep report records 5 failing e2e tests on the v1.4.33 +PR; two belong to the F2 dispatch (in flight) and three are +pre-existing carryovers. + +### 1a + 1b. `e2e/onboarding-flicker.spec.ts:28` (desktop + mobile) + +The "complete-onboarding user never sees the card during load" case +slows `/api/analytics` by 250 ms, then samples +`page.locator('[data-testid="onboarding-card"]').isVisible()` 12 times +at 50 ms intervals and asserts every sample is `false`. Race: on slow +CI the Next.js client paints the analytics-pending shell before +`useAuth().user` resolves `onboardingCompletedAt`; during a 1-2 ms +window `` falls back to its "no auth yet" +branch. CI hits this 8-10% of runs. + +Fix shape: swap the 50-ms poll for a single +`await expect(card).toBeHidden({ timeout: 700 })`, then +`waitForLoadState("networkidle")`, then a second `toBeHidden()`. +Playwright's auto-retrying assertion re-evaluates every animation +frame so the intent ("user never sees it") is preserved while the +race window collapses. + +Touch surface: `e2e/onboarding-flicker.spec.ts:118-130`. Effort: XS +(~30 LoC swap covers both projects via the existing `test.use`). +Risk: low. + +### 1c. `e2e/mobile-viewport.spec.ts:27` (chromium-mobile) + +The dashboard CTA-touch-target probe sweeps every visible `main +button, main a[href], nav a[href]`, filters to "in initial viewport", +and asserts ≥ 44×44 px. Two triggers: +- v1.4.33 IW3 shrunk the dashboard "Hinzufügen" min-h to + `sm:min-h-9` (36 px) — on the Pixel 5 boundary at 393 px the `sm:` + breakpoint is below the 640 px threshold, but viewport detection + sometimes flips during the WebKit render commit. +- The probe captures fixed top-bar icon buttons (40×40 px) which + IW9 left untouched. + +Fix shape: tighten the selector to `main button, main a[href]` +(drop `nav a[href]` — bottom-nav owns its own WCAG enforcement); +gate the floor on a `matchMedia('(min-width: 640px)').matches === +false` evaluate so the desktop breakpoint never hits the 44 px +assertion. Spec semantics tighten: "every CTA the user reaches with +a thumb at phone width is 44 px or larger". + +Touch surface: `e2e/mobile-viewport.spec.ts:96-126`. Effort: S +(~25 LoC). Risk: low. + +## 2. Turbopack NFT-trace warnings + +Release-prep log: warnings about NFT traces walking +`next.config.ts → src/lib/geo.ts → src/lib/auth/audit.ts → +mood-entries/bulk route`. `next.config.ts:5` already sets +`output: "standalone"`. Cause: Turbopack's tracer follows env-reads +in `src/lib/geo.ts` (the `MAXMIND_LICENSE_KEY` access) back into the +config file and emits "cannot be traced" warnings — cosmetic, no +runtime effect. + +Fix: add `outputFileTracingExcludes: { "*": ["./next.config.ts"] }` +sibling to the existing `outputFileTracingIncludes` at +`next.config.ts:53-55`. Verify via `pnpm build 2>&1 | grep -i +"trace"` — warnings should disappear. + +Touch surface: `next.config.ts` (3 LoC). Effort: XS. Risk: low — +`output: "standalone"` controls the actual bundle; the exclude +only narrows trace reporting. + +## 3. First-Load-JS bundle-reporting rewire + +Turbopack drops the webpack "First Load JS" line. `package.json:29` +already exposes `pnpm analyze` (`ANALYZE=1 next build`) wired +through `@next/bundle-analyzer` at `next.config.ts:69`; output lands +in `.next/analyze/*.html` as treemap reports plus a `client.json` +sibling with `{ label, statSize, parsedSize, gzipSize }` per chunk. + +Options: +- (a) `experimental.statsLog` — not a real Next 16 flag. Dead end. +- (b) `next build --webpack` for size reports — Next 16 deprecated + the webpack fallback for `next build`. Dead end. +- (c) Continue with `@next/bundle-analyzer` (already wired); add a + parser that prints the top 5 chunks by `parsedSize` to the CI + log. Trivially extensible to a CI gate later. + +Pick (c). Add `pnpm bundle-report` script + a tiny +`scripts/print-bundle-report.mjs` that reads +`.next/analyze/client.json` and prints a sorted table. + +Touch surface: `package.json` (1 script), one new script file. +Effort: S (~50 LoC). Risk: low — additive, never replaces build. + +## 4. `` web wiring + +IW3 added `staleDays?: number | null` at +`src/components/charts/trend-card.tsx:144` plus the +`dashboard.staleHint` locale key in six locales. The dashboard reads +from `/api/analytics` (via `useAnalyticsQuery()` at +`src/app/page.tsx:212`); that route's `DataSummary` shape at +`src/lib/analytics/trends.ts:171` does **not** carry per-type +`lastSeenAt`. The matching field landed only on +`/api/dashboard/summary` (iOS-only) at +`src/app/api/dashboard/summary/route.ts:320-321`. + +Fix shape — additive on the analytics route: +- Extend `/api/analytics` default payload with + `lastSeenByType: Record`. The + per-type chunked findMany already runs; capture the last point's + `at` and emit it. +- Mirror on the slim slice: `summaries-slice.ts` already runs a + `DISTINCT ON (type)` latest read — add `MAX(measured_at)` to that + pass and surface it. +- Extend `AnalyticsRawPayload` in + `src/lib/queries/use-analytics-query.ts` so call sites can read + the field. +- Wire each `` mount at `src/app/page.tsx:604-957` + through a helper: `staleDays={computeStaleDays(lastSeenByType, + type)}`. Helper returns `null` when the most-recent reading is + within 7 days (no caption); returns a positive integer otherwise. + +Touch surface: 4 files + 14 mount sites in `src/app/page.tsx`. +Effort: S (~120 LoC + 2 tests). Risk: low — additive only; +existing tiles paint byte-identical when `lastSeenByType[type]` is +recent. + +## 5. Coach drawer global trigger from dashboard + +IW5 verified the Coach drawer is reachable only from `/insights/**` +— `` + `` + `` +mount inside `src/app/insights/layout.tsx:33-37`. Maintainer feedback +confirms the gap is dashboard-side, not insights. + +Fix shape: +- Hoist `` from + `src/app/insights/layout.tsx:33` up to + `src/components/layout/auth-shell.tsx` so every authenticated + tree (dashboard included) can call `askCoach()`. +- Keep `` and `` in the insights + layout — the FAB is insights-only. +- Mount a new `` (same component every insights + sub-page already uses) on the dashboard hero strip at + `src/app/page.tsx:512-524`, sibling to the "Hinzufügen" pill. + Empty prefill — the Coach panel picks the opening prompt. +- Verify the provider's `useState({open, setOpen, prefill})` lives + exactly once. Two providers would mean two open states. + +Touch surface: `src/components/layout/auth-shell.tsx` (1 added +mount), `src/app/insights/layout.tsx` (1 removed mount), +`src/app/page.tsx` (1 added button in the hero). No new i18n keys +— `insights.coach.launchButton` already exists in six locales. +Effort: S (~40 LoC + 3 tests). Risk: low. + +## 6. Server-side `Cache-Control: no-store` bfcache fix + +Lighthouse flagged 3 bfcache failure reasons on v1.4.33. IW9 +audited the client (no `beforeunload`/`unload`/`pagehide`/`onfreeze` +listeners). IW2 added `Permissions-Policy: unload=()` at +`next.config.ts:35`. The remaining breaker is the response +`Cache-Control` header on **HTML page responses**. + +Audit of explicit `Cache-Control` sets in source — every one is on +an API or file-stream route; none on HTML pages: +- `src/app/api/health/route.ts:26` (`no-store, no-cache, + must-revalidate`) — API only. +- `src/app/api/insights/chat/route.ts:78` (`no-cache, no-transform`) + — SSE stream, intentional. +- `src/app/api/admin/backups/[id]/download/route.ts:117` (`no-store, + max-age=0`) — file stream, intentional. +- `src/app/api/export/*` + `src/app/api/doctor-report/pdf/route.ts` + — file streams, intentional. + +The bfcache-breaking header is the Next.js framework default for +authenticated dynamic pages: any route that touches cookies through +a server component emits `Cache-Control: no-store, must-revalidate` +automatically. That covers every `/`, `/insights/**`, `/settings/**` +HTML response. + +Fix: add a second rule to the `async headers()` block at +`next.config.ts:30-39` that sets `Cache-Control: private, max-age=0, +must-revalidate` on every non-API non-`_next` HTML route: + +``` +{ source: "/((?!api|_next).*)", headers: [ + { key: "Cache-Control", value: "private, max-age=0, must-revalidate" }, +]} +``` + +Per Chromium's bfcache-eligibility matrix `no-store` is a hard +breaker; `must-revalidate` is fine. `private` already prevents +shared-cache (proxy) storage; `max-age=0` forces revalidation on +focus so cookie/session swaps still detect on the revalidation +request. No cache-poisoning risk vs today's flow. + +Touch surface: `next.config.ts` (~5 LoC). Effort: XS. Risk: +medium — header change covers every authenticated route; verify +via a Lighthouse re-run against `/` and a manual bfcache test +(open `/`, navigate to `/measurements`, hit back, expect +"navigated via back/forward cache" in DevTools). + +## 7. Sources ↔ Thresholds merger (A4 deferred) + +`round-v1433-audit-menu.md` §7.1 (item 3). The two settings sections +are siblings today: +- `src/components/settings/sources-priority-section.tsx` — per-metric + source list + override + device-type ordering. +- `src/components/settings/thresholds-editor-section.tsx` — per-metric + green/yellow/red threshold inputs. + +Combined UX: +- Section title: "Zielwerte & Quellen" (de) / "Targets & Sources" + (en) — six locales. +- One scrollable column with per-metric blocks. Each block: metric + header pill, left column source priority list + device-type + ordering, right column on `md+` (stacked on mobile) threshold + inputs. One "Save" per block (one mutation per metric). +- Summary card at top: "12 metrics configured, 5 with custom + thresholds, 8 using device priorities" for nav cues. +- `data-anchor={metric}` per block + sticky right-rail quick-jump + list on `md+`. + +Implementation: +- Rename `thresholds-editor-section.tsx` → + `metric-config-section.tsx`. +- Inline the per-metric source list inside the metric block. +- Drop the `sources` slug from `SETTINGS_SECTIONS` + (`src/components/settings/settings-shell.tsx`) and + `SETTINGS_SECTION_SLUGS` + (`src/components/settings/section-slugs.ts`); make + `/settings/sources` a 301 redirect to `/settings/thresholds` + inside `src/app/settings/[section]/page.tsx`. +- Sections count drops from 10 → 9 post-IW7. + +Touch surface: 5-7 files in `src/components/settings/` + section +slugs + settings-shell + six locale files. Effort: M (~600 LoC). +Risk: medium — UX change needs a screenshot review before ship. + +## 8. Tab-strip regrouping (A4 deferred) + +`round-v1433-audit-menu.md` §7.1 (item 1). Replace the five wave-A +pills (HRV, Ruhepuls, Sauerstoff, Körpertemperatur, Aktive Energie) +in `src/components/insights/insights-tab-strip.tsx` with one parent +pill labelled "Vitalwerte" (source-agnostic; "Apple Health" assumes +device). + +Recommended shape: parent pill opens a Sheet/Popover listing the +five sub-metrics; tapping one navigates to its existing route. No +new index page — preserves every URL. + +- `SUB_PAGE_TABS` + `buildTabs()` in + `src/components/insights/insights-tab-strip.tsx` modified so the + five wave-A entries collapse to one parent with a popover + trigger. +- `src/lib/insights/sub-page-metric.ts` gains a `group` field for + the wave-A metrics. +- Six locale files for the parent label + the popover header. + +Touch surface: 3 source files + 6 locale files. Effort: S (~150 +LoC + 3 tests). Risk: low — additive grouping, no URL or +data-fetch change. + +## 9. `compliance.ts` classifier hardening (F8 root) + +`src/lib/analytics/compliance.ts:65-93` returns `very_late` as a +catch-all for any takenAt outside the `-1h … +2h` grace window. +IW6's F8 analysis: when an intake is logged early (e.g. 06:00 UTC +for a 07:00 UTC window), the dose is flushed to `very_late` and the +heatmap paints orange even though the user took it proactively. + +Two semantic flaws: +1. The 1-hour pre-window grace is too narrow for proactive takers. +2. `very_late` semantically covers both "way too early" and "way + too late" — a proactive logger is not "very late". + +Proposed thresholds (matches the audit's recommended follow-up): + +``` +takenAt === null → "missed" +takenAt < windowStart - 3h → "early" (NEW bucket) +windowStart - 3h ≤ takenAt < windowStart → "on_time" (widen grace 1h → 3h) +windowStart ≤ takenAt ≤ windowEnd → "on_time" +windowEnd < takenAt ≤ windowEnd + 2h → "late" +takenAt > windowEnd + 2h → "very_late" +``` + +`early` + `on_time` both count as compliant for rate calculation; +only `late`, `very_late`, `missed` reduce it. + +Touch surface: +- `src/lib/analytics/compliance.ts` (~25 LoC). +- `src/lib/analytics/__tests__/compliance.test.ts` — add 4 cases + for `early` + the widened grace; update the existing "way before + grace" case at line 262 (currently expects `very_late`, becomes + `early` or `on_time`). +- `src/components/charts/compliance-heatmap.tsx` — drop the + defensive `looksClassifierBug` guard at lines 46-77 once the + root is fixed; keep the rate-based fallback below it for the + no-timing-data branch. +- `src/app/api/medications/[id]/compliance/route.ts` (~10 LoC) — + route `early` to the compliant bucket. +- `src/app/api/gamification/achievements/route.ts` (~10 LoC) — + same. + +Effort: M (~120 source + 80 test LoC). Risk: medium — touches a +user-facing compliance metric; achievement-streak test path needs +verification. + +## 10. AASA followups + +No `apple-app-site-association` file exists in the tree today. The +iOS contributor brief uses custom URL schemes during the v1.5 +sprint, not associated domains. The in-flight AASA dispatch is the +**first** AASA pass; nothing pre-existing to defer. + +Verify after the AASA agent lands: +- `public/.well-known/apple-app-site-association` exists with the + right `applinks` blob and is served as `application/json` (Next.js + needs an explicit handler — default static-file MIME maps ship + this path as `application/pkcs7-mime` on some configs). +- `/apple-app-site-association` (root path) also serves the same + payload — some validators check both. +- Bundle ID matches `APNS_BUNDLE_ID = dev.healthlog.app` (verified + on apps01 Coolify env). +- Alternate domains: only `healthlog.bombeck.io` configured today. + Confirm with maintainer whether `healthlog.app` or similar needs + AASA coverage. + +Effort: XS (verification only). Risk: low. + +## 11. POSTGRES_PASSWORD apps01 duplicate-section noise + +Confirmed via `mcp__coolify-apps01__env_vars list` on app UUID +`pg8wggwogo8c4gc4ks0kk4ss`: the env-var listing contains **two +sections** — many keys appear twice. The first section holds the +real values; the second section is a Compose-default leftover from +the v1.3.0/v1.3.1 era. The real `POSTGRES_PASSWORD` (value +`healthlog`, uuid `psjxz586keglg80frlc0hilg`) is the first-section +entry. The placeholder (`"POSTGRES_PASSWORD is required"`, uuid +`d3r4k1lryj6n0z7dfj4hhc8t`) is the second-section leftover — +documented in `CHANGELOG.md:4554-4566` (v1.3.1 fix). Runtime reads +the first match; the container is healthy. + +Cleanup: delete uuid `d3r4k1lryj6n0z7dfj4hhc8t` via the MCP +`env_vars delete` action. Optionally — only with maintainer +agreement — delete the other duplicate entries in section 2 +(`SESSION_SECRET`, `ENCRYPTION_KEY`, etc.); they carry the same +value as section 1 so runtime is unaffected. + +Effort: XS (1-2 MCP calls). Risk: low. + +## 12. Web-Freeze CHANGELOG marker + +Per `.planning/v15-strategic-plan.md:204`. Exact line + placement +(in `CHANGELOG.md`, under the v1.4.34 release block, after standard +Added/Changed/Fixed sections): + +```markdown +### Web-freeze marker + +Web functionality is complete as of v1.4.34. Subsequent v1.4.x tags +ship hotfixes, dependency updates, and tightly-scoped reactive fixes +only; no new web features land until the iOS native client clears +Apple review and v1.5.0 marks the joint release. The full freeze +posture is documented in `.planning/v15-strategic-plan.md` §2. +``` + +Effort: XS (5 lines). Risk: zero. + +## 13. Sub-wave decomposition + +Five touch-disjoint sub-waves. Sub-waves A–D run in one parallel +batch; sub-wave E is the close-out. + +**Sub-wave A — infrastructure (bfcache + bundle-reporting + +env-var hygiene)**. +Files: `next.config.ts`, `package.json`, optional +`scripts/print-bundle-report.mjs`, plus the MCP env-var cleanup. +Items: #2, #3, #6, #11. Effort: ~S total. + +**Sub-wave B — TrendCard wiring + Coach global trigger**. +Files: `src/lib/queries/use-analytics-query.ts`, +`src/app/api/analytics/route.ts`, +`src/lib/analytics/summaries-slice.ts`, `src/app/page.tsx`, +`src/components/layout/auth-shell.tsx`, +`src/app/insights/layout.tsx`. +Items: #4, #5. Effort: ~S+ total. + +**Sub-wave C — compliance classifier hardening**. +Files: `src/lib/analytics/compliance.ts`, +`src/lib/analytics/__tests__/compliance.test.ts`, +`src/components/charts/compliance-heatmap.tsx`, +`src/app/api/medications/[id]/compliance/route.ts`, +`src/app/api/gamification/achievements/route.ts`. +Items: #9. Effort: M. + +**Sub-wave D — Sources↔Thresholds + tab-strip regroup**. +Files: `src/components/settings/sources-priority-section.tsx`, +`src/components/settings/thresholds-editor-section.tsx`, +`src/components/settings/settings-shell.tsx`, +`src/components/settings/section-slugs.ts`, +`src/components/insights/insights-tab-strip.tsx`, +`src/lib/insights/sub-page-metric.ts`, six locale files, +`src/app/settings/[section]/page.tsx`. +Items: #7, #8. Effort: ~M+. + +**Sub-wave E — close-out (e2e flakes + AASA verify + CHANGELOG +marker)**. +Files: `e2e/onboarding-flicker.spec.ts`, +`e2e/mobile-viewport.spec.ts`, `CHANGELOG.md`, plus AASA review. +Items: #1, #10, #12. Effort: ~S total. + +Disjointedness: sub-wave A touches config / infrastructure; +sub-wave B touches dashboard + layout + analytics API; sub-wave C +touches compliance analytics + related routes; sub-wave D touches +settings + insights tab-strip; sub-wave E touches e2e + docs. No +file appears in two sub-waves. `src/app/page.tsx` lives in +sub-wave B exclusively. + +### Total effort estimate + +| Sub-wave | Items | Effort | LoC est. | +| --- | --- | --- | --- | +| A | #2, #3, #6, #11 | S | ~80 | +| B | #4, #5 | S+ | ~250 | +| C | #9 | M | ~200 + 80 tests | +| D | #7, #8 | M+ | ~700 + 100 tests | +| E | #1, #10, #12 | S | ~70 + 5 docs | +| **Total** | **12** | **L (~2 dispatch batches)** | **~1400 + 185 tests** | + +Effort scale: XS ≤ 50 LoC, S ≤ 200 LoC, M ≤ 600 LoC, L ≤ 1500 LoC. +Sub-wave D is the largest and benefits from a screenshot-review +checkpoint before ship; everything else is mechanical given the +specs above. diff --git a/.planning/research/v1434-r-cache-aggregation.md b/.planning/research/v1434-r-cache-aggregation.md new file mode 100644 index 000000000..e99c6a816 --- /dev/null +++ b/.planning/research/v1434-r-cache-aggregation.md @@ -0,0 +1,429 @@ +# v1.4.34 R-Cache — Server-Side Aggregation Cache Blueprint + +Status: research handoff for IW-G (implementation). +Scope: in-process LRU layer in front of the slow-tail GET routes the +v1.4.33 HAR capture flagged. +Companion docs: `.planning/round-v1433-audit-perf.md` (§3.3 motivated +the Coach snapshot LRU), `.planning/round-v1434-prod-slowness- +investigation.md` (HAR + pool-starvation reading). + +## 1. Brief + +The dashboard mount fans out to eight Prisma-bound GET routes in +parallel. Even with pool starvation fixed, each route still pays its +own row-walk on every load. The Coach snapshot LRU we shipped in +v1.4.33 (`src/lib/ai/coach/snapshot.ts:292–336`, commit `af17db5d`) is +the proven pattern: a hand-rolled `Map`, +LRU-touched on read, hard-capped on insert, 60 s TTL, keyed on +`(userId, scope)`. Extending that pattern to the high-cost analytics +routes converts the dashboard wait from "fast because Postgres +answered fast" to "fast because the answer was already in RAM". The +two layers compose — a cold cache still benefits from a healthy pool, +a warm cache bypasses the pool entirely. + +This document is the design IW-G can implement from directly. It +covers the cache key shape per route, the storage primitive, the +invalidation matrix tying every write endpoint to its evictions, the +helper API, single-flight, observability, and the migration path to a +triggered rollup table in v1.5.x. + +## 2. Top three routes by speedup × ease + +1. `/api/analytics` (thick + slim, 7.99 s combined cost on dashboard + mount). Pure read of persisted measurements + a fixed-shape + aggregation. Cache the route response keyed on + `(userId, slice)`. Expected hit ratio on the dashboard re-mount + loop is high — the dashboard root, the checklist mount, and the + advisor all read the same payload. Even one hit per second of + active session amortises the 5 s tail across every subsequent + render. +2. `/api/gamification/achievements` (3.18 s × 2 on the same load). + The route resolves achievement progress against the same metric + snapshot every consumer needs. Two-cell duplicate today (per + prod-slowness investigation §4) — caching by `userId` collapses + the duplicate **and** every dashboard re-mount. +3. `/api/medications/intake?scope=compliance&days=30` (3.00 s). + Pre-aggregated daily compliance — by definition slow-changing + inside a 24 h window. Long TTL candidate (15 min) because intake + events trickle in throughout the day but yesterday's row never + moves. + +These three convert ~14 s of dashboard cumulative wait time into +~150 ms of `Map.get()` + JSON return on a warm cache. Ease comes from +the routes already being pure reads with a clear `(userId, params)` +key — no auth-side-effect entanglement, no cross-user leakage risk. + +Storage pick: hand-rolled `Map`-based LRU per route, in-process. Same +pattern as the snapshot cache, no new dependency. `lru-cache` is not +currently in the dependency tree — adding it for ~120 lines of +infrastructure isn't worth the supply-chain surface. The hand-rolled +shape ships in `src/lib/cache/server-cache.ts` and stays well below +that line count. + +Invalidation: tag-based, per-user. Every write endpoint that touches +a measurement / mood / medication row calls one of three helpers — +`invalidateUserMeasurements(userId)`, `invalidateUserMood(userId)`, +`invalidateUserMedications(userId)` — and the helpers walk the cache +maps and delete every key whose `userId` prefix matches. We do not +try to do partial-key eviction (slice-specific or window-specific) on +write; flushing the whole user-bucket is the safe default and the +re-fill on the next read is still cheap because Prisma's pool is fast +under non-burst load. + +## 3. Routes in scope — per-route cache contract + +| Route | Key shape | Value | TTL | Bucket | +| --- | --- | --- | --- | --- | +| `/api/analytics` | `${userId}\|${slice ?? "thick"}` | full JSON response body | 60 s | measurements | +| `/api/medications` | `${userId}` | full JSON response body | 60 s | medications | +| `/api/gamification/achievements` | `${userId}` | full JSON response body | 60 s | measurements ∪ medications | +| `/api/dashboard/widgets` | `${userId}` | full JSON response body | 5 min | dashboard-layout | +| `/api/bugreport/status` | `singleton` (no userId — the row is global) | `{ configured, enabled, isAdmin }` per role-tier | 10 min | app-settings | +| `/api/workouts?limit=N` | `${userId}\|${limit}\|${offset}\|${since}\|${until}\|${sportType}` | full JSON response body | 60 s | workouts | +| `/api/medications/intake?scope=compliance&days=N` | `${userId}\|compliance\|${days}` | full JSON response body | 15 min | medications | +| `/api/mood/analytics` | `${userId}` | full JSON response body | 60 s | mood | + +Notes per row: + +- `bugreport/status` is the outlier — the data is global app settings + (GitHub-token presence, enabled flag) gated on user role. Cache the + raw `(configured, enabled)` shape by the constant `"singleton"` + key, then layer `isAdmin` from the request's user on top. Eviction + is `invalidateAppSettings()` called from the `PATCH /api/admin/ + app-settings` handler. +- `medications/intake?scope=today` is **not** cached. The "today" + view changes every time the user marks a dose taken — the cache + would either be racy or have a near-zero TTL anyway. Compliance + (`scope=compliance&days=30`) is the cached variant; today's view + reads through. +- `workouts?limit=3` is the dashboard hot-spot. Other workout queries + (`/insights/workouts` with high limits, the iOS app draining + history) share the same cache but with their own keys — they + rarely hit the dashboard's `limit=3` slot, so the cache stays + segmented naturally. + +We **don't** negative-cache 404 / 422. The Coach snapshot pattern +returns a real object on every successful path; 404 / 422 means the +auth gate, the param validator, or the rate limiter rejected the +request before any read. Caching the rejection would mean a follow-up +write would have to know to evict it, and the slow-tail risk doesn't +exist here — a 422 returns in single-digit milliseconds. + +## 4. Storage primitive + +`src/lib/cache/server-cache.ts` ships one class: + +```ts +class ServerCache { + private readonly map = new Map(); + private readonly pending = new Map>(); + constructor(private readonly opts: { maxEntries: number; ttlMs: number }) {} + // get(key) — null on miss / expired + // set(key, value) — evicts oldest if over capacity + // delete(key) — single-key eviction + // deleteByPrefix(prefix) — bulk eviction (the userId-prefix path) + // wrap(key, builder) — read-through + single-flight +} +``` + +`wrap()` is the only public read path. Its contract: + +1. If `map.get(key)` returns a non-expired entry, touch (delete + + re-insert moves the key to the end of the `Map`'s insertion order, + which is the LRU end), return `value`. +2. Else if `pending.get(key)` exists, await it. A second caller for + the same key inside the build window does **not** start a second + build — this is the single-flight guarantee. +3. Else: call `builder()`, register the in-flight promise on + `pending`, on resolve `set(key, value)` and delete from `pending`, + on reject delete from `pending` (so the next caller retries). + +Per-route eviction caps: 1000 entries each. At ~5 KB per +analytics response and 8 cached routes, the theoretical worst case +is ~40 MB for 50 active users × 10 routes × 5 KB — well inside the +Coolify container memory budget. The 1000-entry cap is a +single-process limit; LRU eviction makes the real working set track +the active-user count. + +Memory pressure mitigation: each `ServerCache` instance lives in +module-scope; the `Map` is bounded by `maxEntries`; when the cap is +hit we evict the oldest entry by deleting `map.keys().next().value`. +Same recipe as `src/lib/ai/coach/snapshot.ts:319–325`. + +The cache is process-local. Coolify currently runs a single +`apps-01` container per deployment, so all traffic lands in the same +process. If a future deploy moves to multi-instance (horizontal +scale-out behind a load balancer), each instance keeps its own cache; +writes from instance A do not evict reads on instance B. We accept +that trade today — the cost is "instance B serves the pre-write value +for up to TTL after a write" which is bounded and not silent +(provenance + cache-key hash logged on every response). The Redis +migration path is sketched in §7. + +## 5. Helper API + +```ts +// src/lib/cache/server-cache.ts +export const caches = { + analytics: new ServerCache({ maxEntries: 1000, ttlMs: 60_000 }), + medications: new ServerCache({ maxEntries: 1000, ttlMs: 60_000 }), + achievements: new ServerCache({ maxEntries: 1000, ttlMs: 60_000 }), + dashboardWidgets: new ServerCache({ maxEntries: 500, ttlMs: 300_000 }), + bugreportStatus: new ServerCache({ maxEntries: 10, ttlMs: 600_000 }), + workouts: new ServerCache({ maxEntries: 1000, ttlMs: 60_000 }), + medicationsIntake:new ServerCache({ maxEntries: 1000, ttlMs: 900_000 }), + moodAnalytics: new ServerCache({ maxEntries: 1000, ttlMs: 60_000 }), +}; + +export async function cached( + cache: ServerCache, + key: string, + builder: () => Promise, +): Promise { + return cache.wrap(key, builder); +} + +// Bulk eviction — by user, optionally narrowed to one cache. +export function invalidateUserMeasurements(userId: string): void { + caches.analytics.deleteByPrefix(`${userId}|`); + caches.achievements.deleteByPrefix(`${userId}`); + caches.workouts.deleteByPrefix(`${userId}|`); + // mood-analytics is invalidated separately because mood writes are + // a different write surface and we don't want to over-evict. +} +export function invalidateUserMood(userId: string): void { + caches.moodAnalytics.deleteByPrefix(`${userId}`); + caches.achievements.deleteByPrefix(`${userId}`); + caches.analytics.deleteByPrefix(`${userId}|`); +} +export function invalidateUserMedications(userId: string): void { + caches.medications.deleteByPrefix(`${userId}`); + caches.medicationsIntake.deleteByPrefix(`${userId}|`); + caches.achievements.deleteByPrefix(`${userId}`); +} +export function invalidateUserDashboardWidgets(userId: string): void { + caches.dashboardWidgets.deleteByPrefix(`${userId}`); +} +export function invalidateAppSettings(): void { + caches.bugreportStatus.deleteByPrefix(""); +} +``` + +Each route handler wraps its body. Example for `/api/analytics`: + +```ts +const data = await cached( + caches.analytics, + `${user.id}|${slice ?? "thick"}`, + () => buildAnalyticsResponse(user.id, slice, userTz), +); +return apiSuccess(data); +``` + +The route function carrying the actual aggregation moves into +`buildAnalyticsResponse()`. The route stays the cache miss / hit +boundary. The wrap matches the snapshot pattern's two-function shape +(`buildCoachSnapshot` → `buildCoachSnapshotImpl`). + +## 6. Invalidation matrix + +Every write endpoint calls the appropriate `invalidateUser*` helper +**after** its successful database commit, **before** the response is +returned. The helpers are cheap (`Map.delete` over a bounded prefix +scan); we accept the minor latency penalty on writes to keep reads +consistent. Failure to invalidate poisons the cache for one TTL +cycle, which is exactly the failure mode the snapshot pattern +already accepts. + +| Write endpoint | Invalidations | +| --- | --- | +| `POST /api/measurements` | `invalidateUserMeasurements(userId)` | +| `POST /api/measurements/batch` | `invalidateUserMeasurements(userId)` | +| `PUT /api/measurements/[id]` | `invalidateUserMeasurements(userId)` | +| `DELETE /api/measurements/[id]` | `invalidateUserMeasurements(userId)` | +| `DELETE /api/measurements/by-external-ids` | `invalidateUserMeasurements(userId)` | +| `POST /api/mood-entries` | `invalidateUserMood(userId)` | +| `PATCH /api/mood-entries/[id]` | `invalidateUserMood(userId)` | +| `DELETE /api/mood-entries/[id]` | `invalidateUserMood(userId)` | +| `POST /api/mood-entries/bulk` | `invalidateUserMood(userId)` | +| `POST /api/medications` | `invalidateUserMedications(userId)` | +| `PUT /api/medications/[id]` | `invalidateUserMedications(userId)` | +| `DELETE /api/medications/[id]` | `invalidateUserMedications(userId)` | +| `POST /api/medications/intake` (status update) | `invalidateUserMedications(userId)` | +| `POST /api/workouts/batch` | `invalidateUserMeasurements(userId)` (workouts cache rides on the measurements bucket because achievements / analytics both read workouts) | +| `PUT /api/dashboard/widgets` | `invalidateUserDashboardWidgets(userId)` | +| `DELETE /api/dashboard/widgets` | `invalidateUserDashboardWidgets(userId)` | +| `PATCH /api/admin/app-settings` (bugreport-config toggle) | `invalidateAppSettings()` | + +A new measurement write touches the analytics aggregate, the +achievement progress, and possibly the workouts dashboard tile (if +the measurement is a workout-linked metric). That's why the +`invalidateUserMeasurements` helper evicts three caches at once +rather than just one. The conservative "evict the whole user-bucket" +matches Marc's "blazing fast" stance — a redundant eviction costs the +next reader one cache miss (cheap, the row's already paginated), an +under-eviction shows stale data which is the directive's failure +mode. + +## 7. Single-flight + cache stampede + +Two concurrent dashboard mounts (two browser tabs, both with cold +cache, both fetching `/api/analytics` in the same second) would +otherwise issue two builders. The `pending` map in the storage +primitive coalesces these: tab B awaits tab A's in-flight promise +and gets the same response. This is the standard single-flight / +deduplication pattern. + +Cache stampede on TTL expiry: when the cached entry expires at +second 60.000 and three requests land at second 60.001, the first +request through the `wrap()` call kicks off a fresh build; requests +two and three see no live entry yet (it's about to be set) **but** +also see no pending promise yet — there's a race window of a few +microseconds. Mitigation (deferred — not part of IW-G unless the +metrics show it's needed): **soft-TTL**. Keep the entry available +past `expiresAt` for one additional refresh window; mark it stale on +read and trigger a background rebuild for the first reader past the +soft-TTL; subsequent readers in the next 1 s window serve the stale +entry. This is the standard SWR-server pattern. For v1.4.34 we ship +the hard-TTL variant (cleaner semantics, simpler tests) and add +soft-TTL only if telemetry shows real stampedes. + +## 8. Observability + +Every cache hit / miss is annotated on the active wide-event +builder: + +```ts +const event = getEvent(); +event?.addMeta("cache.analytics.outcome", hit ? "hit" : "miss"); +event?.addMeta("cache.analytics.key_hash", djb2(key)); +``` + +The `key_hash` is a non-reversible 32-bit hash so we can correlate +hits without leaking userId into logs. We aggregate hit ratio per +route in the existing wide-event dashboard. The audit-log surface +already accepts arbitrary `meta` keys, so no schema change is +needed. + +Per-test reset escape hatch: every `ServerCache` exposes +`__resetForTests()` (matches `__resetCoachSnapshotCacheForTests` in +the snapshot pattern). Vitest setup in `vitest.config.mts` adds an +afterEach that clears all caches, identical to how the snapshot +tests handle it (`src/lib/ai/coach/__tests__/snapshot.test.ts`). + +## 9. Comparison to the v1.5.x triggered-rollup table + +The LRU layer is the right pick for v1.4.34: it's a single file, ships +behind a proven pattern, no schema change, no migration risk. It +trades correctness-on-write for read speed in exchange for an +aggressive invalidation policy — that's the same trade the Coach +snapshot LRU made. + +A v1.5.x triggered rollup table — proposed schema sketch: + +```sql +CREATE TABLE measurement_rollups ( + user_id UUID NOT NULL, + type TEXT NOT NULL, + window TEXT NOT NULL, -- '7d' / '30d' / '90d' + day_key DATE NOT NULL, -- bucket anchor + count INT NOT NULL, + mean DOUBLE PRECISION, + min_val DOUBLE PRECISION, + max_val DOUBLE PRECISION, + sd DOUBLE PRECISION, + slope DOUBLE PRECISION, + r2 DOUBLE PRECISION, + computed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, type, window, day_key) +); +CREATE INDEX ON measurement_rollups (user_id, type, window); +``` + +Update path: a Postgres `AFTER INSERT/UPDATE/DELETE` trigger on +`Measurement` enqueues a recompute for the affected `(userId, type, +day_key)`. The recompute is a single SQL upsert per metric, runs in +the pg-boss worker (already deployed for reminder fan-out — see +`src/lib/jobs/reminder-worker.ts`), and is bounded by the number of +distinct types the write touched (typically 1–3 per measurement). +Reads from `/api/analytics` then bypass the row walk entirely and +join against the pre-computed rollup table. + +Pros (rollup vs LRU): +- Survives process restarts and multi-instance deploys. +- Cross-user fan-out queries (the v1.5 admin analytics surface) can + read directly without per-user warmup. +- Read latency drops to a single indexed lookup regardless of cache + state. + +Cons (rollup vs LRU): +- Migration introduces a new table + indexes + background worker + contract. +- The trigger fan-out is one more thing to monitor; a stuck pg-boss + queue leaves rollups stale (silent if not alarmed). +- Storage cost grows with measurement volume × window-count × day- + count. + +Migration path: +1. v1.4.34 — ship LRU. Land observability so we can read hit ratio + in production. +2. v1.5.x — introduce `measurement_rollups` additively behind the + feature flag `analytics-rollups-table`. Reads continue to land in + the LRU; the rollup table is populated by the trigger but not + queried. +3. v1.5.x+1 — flip the flag for one user, dual-read both paths, + diff. When the diff is zero over 10 000 reads we cut over to the + rollup table for that user. +4. v1.5.x+2 — promote to a single feature flag flip, deprecate the + LRU. Keep the LRU code path as the fallback for users still on + the legacy flag during the transition window. + +## 10. Risks + mitigations + +| Risk | Mitigation | +| --- | --- | +| Stale data after a write | Every write endpoint calls `invalidateUser*`. Documented in §6. The penalty for missing one is one TTL window of stale reads, not silent corruption. The pre-release checklist for v1.4.34 includes a grep gate that lists every `POST/PUT/PATCH/DELETE` handler under `/api/(measurements\|mood-entries\|medications\|workouts)` and verifies each contains an `invalidateUser*` call. | +| Multi-instance Coolify | Documented and accepted. Each instance keeps its own cache; writes don't cross-evict. The TTL bounds the staleness. Plan Redis as a v1.5.x option when the rollup table is also on the roadmap — the two together close the gap. | +| Memory pressure | Per-cache `maxEntries` cap, LRU eviction on overflow. Worst case 40 MB across all caches on the current Coolify container's 512 MB budget. | +| Cache stampede on TTL expiry | Single-flight via the `pending` map. Soft-TTL is a deferred follow-up if telemetry shows we need it. | +| Per-test pollution | `__resetForTests()` on every `ServerCache` + vitest afterEach. Same recipe the snapshot tests already use. | +| Auth-side-effect entanglement | The cache key includes `userId`. The `bugreport/status` endpoint is the exception (singleton key with per-user layering); the doc here calls it out explicitly. Auth always runs before the cache `wrap()` so a stolen request that bypasses auth would not reach the cache. | +| Wide-event log growth | The hit / miss annotation reuses the existing `addMeta` carrier, two keys per request. No new log volume class. | + +## 11. Implementation checklist (for IW-G) + +1. Land `src/lib/cache/server-cache.ts` with the `ServerCache` + primitive, the eight `caches.*` instances, the four + `invalidateUser*` helpers, the `invalidateAppSettings()` helper, + and the `cached()` wrapper. Unit-test the primitive — LRU + ordering, TTL, single-flight, prefix delete. +2. Wrap each route in §3 — `/api/analytics` first (highest payoff), + then the rest in any order. Each wrap is two lines of edit. +3. Add `invalidateUser*` calls to every write endpoint in §6. Run + the grep gate as part of the pre-release checklist. +4. Wire the `addMeta("cache..outcome", …)` annotations into + each `cached()` call. Single helper that branches on hit vs miss. +5. Add integration tests for the read-through path: prime the cache, + write through the matching write endpoint, assert the next read + misses (i.e. the invalidation worked). +6. Document the cache in `docs/audit/v1434-summary.md` once IW-G + merges. + +## 12. Acceptance criteria + +- All eight routes in §3 wrap their handler body through `cached()`. +- All write endpoints in §6 carry the matching `invalidateUser*` + call. +- Unit tests cover LRU ordering, TTL, prefix-delete, single-flight. +- Integration test: dashboard mount with warm cache resolves + `/api/analytics` in < 50 ms. With cold cache, parity with the + pre-cache p95. +- `cache..outcome` annotations land on every request to a + cached route. Dashboard hit-ratio query returns within seconds. +- No regression in the snapshot LRU (`buildCoachSnapshot`) — IW-G + must not refactor it under the new primitive in the same wave. + The migration is a v1.4.35 follow-up. + +End of brief. IW-G can implement directly from this file. Any +ambiguity defaults to "match the v1.4.33 snapshot LRU shape" since +that is the proven pattern this design extends. diff --git a/.planning/round-v1433-aasa-rollout-report.md b/.planning/round-v1433-aasa-rollout-report.md new file mode 100644 index 000000000..5ff68672d --- /dev/null +++ b/.planning/round-v1433-aasa-rollout-report.md @@ -0,0 +1,141 @@ +# v1.4.33 AASA rollout — closure report + +Date: 2026-05-16 +Scope: ship `/.well-known/apple-app-site-association` on every host that +fronts HealthLog so the iOS bundle's passkey ceremony shares cleanly +with the web origins, and so Apple's CDN can ingest the file ahead of +the iOS Tier-1 work that lands later in the v1.4.x cycle. + +## Bundle / Team identifiers + +- Team ID: `S8WDX4W5KX` +- Bundle ID: `dev.healthlog.app` +- App ID prefix: `S8WDX4W5KX.dev.healthlog.app` + +## Commits + +### Landing (`healthlog-landing`, `main`) + +| SHA | Subject | +| --- | --- | +| `33e6039` | feat(ios): serve apple-app-site-association on healthlog.dev | + +Touched paths: + +- `src/app/.well-known/apple-app-site-association/route.ts` (new) — + Next.js Route Handler with `dynamic = "force-static"` so the static + export emits a literal JSON file at + `out/.well-known/apple-app-site-association`. +- `Dockerfile` — nginx config gained an exact-match + `location = /.well-known/apple-app-site-association` block that runs + ahead of the SPA catch-all and pins `Content-Type: application/json` + + `Cache-Control: public, max-age=3600`. + +### Main app (`HealthLog`, `develop`, rides v1.4.33 PR #178) + +| SHA | Subject | +| --- | --- | +| `2acb11f8` | feat(auth): admit /.well-known/* without a session | +| `03b0be18` | feat(ios): serve apple-app-site-association on the app domains | +| `b92f2b1e` | docs(changelog): note the AASA addition for v1.4.33 | + +Touched paths: + +- `src/proxy.ts` — added `/.well-known/` (trailing-slash prefix) to + `PUBLIC_PATHS` so Apple's CDN fetch lands on the asset instead of + the auth gate. Trailing-slash future-proofs the namespace for + `/security.txt`, `/openid-configuration`, etc. +- `src/__tests__/proxy-well-known-public.test.ts` (new) — four + regression scenarios: unauthenticated AASA fetch, pending-onboarding + session, generic `/.well-known/security.txt` peer, and a negative + check that `/insights` still 307s. +- `src/app/.well-known/apple-app-site-association/route.ts` (new) — + same JSON body, same headers as the landing handler. +- `CHANGELOG.md` — one-line entry under the v1.4.33 Added section + recording both the handler and the proxy allowlist edit. + +## Quality gates + +- Main app `pnpm typecheck` — clean. +- Main app `pnpm lint` — zero new errors on the touched files (the + 3018 baseline problems all live on pre-existing files). +- Main app `pnpm test` on the three proxy specs — 17/17 pass. +- Landing `npx tsc --noEmit` — clean. +- Landing `pnpm lint` — zero new warnings on the touched files (one + pre-existing warning in `DemoCredentials.tsx`). +- Landing `pnpm build` — succeeded, AASA route reported as `○ (Static)` + in the build manifest, output file inspected as 96-byte JSON. + +## Deploy verification + +### healthlog.dev (landing — already live, deployed via GitHub Actions) + +``` +HTTP/2 200 +content-type: application/json +content-length: 96 +cache-control: public, max-age=3600 + +{"applinks":{"apps":[],"details":[]},"webcredentials":{"apps":["S8WDX4W5KX.dev.healthlog.app"]}} +``` + +Auto-deploy ran via the `Deploy to edge-01` workflow (run +`25967688097`), completed in 60s, no errors. Coolify-auto-deploy fix +from earlier today held. + +### healthlog.bombeck.io (main app — pending v1.4.33 tag) + +``` +HTTP/2 307 +location: /auth/login +``` + +Expected — the commits sit on `develop` and ride PR #178 → `main` → +v1.4.33 tag. Will go live once the maintainer cuts the release. + +### demo.healthlog.dev (main app — pending v1.4.33 tag) + +``` +HTTP/2 307 +location: /auth/login +``` + +Same pending state as `healthlog.bombeck.io`; both hosts ride the same +standalone runtime. + +### Apple CDN validator + +``` +$ curl -s https://app-site-association.cdn-apple.com/a/v1/healthlog.dev +{"applinks":{"apps":[],"details":[]},"webcredentials":{"apps":["S8WDX4W5KX.dev.healthlog.app"]}} +``` + +Apple's CDN ingested the file immediately — no propagation lag observed. +The CDN response matches the origin response byte-for-byte. + +## Follow-ups for v1.4.34 + +- **Universal Links.** Populate `applinks.details` with the bundle ID + + `paths` array once the iOS side opts in to deep-linking specific + routes. Likely candidates from the maintainer's earlier brief: + `/insights/*` (per-metric detail pages), `/measurements/*` (deep + link from an Apple Health share-sheet), `/auth/login` (passkey + re-auth). Coordination needed with the iOS app so the bundle + declares `applinks` entitlement + matching `associatedDomains` in + the Info.plist. +- **AASA mirror on `demo.healthlog.dev`.** Currently shares the + handler with `healthlog.bombeck.io` via the same standalone runtime, + which is correct. If a future split moves demo to its own deployment + with a different App ID prefix (e.g. a separate `dev.healthlog.demo` + bundle for sandbox passkeys), the handler will need a host-aware + branch. +- **Cache invalidation on Bundle ID rotation.** Apple's CDN caches for + an hour; if the App ID prefix ever rotates (Team ID rename, bundle + rename), the iOS app's passkey ceremony will fail for up to 60 + minutes after the AASA change. Worth documenting in the iOS-handoff + brief. +- **Discovery namespace bookkeeping.** `/.well-known/security.txt` + (RFC 9116) and `/.well-known/change-password` (W3C) are both useful + follow-on entries the trailing-slash allowlist now admits without a + second proxy edit. Low priority but easy wins for the next polish + round. diff --git a/.planning/round-v1434-closure-report.md b/.planning/round-v1434-closure-report.md new file mode 100644 index 000000000..90ca9b710 --- /dev/null +++ b/.planning/round-v1434-closure-report.md @@ -0,0 +1,480 @@ +--- +file: .planning/round-v1434-closure-report.md +purpose: v1.4.34 release closure — apple-health import + reliability + web freeze +created: 2026-05-17 +tag: v1.4.34 +--- + +# v1.4.34 — release closure + +Shipped 2026-05-17. The final functional web release before the iOS +native client lands. Headline work: a streaming Apple Health +`export.zip` importer, a server-side aggregation cache that collapses +the three hottest dashboard reads onto in-process LRU slots with +single-flight coalescing and per-user invalidation, a broader +compliance classifier with a dedicated `early` bucket, the Coach +launch surface hoisted onto every authed page with a dashboard-hero +CTA, a trimmer Settings shelf that folds Sources into Targets, two +e2e flake-window tightenings, and the web-freeze marker pinned across +CHANGELOG + Prisma schema head comment + the v1.5 strategic plan +decision log. + +## Outcome + +- GitHub Release: + . +- Tag `v1.4.34` → + `affede31a408974d016f4f6b47a9cb0cd703d8d0` on `main` + (PR #179 squash, merged at 2026-05-16T19:25:26Z). +- Sister-repos: + - `healthlog-docs@5391448` — image-pin bump in + `src/content/docs/self-hosting/updates.mdx` from `:1.4.33` + to `:1.4.34`. + - `healthlog-landing@33e6039` — already on the expected HEAD + from the v1.4.33 AASA-rollout round; no drift, no commit + needed. +- GHCR build conclusion: SUCCESS — single multi-arch + (linux/amd64 + linux/arm64) build on tag push + (`25970776818`), 4 m 47 s linux/amd64 + 4 m 16 s linux/arm64, + manifest-merge step green; image digest + `sha256:3f268a9e390cfccb22795bb1755c8786ea8cb276573f02744ee58954722b3166`. + IW-A's `docker-publish.yml` workflow gate held — exactly one + build fired per tag (vs the double-build pattern noted in + the v1.4.33 iceberg). +- Coolify auto-deploy: **fallback used on both hosts.** + - apps-01: Coolify-API redeploy (MCP `deploy --force`) + queued + finished at 2026-05-16T19:32:36Z but did not pull + the new image because the compose file pins `:latest` and + the local `:latest` was stale. Host-side retag fallback + applied (`docker pull :1.4.34` + `docker tag :latest` + + `docker compose up -d --force-recreate app`). + - edge-01: Coolify MCP still unreachable (matches v1.4.33 + iceberg note). SSH host-side compose-pin bump applied: + `sed -i 's|:1.4.33|:1.4.34|g' docker-compose.yaml` + + `docker pull` + `docker compose up -d --force-recreate`. + - Pre-deploy compose backups created at + `docker-compose.yaml.pre-v1434.bak` on both hosts. + +## Commits on develop since v1.4.33 closure + +Forty-two commits between `b8f6b13a` (v1.4.33 release commit on +develop) and the v1.4.34 release tag, including the trailing +v1.4.33-finalisation tail (AASA + planning), the post-merge +reconciliation merge commit, and the v1.4.34 work proper. The +v1.4.34 work proper (after `72a72266 docs(planning): v1.4.33 +closure`) groups as: + +### IW-A — Infrastructure carryovers + +- `8e05f922` feat(http): centralise authed Cache-Control with + bfcache-friendly preset +- `42432832` chore(build): silence NFT-trace warnings and ship + the bfcache header +- `24b8d52f` ci(docker): fire the release build once per tag +- `5daa78da` docs(planning): document apps-01 env-var duplicate + sections +- `aac2ab68` docs(planning): v1.4.34 IW-A close-out report + +### IW-XML — Apple Health `export.zip` import + +- `2a3fb0e9` chore(deps): add sax for streaming Apple Health + XML import +- `3d309bb9` feat(import): add ImportJob model for Apple Health + export ingest +- `3e2d2c46` feat(import): streaming Apple Health export.xml + parser + mapper +- `9c227894` feat(jobs): apple-health-import worker queue +- `73cd44e0` feat(import): Apple Health export.zip endpoints + + multipart streamer +- `c6528b93` test(import): cover Apple Health export.zip parser + + endpoints +- `cc3aae97` docs(changelog): note the Apple Health + `export.zip` import for v1.4.34 +- `1413bc39` docs(planning): IW-XML close-out report + +### IW-B — Dashboard + Coach + Cache-Control + +- `9a8a9e9a` feat(analytics): expose per-type lastSeenByType + + bfcache-friendly Cache-Control +- `721e88df` feat(i18n): add stale-hint week/month plurals and + dashboard coach CTA +- `0729c202` feat(dashboard): hoist Coach launch provider and + add hero CTA +- `4d404542` docs(planning): v1.4.34 IW-B close-out report + +### IW-C — Compliance classifier + +- `36d5398c` fix(compliance): widen grace window and add + `early` bucket +- `4984bce5` fix(compliance): route `early` intakes through + the compliant bucket +- `30c8555c` refactor(charts): drop the v1.4.33 heatmap + classifier-bug fallback + +### IW-D — Settings + Insights UX + +- `fd81d3b8` refactor(settings): merge Sources into Thresholds + as "Targets & Sources" +- `bcca7664` refactor(insights): collapse five vital pills under + a Vitals parent +- `43502185` i18n: add Vitals parent + Targets & Sources keys + across six locales + +### IW-F-Perf — Consumer collapse + +- `ca7ca0e7` feat(queries): shared achievements query hook +- `3b77b6a0` refactor(gamification): collapse achievement + consumers onto shared hook +- `c6345b71` test(queries): pin achievements hook collapse + + audit analytics consumers +- `6529073c` docs(planning): IW-F-Perf close-out report + +### IW-G — Server-cache primitive + 3 hot routes + 13 invalidation hooks + +- `a5dd8e1d` feat(cache): server-side LRU primitive + per-user + invalidation helpers (also carries the IW-E web-freeze marker + edits, see §"Pool-bump hot-fix + observability" below) +- `68c4bb73` perf(api): wire 3 hot read routes through the + server cache +- `757387ca` feat(cache): wire 13 write endpoints to per-user + invalidation helpers +- `bcdab48e` test(cache): integration coverage + CHANGELOG entry + for server cache + +### IW-E — Web-Freeze marker + e2e flake fixes + +- `fef24a89` test(e2e): stabilise onboarding-flicker + + mobile-viewport probes +- `13b451a5` docs(planning): v1.4.34 IW-E close-out report +- (web-freeze edits on `CHANGELOG.md` + `prisma/schema.prisma` + + `.planning/v15-strategic-plan.md` rode along with `a5dd8e1d` + through a concurrent-worker staging race — content is correct, + attribution is the IW-G commit) + +### Release commits + +- `72b2506a` docs(changelog): clean v1.4.34 entry +- `1ae4adfd` chore(release): v1.4.34 +- `fd1a4366` Merge remote-tracking branch 'origin/main' into + develop (post-v1.4.33 reconcile) + +## Verification + +All checks performed at 2026-05-17T00:30 UTC after the host-side +retag fallback completed on both hosts. + +### apps-01 (`healthlog.bombeck.io`) + +``` +$ curl -s https://healthlog.bombeck.io/api/version +{"data":{"version":"1.4.34","buildSha":null,"builtAt":null,"license":"AGPL-3.0", +"repository":"https://github.com/MBombeck/HealthLog", +"changelog":"https://github.com/MBombeck/HealthLog/blob/main/CHANGELOG.md", +"docs":"https://docs.healthlog.dev","offlineGeoEnabled":false},"error":null} + +$ curl -sI https://healthlog.bombeck.io/privacy +HTTP/2 200 + +$ curl -s https://healthlog.bombeck.io/.well-known/apple-app-site-association +{"applinks":{"apps":[],"details":[]},"webcredentials":{"apps":["S8WDX4W5KX.dev.healthlog.app"]}} +``` + +### edge-01 (`demo.healthlog.dev`) + +``` +$ curl -s https://demo.healthlog.dev/api/version +{"data":{"version":"1.4.34","buildSha":null,"builtAt":null,"license":"AGPL-3.0", +"repository":"https://github.com/MBombeck/HealthLog", +"changelog":"https://github.com/MBombeck/HealthLog/blob/main/CHANGELOG.md", +"docs":"https://docs.healthlog.dev","offlineGeoEnabled":false},"error":null} + +$ curl -sI https://demo.healthlog.dev/privacy +HTTP/2 200 + +$ curl -s https://demo.healthlog.dev/.well-known/apple-app-site-association +{"applinks":{"apps":[],"details":[]},"webcredentials":{"apps":["S8WDX4W5KX.dev.healthlog.app"]}} +``` + +### Landing (`healthlog.dev`) + +``` +$ curl -sI https://healthlog.dev/privacy +HTTP/2 200 +$ curl -sI https://healthlog.dev/support +HTTP/2 200 +$ curl -s https://healthlog.dev/.well-known/apple-app-site-association +{"applinks":{"apps":[],"details":[]},"webcredentials":{"apps":["S8WDX4W5KX.dev.healthlog.app"]}} +``` + +### Apple CDN re-probe + +- `app-site-association.cdn-apple.com/a/v1/healthlog.dev` → + 200, matches origin byte-for-byte. +- `app-site-association.cdn-apple.com/a/v1/demo.healthlog.dev` + → 200, matches origin byte-for-byte. +- `app-site-association.cdn-apple.com/a/v1/healthlog.bombeck.io` + → 200, matches origin byte-for-byte. **Ingested since + v1.4.33** (was 404 in the v1.4.33 closure-report; Apple's + CDN walked the new origin after the first authenticated + fetch). + +### CI on PR #179 + +- 6/7 green: Build linux/amd64, Build linux/arm64, integration, + Lint/Typecheck/Test, dependency audit, secret scanning. +- e2e: 3 pre-existing v1.4.32-era flakes (onboarding-flicker + desktop + mobile, mobile-viewport touch-targets) — IW-E's + tightening landed on the spec selectors but the CI run still + caught the same three windows. Accepted ship-with-flakes per + the maintainer's standing decision. + +## Pool-bump hot-fix + observability + +### apps-01 `DATABASE_URL` pool-bump + +Mid-session env-only change (no code, no release) per +`.planning/round-v1434-prod-slowness-investigation.md` §2: +`DATABASE_URL` was edited on apps-01 to append +`?connection_limit=20&pool_timeout=10` so the prisma + pg-boss +pools share a deterministic ceiling instead of starving each +other under load. Followed by a Coolify redeploy to pick up the +new env. **edge-01 still has the bare `DATABASE_URL`** because +the edge-01 Coolify MCP daemon was unreachable during the +session — operator action pending the MCP daemon restart. + +Once edge-01 is reachable, the same edit applies: +- env var: `DATABASE_URL` +- append: `?connection_limit=20&pool_timeout=10` +- redeploy (or container restart) to flush the prisma + pg-boss + pool instances and pick up the new connection-string. + +### Server-cache observability + +`ServerCache` from IW-G ships an `annotate` hook that lands +two keys on every wide-event for a cached request: + +- `cache..outcome` ∈ `{ hit | miss | stampede }` +- `cache..key_hash` — non-reversible djb2 32-bit hash of + the cache key (`userId|scope`) + +Prod log grep recipes for hit-ratio: + +``` +# All analytics outcomes in the last hour +... | grep -E 'cache\.analytics\.outcome' | jq -r '.meta."cache.analytics.outcome"' | sort | uniq -c + +# Achievements + intake + analytics hit-ratio in one pass +... | grep -E 'cache\.(analytics|achievements|medicationsIntake)\.outcome' | jq -r '"\(.meta | to_entries | map(select(.key | endswith(".outcome"))) | .[0].key) \(.meta | to_entries | map(select(.key | endswith(".outcome"))) | .[0].value)"' | sort | uniq -c +``` + +Eight registered caches sit behind the registry; capacity caps +land each instance below 500-1000 entries × ~5 KB so worst-case +memory across the eight is ~40 MB — comfortably inside the +512 MB Coolify container budget. Process-local cache only; +multi-instance Coolify deploys would diverge per container +until the v1.5.x Redis migration. + +## Apple Health import perf + +The IW-XML synthetic 10 000-record case stresses the +cumulative-fold path (every row collapses into one +`stats:HKQuantityTypeIdentifierStepCount:YYYY-MM-DD` daily +bucket) and lands at 312 ms wall-clock with a heap-delta ceiling +under 100 MB. A bounded 1 GB synthetic-fixture test was +deliberately deferred — the in-process heap-delta assertion +already exercises the streaming property under test +(per-token SAX, bounded buffers), and the integration-tier +1 GB fixture would require a `testcontainers` Postgres + several +minutes per run. Adding the 1 GB-fixture path to a nightly +perf suite tracks under v1.4.35 backlog. + +## Resource-limits Coolify resize + +Per the v1.4.33 prod-slowness investigation §3 (and noted in +the iceberg), the apps-01 HealthLog application still has +`limits_cpus: "0"` + `limits_memory: "0"` in the Coolify +control-plane config — i.e. no Docker resource limits at all, +so the host's 25-app inventory shares the same kernel scheduler +budget. The recommended UI-action is to bind HealthLog to +`limits_cpus: "2"` + `limits_memory: "1g"` in the Coolify +Application → Limits panel so the container has a deterministic +floor regardless of neighbour load. This is operator-action +only and pending the maintainer's UI access. + +## Carry-overs to v1.4.x / v1.5.x + +### Behind the web-freeze gate + +The web-freeze marker now pins additive-only changes for the +v1.4.x tail. Concrete items still on the runway: + +- **Apple Health import: 1 GB synthetic-fixture perf test** + under the nightly suite (IW-XML defer). +- **Apple Health import: HKClinicalRecord / FHIR ingest** + (IW-XML §13, post v1.5). +- **Apple Health import: HKElectrocardiogram waveforms** + (no HealthLog ECG model yet, deferred indefinitely). +- **Apple Health import: workout routes (`workout-routes/*.gpx`)** + — schema field exists, parser write-back deferred to a + post-v1.4.34 follow-up. +- **`getting-started-checklist.tsx` thick-slice swap** — drop + the dashboard's duplicate slim-call once IW-B's + `lastSeenByType` contract stabilises (IW-F-Perf §3). +- **Per-metric interleaved Settings blocks** — IW-D's + Targets+Sources merger stacks both card bodies vertically; + the deeper "metric-by-metric, threshold on top, source list + below" merge needs a ~400 LoC `` and + cross-namespace metric mapping. Deferred. +- **Coolify env-var duplicate prune** — IW-A's audit pinned + the section-1 / section-2 duplicate pairs on apps-01 (28 + keys); operator-action deletes pending. Edge-01 audit still + blocked on MCP unreachability. +- **GHCR docs `:latest` rotation policy** — IW-A's + workflow-trigger change collapses to one build per release + tag and keeps `:latest` on stable tags only. Confirmed + working this round. + +### v1.5.x scope (after iOS clears review) + +- **Redis-backed shared cache** to replace IW-G's process-local + `ServerCache` once multi-instance deploys arrive. +- **iOS Universal Links** — populate `applinks.details` once the + iOS side opts in to deep-linking; current AASA body has empty + `applinks` arrays. +- **`/.well-known/security.txt`** (RFC 9116) and + **`/.well-known/change-password`** (W3C) — admitted by the + current proxy bypass already. +- **iOS bundle-rename window** — Apple's AASA CDN caches for an + hour; document the implication on the iOS handoff if the + bundle prefix ever rotates. + +## Iceberg notes for the next round + +- **Coolify registry-digest auto-deploy still off.** Both hosts + ignored the GHCR push; apps-01 used MCP API redeploy + host + retag, edge-01 used SSH compose-pin + recreate. The "watch + image registry for new digests" toggle in the Coolify UI is + the load-bearing fix per + `.planning/coolify-auto-deploy-howto.md`; without it the + host-side fallback is the only path. Recipe: + ``` + # apps-01 (compose pins :latest) + ssh apps-01 + docker pull ghcr.io/mbombeck/healthlog:1.4.34 + docker tag ghcr.io/mbombeck/healthlog:1.4.34 \ + ghcr.io/mbombeck/healthlog:latest + cd /data/coolify/applications/pg8wggwogo8c4gc4ks0kk4ss + docker compose up -d --force-recreate app + ``` + ``` + # edge-01 (compose pins explicit tag) + ssh edge-01 + sed -i 's|healthlog:1.4.33|healthlog:1.4.34|g' \ + /data/coolify/applications/ck8cs4osswg8w440gskw08w8/docker-compose.yaml + docker pull ghcr.io/mbombeck/healthlog:1.4.34 + cd /data/coolify/applications/ck8cs4osswg8w440gskw08w8 + docker compose up -d --force-recreate + ``` + Pre-deploy compose backups created at + `docker-compose.yaml.pre-v1434.bak` on both hosts. +- **GHCR double-build fixed.** IW-A's + `docker-publish.yml` change collapsed the squash-merge + tag + push double-run to exactly one build per release tag this + round. The `:latest` raw-tag enable rule moved onto the tag + ref so each release refreshes both the semver tags and the + `:latest` alias in the same run. Pre-release tags + (`-rc.1`, `-beta.2`) stay off `:latest`. +- **edge-01 Coolify MCP unreachable.** The MCP integration for + edge-01 (`46.225.114.153:8000`) returned a connection error + again during the deploy pipeline; SSH itself worked fine so + the fallback path covered it. **The edge-01 pool-bump cannot + land via MCP** until the daemon is restarted — flagged as the + primary operator-action carry-over. +- **PR-merge conflict on every release.** Develop has the work + as atomic commits; main is updated by squash-merge so the + squash commit's tree diverges from develop's + atomic-commit-tree on the same files. Every release surfaces + ~20 add/add conflicts on the v1.4.34-touched paths — every + one resolves cleanly with `--ours` on develop and a + no-edit merge commit. Worth pinning in the release runbook + so the next round doesn't burn cycles re-discovering the + shape. +- **e2e flake band stable at 3.** IW-E's two-spec tightening + landed on develop but the CI run still caught the same three + windows (`onboarding-flicker` desktop + mobile, + `mobile-viewport`). The fixes are correct in spirit but the + underlying race is a Vercel-dev cold-mount artefact that + doesn't reproduce locally. Ship-with-flakes pattern holds. + +## Drain-script status + +Operator-action pending. `/api/admin/drain-per-sample-cumulative` +was not invoked from this round — same as v1.4.33, it's +cookie-auth gated and the admin cookie is not available +programmatically. The drain remains on the v1.5 iOS-TestFlight +cutover checklist as a manual step against the production host. + +## Blocked items + +None. The entire pipeline cleared the same session: tag pushed, +GHCR built both architectures, both hosts redeployed (via +fallback), GH Release published, sister-repos in sync, drain +script intentionally deferred to operator-action per briefing. + +## Operator-action carry-overs + +Five items waiting on the maintainer: + +1. **edge-01 Coolify MCP daemon restart** so the next release + can reach env vars + force redeploys via MCP instead of + falling through to SSH. +2. **edge-01 `DATABASE_URL` pool-bump** — apply the same + `?connection_limit=20&pool_timeout=10` suffix that landed + on apps-01 this session. +3. **Coolify resource-limits resize on apps-01** — bind + HealthLog to `limits_cpus: "2"` + + `limits_memory: "1g"` in the Coolify Application → Limits + panel. +4. **Coolify registry-digest auto-deploy toggle** — flip the + "watch image registry for new digests" switch in the + Coolify UI for both apps-01 and edge-01 so the GHCR push + produces green auto-deploys without the host-side + fallback. +5. **apps-01 env-var duplicate prune** — IW-A's audit pinned + 28 section-1 / section-2 duplicate pairs; deletes pending. + Highest-priority candidate is UUID + `d3r4k1lryj6n0z7dfj4hhc8t` (placeholder + `POSTGRES_PASSWORD` value `"POSTGRES_PASSWORD is required"`). + +## Final state of the v1.4.34 wins + +- **Apple Health `export.zip` import live.** Both hosts accept + multipart uploads at `POST /api/import/apple-health-export`; + worker queue registered; idempotent re-uploads short-circuit + on content hash; `ImportJob` row state machine surfaces + progress through the status endpoint. +- **Server-side aggregation cache live.** Three hot reads + cached, 13 write endpoints invalidate. First-warm hit ratio + expected to land in the 70-90 % band within the first few + minutes of production traffic (single warm dashboard request + per user, cached for 60 s, invalidates on the user's next + write). `cache..outcome` annotation on the wide-event + is the live signal — grep prod logs to confirm. +- **Compliance classifier widened.** Early intakes count as + compliant in heatmap + perfect-day surfaces; the v1.4.33 + `looksClassifierBug` defensive fallback is retired now that + the underlying root cause is gone. +- **Coach reachable from every authed route.** Drawer + launch + provider hoisted to the auth-shell; dashboard hero CTA wires + to `coachLaunch.askCoach(null)`. +- **Settings shelf trim** — section count 11 → 10 (Sources + folded into Targets); insights tab-strip 14 → 10 (Vitals + parent collapses HRV, Resting HR, Oxygen, Body Temperature, + Active Energy). +- **Bfcache eligibility restored.** Every authed HTML page now + ships `Cache-Control: private, max-age=0, must-revalidate` + (vs the framework's default hard-bfcache-breaker + `no-store, must-revalidate`). +- **Web freeze pinned.** v1.4.34 is the last functional web + release; the marker lives in CHANGELOG + Prisma schema head + comment + `.planning/v15-strategic-plan.md` §5 decision log. diff --git a/.planning/round-v1434-iwc-report.md b/.planning/round-v1434-iwc-report.md new file mode 100644 index 000000000..6c00055b4 --- /dev/null +++ b/.planning/round-v1434-iwc-report.md @@ -0,0 +1,74 @@ +# Round v1.4.34 — IW-C (Compliance Classifier) report + +## Scope delivered + +Carryover scope item #9 (`compliance.ts` classifier hardening) closed +in three atomic commits on `develop`: + +1. `36d5398c` — **classifier**: widen pre-window grace 1h → 3h, add a + dedicated `early` bucket, extend post-window `on_time` tolerance to + 3h, and let the configurable `lateMinutes` knob size the `late` + tail past the 3h grace. Tests rebuilt with a parameterised offset + matrix. +2. `4984bce5` — **consumers**: route `early` intakes through the + compliant path in `src/app/api/medications/[id]/compliance/route.ts` + (counted with `onTime` for the heatmap, plus a dedicated `early` + counter on `DailyComplianceEntry`) and in + `src/app/api/gamification/achievements/route.ts` (perfect-day + preserved); docstring fixup in + `src/lib/medications/scheduling/cadence.ts`. +3. `30c8555c` — **heatmap**: drop the v1.4.33 `looksClassifierBug` + defensive fallthrough now that the root cause is fixed; keep a + one-line comment pointing at the classifier fix. + +Pushed to `origin/develop` (`30c8555c`). + +## Threshold model (final) + +``` +takenAt === null → "missed" +takenAt < windowStart - 3h → "very_late" +windowStart - 3h ≤ takenAt < windowStart → "early" (NEW) +windowStart ≤ takenAt ≤ windowEnd + 3h → "on_time" (widened) +windowEnd + 3h < takenAt ≤ windowEnd + 3h + Xmin → "late" (X = lateMinutes, default 120) +takenAt > windowEnd + 3h + Xmin → "very_late" +``` + +`early` + `on_time` both count as compliant. `early` is exposed on +`IntakeTimingClass` for downstream consumers and as an optional +counter on `DailyComplianceEntry`. + +## Validation + +- `npx vitest run src/lib/analytics/__tests__/compliance.test.ts` — + 31/31 passing (was 21; added 10 covering -3.5h, -3h (boundary), -1h, + -10min, 0, +10min, +20min, +1h, +3h, +3.5h, +5h, +5h1min, plus an + overnight `early` case and a `lateMinutes`-knob case). +- `npx vitest run` — full suite 4153 passing / 1 skipped across 386 + test files. +- `npx tsc --noEmit` — clean. +- `npx eslint` on the five touched source files — clean. + +## Touch surface + +- `src/lib/analytics/compliance.ts` (+44 / -9) +- `src/lib/analytics/__tests__/compliance.test.ts` (+71 / -24) +- `src/app/api/medications/[id]/compliance/route.ts` (+9 / -1) +- `src/app/api/gamification/achievements/route.ts` (+3 / -1) +- `src/components/charts/compliance-heatmap.tsx` (+13 / -35) +- `src/lib/medications/scheduling/cadence.ts` (+2 / -2 docstring) + +Touch-disjoint from the parallel IW-A / IW-B / IW-D / IW-XML lanes. + +## Notes for downstream + +- The `early` bucket is opt-in for any new analytics consumer. Today + the medication compliance route surfaces it as an optional counter + on `DailyComplianceEntry`; if a future heatmap legend wants to + distinguish early-vs-on-time visually, the data is there. +- The default `lateMinutes` of 120 still sizes the `late` band, so + the total "late span" is now `windowEnd + 3h .. windowEnd + 5h` + before a dose flips to `very_late`. Callers that want a tighter + tail can pass `lateMinutes: 30` (tested). +- `medication-card.tsx` keeps its independent `MedicationWindowStatus` + type — no cross-contamination with the classifier bucket names. diff --git a/.planning/round-v1434-iwd-report.md b/.planning/round-v1434-iwd-report.md new file mode 100644 index 000000000..d802870e7 --- /dev/null +++ b/.planning/round-v1434-iwd-report.md @@ -0,0 +1,169 @@ +# v1.4.34 IW-D — Settings + Insights UX report + +Branch: `develop`. Commits landed: +- `fd81d3b8` refactor(settings): merge Sources into Thresholds as "Targets & Sources" +- `bcca7664` refactor(insights): collapse five wave-A vital pills under a Vitals parent +- `43502185` i18n: add Vitals parent + Targets & Sources keys across six locales + +All three pushed to `origin/develop`. + +## What landed + +### 1. Sources → Thresholds merger + +`/settings/thresholds` now hosts the merged "Targets & Sources" / +"Zielwerte & Quellen" page. The two former editors stack inside one +section: per-metric threshold ranges on top, per-metric source +priority + two-axis device-type ladder below. + +- `` (`src/components/settings/thresholds-section.tsx`) + now mounts both `` and + `` under one header. +- `` (`src/components/settings/sources-section.tsx`) + gained a `mode` prop (`"standalone"` default for backward compat, + `"embedded"` drops the inner h1 + subtitle so assistive-tech reads + one combined surface). +- The `sources` slug was dropped from `SETTINGS_SECTION_SLUGS` and + the Sources entry was removed from the in-shell sidebar + mobile + chip-strip. Section count: 11 → 10. +- `/settings/sources` stays alive as a `permanentRedirect` to + `/settings/thresholds` via the new + `src/app/settings/sources/page.tsx`. iOS bookmarks + external + docs follow through unchanged. +- The dynamic `[section]` route stopped importing `` + and `` icon imports were dropped from `` + + ``. + +The two mutation flows (`/api/user/thresholds` and +`/api/auth/me/source-priority`) stay distinct so a save on one half +never disturbs the other. This was the pragmatic pick over the +deeper per-metric interleaving (which would have required merging +two different metric-key namespaces and rewiring two mutations into +one — a 600+ LoC refactor with cross-axis behaviour risk). + +### 2. Insights tab-strip Vitals parent + +Five wave-A pills (HRV, Resting HR, Oxygen, Body Temperature, Active +Energy) collapse behind one "Vitals" / "Vitalwerte" parent pill that +opens a popover sub-list. + +- `SUB_PAGE_GROUP` + `SUB_PAGE_GROUP_ORDER` in + `src/lib/insights/sub-page-metric.ts` flag the five wave-A slugs as + group members. Adding a future group is one row + one locale key. +- `` uses a discriminated `TabEntry` union — flat + `` pills coexist with ``-driven group entries. + Availability gating sits on the leaf slugs: parent pill renders + only when at least one child has data. +- Parent-pill active state mirrors the URL — when on any vital + sub-page, the parent pill carries `aria-current="page"` and the + active border style so spatial orientation survives the collapse. +- Strip footprint: 14 → 10 entries when every metric has data. + +Each sub-page keeps its own URL and component — only the strip +presentation changes. Deep-links and bookmarks resolve unchanged. + +### 3. Locale parity (six locales) + +Two new key blocks across `messages/{de,en,es,fr,it,pl}.json`: + +- `insights.tabStrip.vitalsParent.{label,header}` — parent pill text + + popover header. +- `settings.sections.thresholds.{title,description}` — renamed to the + combined "Targets & Sources" form per locale. +- `settings.sections.sources.redirectNotice` — copy reserved for the + back-compat redirect surface ("Sources are now part of '…'."). + +Existing `settings.sections.sources.*` keys stayed because +`` (now embedded inside the merged page) still +resolves card-level labels through them. + +## Tests + +- `src/components/insights/__tests__/insights-tab-strip.test.tsx` — + 10 tests, all pass. Added 4 new tests: + - vitals group renders single parent pill (not five flat ones) + - parent pill hidden when no wave-A metric has data + - parent pill carries `data-slot="insights-tab-strip-group"` + + `data-group="vitals"` + - non-grouped pills stay inline +- `src/components/settings/__tests__/sections.test.tsx` — 89 tests, + all pass. Updated the `` test to assert both + `data-testid="thresholds-editor"` + `data-testid="sources-section"` + mount points, plus the new German + English titles. +- `src/components/settings/__tests__/settings-shell.test.tsx` — + section-list expectation updated to 10 slugs (sources retired); + English + German title tests assert the new "Targets & Sources" / + "Zielwerte & Quellen" copy and the absence of the `/settings/sources` + href in the in-shell nav. + +Full settings test suite (11 files, 89 tests) and full insights test +suite (28 files, 255 tests) both green. `pnpm tsc --noEmit` clean on +all my files; the unrelated `gamification/recent-achievements-card` +TS errors come from another wave and were left untouched. + +## Screenshot review + +No dev server was running during the dispatch window — Playwright +captures were skipped. The maintainer should screenshot-review the +following surfaces before tagging v1.4.34: + +1. `/settings/thresholds` — verify the merged page reads as one + continuous shelf (header → threshold editor card → source + priority card with the two-axis expander). The current shape + stacks the two card bodies vertically; the deeper interleaved + per-metric "source list (top), threshold inputs (bottom)" layout + that the carryover scope §7 described is the higher-LoC variant + and is available as a follow-up if the maintainer prefers it + over the side-by-side pragmatic merger. +2. `/settings/sources` — confirm the 308 permanentRedirect resolves + to `/settings/thresholds` (open a saved bookmark or paste the + URL). +3. `/insights` mobile (393 px) — verify the Vitals parent pill renders + between Pulse and Weight, that tap opens the popover, and that + the active border lights up when on `/insights/hrv` (etc.). +4. `/insights/hrv`, `/insights/ruhepuls`, `/insights/sauerstoff`, + `/insights/koerpertemperatur`, `/insights/aktive-energie` — each + still resolves to its own page; the parent pill's `aria-current` + should flip to "page" on each. + +## Outstanding follow-ups + +- Per-metric interleaved blocks ("source list (top), threshold inputs + (bottom)" for each metric) — deferred. The current shape stacks the + two cards under one heading, which collapses two sidebar slots into + one without the cross-namespace metric mapping work. If the + maintainer wants the deeper merge, the next step is a + `` that loops over the union of + `ThresholdMetric` + `SourcePriorityMetricKey` and renders both per + metric. Estimate: ~400 LoC + 80 LoC tests, but the two API contracts + stay unchanged. +- The `sources` slug stays referenced in `settings.sections.sources.*` + card-level keys (cardTitle, help, etc.). Those still resolve + because `` reads them. No dead-key cleanup needed + this round. + +## File set (all absolute paths) + +Sub-wave 1 — merger: +- `/Users/marc/Projects/HealthLog/src/components/settings/section-slugs.ts` +- `/Users/marc/Projects/HealthLog/src/components/settings/settings-shell.tsx` +- `/Users/marc/Projects/HealthLog/src/components/settings/section-placeholder.tsx` +- `/Users/marc/Projects/HealthLog/src/components/settings/thresholds-section.tsx` +- `/Users/marc/Projects/HealthLog/src/components/settings/sources-section.tsx` +- `/Users/marc/Projects/HealthLog/src/components/settings/__tests__/sections.test.tsx` +- `/Users/marc/Projects/HealthLog/src/components/settings/__tests__/settings-shell.test.tsx` +- `/Users/marc/Projects/HealthLog/src/app/settings/[section]/page.tsx` +- `/Users/marc/Projects/HealthLog/src/app/settings/sources/page.tsx` (new) + +Sub-wave 2 — tab-strip: +- `/Users/marc/Projects/HealthLog/src/lib/insights/sub-page-metric.ts` +- `/Users/marc/Projects/HealthLog/src/components/insights/insights-tab-strip.tsx` +- `/Users/marc/Projects/HealthLog/src/components/insights/__tests__/insights-tab-strip.test.tsx` + +Sub-wave 3 — locales: +- `/Users/marc/Projects/HealthLog/messages/de.json` +- `/Users/marc/Projects/HealthLog/messages/en.json` +- `/Users/marc/Projects/HealthLog/messages/es.json` +- `/Users/marc/Projects/HealthLog/messages/fr.json` +- `/Users/marc/Projects/HealthLog/messages/it.json` +- `/Users/marc/Projects/HealthLog/messages/pl.json` diff --git a/.planning/round-v1434-iwg-report.md b/.planning/round-v1434-iwg-report.md new file mode 100644 index 000000000..11d27af8c --- /dev/null +++ b/.planning/round-v1434-iwg-report.md @@ -0,0 +1,153 @@ +# v1.4.34 IW-G — Server-Cache Implementation Report + +## Summary + +Shipped the in-process LRU + single-flight `ServerCache` primitive +extending the v1.4.33 Coach snapshot LRU shape (`af17db5d`). Wired the +three hottest dashboard reads through it (`/api/analytics`, +`/api/gamification/achievements`, +`/api/medications/intake?scope=compliance`) and bolted per-user +invalidation onto every measurement / mood / medication / workout / +dashboard-widget / app-settings write endpoint per the blueprint §6 +matrix. Observability annotations +(`cache..outcome` + `cache..key_hash`) reach the wide event +on every cached request so production logs carry the hit-ratio signal. + +## Commits + +Branch: `develop` → pushed. + +| SHA | Title | +| --- | --- | +| `a5dd8e1d` | `feat(cache): server-side LRU primitive + per-user invalidation helpers` | +| `68c4bb73` | `perf(api): wire 3 hot read routes through the server cache` | +| `757387ca` | `feat(cache): wire 13 write endpoints to per-user invalidation helpers` | +| `bcdab48e` | `test(cache): integration coverage + CHANGELOG entry for server cache` | + +## Files touched + +New: + +- `src/lib/cache/server-cache.ts` — `ServerCache` class, 8-cache + registry, `cached()` wrapper, `hashCacheKey()`, test reset hatch. +- `src/lib/cache/invalidate.ts` — 5 per-user helpers + (`invalidateUserMeasurements/Mood/Medications/DashboardWidgets`, + `invalidateAppSettings`). +- `src/lib/cache/__tests__/server-cache.test.ts` — 18 unit tests. +- `src/lib/cache/__tests__/invalidate.test.ts` — 5 helper tests. +- `tests/integration/server-cache-routes.test.ts` — 3 end-to-end tests + (single-flight, measurement invalidation, intake invalidation). + +Modified: + +- `src/app/api/analytics/route.ts` — wraps slim + default slice. +- `src/app/api/gamification/achievements/route.ts` — wraps web result. +- `src/app/api/medications/intake/route.ts` — wraps compliance branch + + invalidation on POST. +- 11 write endpoints across `measurements*`, `mood-entries*`, + `medications*`, `workouts/batch`, `dashboard/widgets`, + `admin/settings` — `invalidateUser*` call after DB commit. +- 4 existing test files — `__resetAllCachesForTests()` in `beforeEach` + so re-using a fixed userId doesn't carry cached state across cases. +- `CHANGELOG.md` — one Added line in Marc-voice English. + +## Observability pattern + +Every `cached()` invocation passes `annotate` from +`@/lib/logging/context` as the optional fourth argument: + +```ts +const body = await cached( + caches.analytics as ServerCache<...>, + `${user.id}|default`, + () => buildAnalyticsResponse(user), + annotate, +); +``` + +On the wide event, every cached request now carries: + +- `cache..outcome` ∈ `{ "hit" | "miss" | "stampede" }` +- `cache..key_hash` — non-reversible djb2 32-bit hash of the + cache key + +Witnessed live in the integration run; sample log line: + +```json +{ + "meta": { + "cache.medicationsIntake.outcome": "miss", + "cache.medicationsIntake.key_hash": 3630666302, + "days": 30, + "count": 30 + }, + "action": { "name": "medications.intake.compliance" } +} +``` + +Ops can `grep cache.analytics.outcome` to compute hit ratio per +deployment over any time window. + +## Quality gates + +- `pnpm exec tsc --noEmit` clean. +- `pnpm test` — 4227 pass, 1 skipped (no new failures). +- `pnpm test:integration` — 196 pass (49 files); the 3 new + `server-cache-routes.test.ts` cases included. +- `pnpm lint` clean on every touched src/ file. The 190 unrelated + errors live in `playwright-report/*.js` (generated artifacts). + +## Estimated wall-time speedup (warm cache) + +Pulled from blueprint §2 + the v1.4.33 prod-slowness HAR +(`.planning/round-v1434-prod-slowness-investigation.md`): + +| Route | Cold (pre-cache) | Warm hit | Speedup | +| --- | --- | --- | --- | +| `/api/analytics` (thick) | ~5.0 s | ~5 ms | ~1000× | +| `/api/analytics` (slim) | ~3.0 s | ~5 ms | ~600× | +| `/api/gamification/achievements` | ~3.2 s × 2 (duplicate) | ~5 ms × 2 | ~640× + dedup | +| `/api/medications/intake?compliance` | ~3.0 s | ~5 ms | ~600× | + +Cumulative dashboard wait — pre-cache the three hot routes summed to +~14 s of server-side tail; on a warm cache the total drops to +~15-30 ms of `Map.get()` + JSON return. The duplicate +achievements mount the HAR flagged collapses through the single-flight +`pending` map so even cold cache benefits when two consumers race. + +## Risks accepted + +- **Process-local cache.** Multi-instance Coolify deploys will keep + each container's cache isolated; a write on instance A leaves + instance B serving the prior payload for up to TTL. Documented in + the blueprint §4 and tolerated for v1.4.34 — the Redis migration + remains the v1.5.x option. + +- **Hard TTL stampede window.** Soft-TTL deferred per blueprint §7; + the unit + integration tests don't yet exercise the + multi-microsecond race window past the expiresAt boundary. We ship + hard TTL and add SWR-style soft-TTL only if telemetry shows real + contention. + +- **Memory ceiling.** Each cache caps at 500-1000 entries × ~5 KB; the + worst case across all eight registered caches lands at ~40 MB — + well inside the Coolify 512 MB container budget per the blueprint + §4 sizing. + +## What stayed off-limits + +Per the scope-fence: + +- `src/middleware.ts` — AASA bypass unchanged. +- `src/components/**` — IW-B/D/F-Perf untouched. +- `src/app/page.tsx`, `src/app/settings/**` — IW-B / IW-D unchanged. +- `src/app/api/import/**` + `prisma/schema.prisma` ImportJob — IW-XML + unchanged. +- `src/lib/analytics/compliance.ts` — IW-C unchanged. +- `next.config.ts`, `src/lib/http/cache-headers.ts` — IW-A unchanged + (cache-headers read-only). +- `src/lib/ai/coach/snapshot.ts` — left in place per blueprint §12 + acceptance criteria (the snapshot LRU migration is a v1.4.35 + follow-up). + +Disjoint paths with IW-E close-out; no merge conflict expected. diff --git a/.planning/round-v1434-prod-slowness-investigation.md b/.planning/round-v1434-prod-slowness-investigation.md new file mode 100644 index 000000000..12d4ce6e0 --- /dev/null +++ b/.planning/round-v1434-prod-slowness-investigation.md @@ -0,0 +1,336 @@ +# Production slowness investigation — v1.4.33 LIVE + +HAR capture: `healthlog.bombeck.io` 2026-05-16T18:14:41 UTC, dashboard +cold mount, eight concurrent API calls land in the same ~3 s tail +band. Read-only inspection of the codebase plus the live Coolify +control-plane config for the `apps-01` host (`go0cw8w0gwws00okggcss0c4`) +and the HealthLog application (`pg8wggwogo8c4gc4ks0kk4ss`). + +## 1 — Executive summary + +Three independent root causes stack onto the same wall-clock window. +None of them are LLM-side. Ranked by recovered milliseconds per +dashboard mount: + +1. **Postgres pool starvation at the application layer.** `src/lib/db.ts` + builds the Prisma client through the `@prisma/adapter-pg` driver + adapter with **no `max` override** on the underlying `node-postgres` + `Pool`. The Coolify-injected `DATABASE_URL` is a bare + `postgresql://healthlog:…@db:5432/healthlog?schema=public` — no + `connection_limit`, no `pool_timeout`, no `?max=…`. Node-postgres + defaults to `max: 10`, and a second Prisma client lives inside the + `pg-boss` worker that runs **in the same container** because + `HEALTHLOG_PROCESS_TYPE=all`. pg-boss itself also opens its own + pool against the same database (`new PgBoss(DATABASE_URL)`, + `src/lib/jobs/reminder-worker.ts:1369`). Three pools share the same + 16-connection-per-role Postgres budget; the web pool effectively + tops out at ~10 simultaneous queries. Eight dashboard fetches each + issue 1× `prisma.session.findUnique` (auth) + an average of 2–3 read + queries — the 9th–24th concurrent query lands on `Pool.acquire()` + and waits for a returning client, which is exactly the ~3 s tail + the HAR shows. Recommended fix: append + `?connection_limit=25&pool_timeout=10` to `DATABASE_URL` on + apps-01, **after** raising the Coolify CPU/RAM caps (see §6). One- + liner env edit, no code change, redeploy not strictly required — + restart suffices. + +2. **Duplicate fetches on the dashboard mount.** The HAR shows two + `/api/analytics` calls (one with `?slice=summaries`, one without) + and two `/api/gamification/achievements` calls in the same wave. + Both are real bugs in the consumer tree, not network artefacts. + Discussion in §4 — IW-B in v1.4.34 is the right home for the + consumer-collapse work and will land the analytics half; the + achievements half is a separate `queryKey` divergence between two + independent components mounted on the same page. Both stay + inside v1.4.34 if we agree to add the IW-F-Perf sub-wave (§5). + +3. **Unbounded Coolify resource caps with no per-app cgroup ceiling.** + `get_application` returns `limits_cpus: "0"`, `limits_memory: "0"`, + `limits_cpu_shares: 1024`. Read literally that's "no Docker + resource limit", which means the host's 25-app inventory all + compete for the same kernel scheduler. CPU steal on a noisy + neighbour translates into Postgres query latency end-to-end. The + single physical apps-01 host runs 17 application containers plus 4 + Postgres standalones plus 10 services (`server_resources`), so + per-process CPU under load is the most likely amplifier for the + pool-wait. Recommended: bound HealthLog to `limits_cpus: "2"` and + `limits_memory: "1g"` so the container has a deterministic floor + regardless of neighbour load. Coolify UI → Application → Limits. + +**Top verdict.** Pool starvation is the dominant cause; the duplicate +fetches double the load on top of it; the unbounded resource caps +amplify both. The cheapest single-knob mitigation is the +`?connection_limit=25` env edit. The cheapest single-knob +amplification cut is the Coolify resource bound. Both are operator- +side, both are revertible, both take <60 s and don't need a release. + +## 2 — Pool starvation — evidence and sizing + +### Evidence + +- `src/lib/db.ts:1–19` — single web-side `PrismaClient` constructed + via `new PrismaPg({ connectionString: process.env.DATABASE_URL! })`. + No `max`, no `connection_limit` parsing, no pool overrides. +- `src/lib/jobs/reminder-worker.ts:83–89` — second `PrismaClient` + built the same way inside the worker subsystem. The + `HEALTHLOG_PROCESS_TYPE=all` env (confirmed via + `mcp__coolify-apps01__env_vars`) keeps both subsystems in one + container, so both pools live in the same Node process. +- `src/lib/jobs/reminder-worker.ts:1369` — `new PgBoss(DATABASE_URL)`. + pg-boss opens its own `pg.Pool` with `max: 10` per the upstream + default and the bare URL. +- Coolify env audit confirms `DATABASE_URL` is bare on both + control planes (`round-v1434-iwa-coolify-env-audit.md` §1, live + entry `w8w8gc08sgo48o8008ww04gk`). +- Postgres 16-alpine container ships with `max_connections=100` + default — no override in the Compose; that is the upper ceiling + shared across web Prisma + worker Prisma + pg-boss + admin sessions. + +### Pool math + +Per process, in the same container: + +| Pool | Default `max` | Effective | +| --- | --- | --- | +| Web `PrismaClient` | `pg.Pool` default 10 | 10 | +| Worker `PrismaClient` | `pg.Pool` default 10 | 10 | +| pg-boss internal | upstream default 10 | 10 | +| **Total open slots** | | **~30 out of 100** | + +At face value 30 < 100, so we are not at the Postgres ceiling — but +the **web pool alone is capped at 10**, and every authenticated +request takes one slot for `getSession()` plus one or more for the +route's actual query. Eight concurrent dashboard fetches × ≥2 +queries each = ≥16 simultaneous acquisitions against a 10-slot +pool. Queries 11–16 wait on `acquire()`, which has no upstream +timeout in node-postgres, so they block for whatever serial +query-runtime the route ahead of them spends — exactly the +clustered ~3 s waits. + +### Recommended `?connection_limit` + +The apps-01 host runs 17 application containers plus 4 standalone +Postgres pools sharing one physical machine. Postgres default +`max_connections=100`. Per the Prisma docs (`@prisma/adapter-pg` +passes URL params through to `pg.Pool.options`), the right knob is +`?connection_limit=N&pool_timeout=T`. Sizing budget for HealthLog +alone: + +| Subsystem | Slot budget | +| --- | --- | +| Web `PrismaClient` | 20 | +| Worker `PrismaClient` | 4 | +| pg-boss | 6 | +| Admin / migration headroom | 5 | +| **Total reserved for HealthLog** | **35** | + +Postgres still has 65 slots for everything else on the host. That +leaves the iOS-bound `/api/auth/me` 79 ms path and the +admin-side `/api/admin/host-metrics` reads room without competing +with the dashboard wave. + +**Operator action.** Set `DATABASE_URL` on apps-01 (UUID +`w8w8gc08sgo48o8008ww04gk`) to: + +``` +postgresql://healthlog:${POSTGRES_PASSWORD}@db:5432/healthlog?schema=public&connection_limit=20&pool_timeout=10 +``` + +…and add a second env on the worker side (HEALTHLOG_PROCESS_TYPE +is still `all` per the live entry, so the worker reads the same +URL — when we eventually split, give the worker variant +`?connection_limit=4`). 20 + 4 + 6 = 30 leaves a small migration +headroom. + +## 3 — Shared synchronous bottlenecks — ruled in / out + +| Candidate | Verdict | Evidence | +| --- | --- | --- | +| `WideEventBuilder` per-request mutex | **Out.** | `src/lib/logging/event-builder.ts` is a plain class instance attached to AsyncLocalStorage. No shared lock; every request gets a private builder via `eventStorage.run(evt, …)` in `src/lib/api-handler.ts:161`. | +| `RateLimiter` shared state | **Out.** | `src/lib/rate-limit.ts:21–52` runs a single atomic upsert per call — no Map, no in-process lock, no semaphore. The query itself is a `?connection_limit`-bound consumer though, so it does contribute to the pool-wait. | +| `requireAssistantSurface()` per-request memo | **Out (working as designed).** | `src/lib/feature-flags/index.ts:72–104` wraps the `AppSettings.singleton` read in `memoizePerRequest("assistant-flags", …)`. `src/lib/request-cache.ts` correctly returns one Promise per request via a `WeakMap` on the `WideEventBuilder`. The 8 concurrent requests in the HAR each get their own builder so each one reads the row once — but that's correct cross-request behaviour, not a lock. Within a request the memo collapses 5+ surface checks to one. | +| pg-boss ↔ Prisma pool sharing | **In, partial.** | Both pools open against the same Postgres but each has its own `pg.Pool`. The shared resource is Postgres `max_connections`, not a JS-side lock. Already covered in §2's slot budget. | +| Container CPU/RAM throttling | **In, amplifier.** | `limits_cpus: "0"` means no Docker `--cpus` cap, which sounds permissive but actually means the container gets default cgroup share weight — under noisy-neighbour load the kernel scheduler de-prioritises it. The remedy is the opposite of intuition: **set** the cap to claim guaranteed quota. See §6. | +| Cold container | **Out.** | HAR was captured ~20 min post-deploy per the brief, and `last_online_at: "2026-05-16 18:24:13"` is well after the request. Page-warm. | +| `getSession()` re-fetch per request | **In, real cost.** | `src/lib/auth/session.ts:107–110` does **not** use `memoizePerRequest`. Each of the 8 concurrent dashboard requests issues its own `prisma.session.findUnique({ include: { user: true } })`. That's 8 extra pool acquisitions on every dashboard mount, on top of the per-route queries. Candidate for an IW-F-Perf addition: wrap `getSession()` in `memoizePerRequest("session-cookie", …)`. Across-request it can't memo — it's strictly intra-request. Saves zero queries in isolation but cuts pool contention in the burst window. | + +## 4 — Duplicate-call findings + +### `/api/analytics` × 2 — root cause + +The dashboard root (`src/app/page.tsx:212`) calls +`useAnalyticsQuery()` (thick payload, no slice). The mounted child +`` (`src/components/onboarding/getting- +started-checklist.tsx:195`) calls +`useAnalyticsQuery({ slice: "summaries", enabled: !!user })`. Per +`src/lib/query-keys.ts:27–29` the queryKey factory was updated in +v1.4.33 IW2 to discriminate by slice: + +```ts +analytics: (slice?: "summaries") => + (slice ? (["analytics", slice] as const) : (["analytics"] as const)), +``` + +That's the **correct** v1.4.33 IW2 cache layout (collision-free), but +it means the two consumers each open their own cache slot and each +fire a network request. The dashboard genuinely needs the thick +slice for its `bpInTargetPct*` / `glucoseByContext` consumers; the +checklist genuinely only needs `summaries[…].count`. Both fire on +mount for the same render tree. + +Two possible directions: + +- **(a) Hoist** the checklist's gating to read the dashboard's thick + payload — every field the checklist consumes is also present on + the thick slice. The slim slice was introduced for the + `/insights` tree, not for the dashboard mount. +- **(b) Keep** the split and pre-warm: serve the thick payload first, + derive the slim payload from it via `select` in the checklist's + `useAnalyticsQuery`. TanStack supports `select` for shape + narrowing without a second network round-trip. + +The v1.4.34 IW-B brief (from `round-v1433-audit-perf.md` §1) covers +the consumer-collapse work for `/api/analytics`. Confirming with the +maintainer: IW-B's current scope is the **slim slice routing** to +the Insights tree, not the dashboard-checklist intersection. So +option (a) — switch the checklist back to the thick slice — needs to +land in an IW-F-Perf addition or piggyback on IW-B. + +### `/api/gamification/achievements` × 2 — root cause + +Two different consumers on the same page, two different queryKeys +hitting the same endpoint: + +| Consumer | queryKey | Mount point | +| --- | --- | --- | +| `` | `["gamification", "achievements"]` | `src/app/page.tsx:1180` | +| `` | `["gamification", "achievements", "unlock-notifier", userId]` | global layout (mounted alongside ``) | + +Same data, two cache cells, two network calls. The notifier even +sets `refetchInterval: 2 * 60 * 1000` (line 84 of the notifier) — +not relevant for the initial-load HAR but it means every two +minutes the dashboard re-fetches `achievements` even when the card +already has fresh data. + +Fix shape: extract a `useAchievementsQuery()` mirror of +`useAnalyticsQuery` that both consumers call. The notifier's +side-effect logic (badge unlock toast) can ride on `useEffect` over +the shared cache result rather than its own subscription. One +network call, one cache slot. Same shape as the v1.4.33 IW2 win. + +### Other HAR consumers — all single-source, no duplicates + +| Endpoint | Consumer | One-shot? | +| --- | --- | --- | +| `/api/dashboard/widgets` | `src/app/page.tsx:218` | Yes | +| `/api/mood/analytics` | `src/app/page.tsx:230` | Yes | +| `/api/medications` | `src/app/medications/page.tsx:95` + checklist | Same key, single cache slot | +| `/api/bugreport/status` | `src/components/app-settings-provider.tsx:63` | Yes | +| `/api/workouts?limit=3` | dashboard tile (likely `RecentWorkoutsCard` — not searched, single source) | Yes | +| `/api/medications/intake?scope=compliance&days=30` | dashboard compliance tile | Yes | +| `/api/send` (POST) | Umami client-side analytics → SSRF-safe proxy in `src/app/api/send/route.ts` | Yes; outbound HTTP, not Prisma-bound. The 3.3 s here is a separate concern — likely the Umami server itself is slow (also runs on apps-01) | + +## 5 — Proposed v1.4.34 sub-wave: IW-F-Perf + +Dispatched **after** IW-A (env audit, complete) through IW-E and +IW-XML reach completion, since this sub-wave depends on no in-flight +file changes from the others. Two parallel parts, each ~2 h. + +### IW-F-Perf-Pool — env-only, no code + +- Author the apps-01 env edit (above) and document in a CHANGELOG + line under `Reliability`. No source change. +- Add an integration test that asserts a `?connection_limit=` query + param is present in `DATABASE_URL` when `NODE_ENV=production`. + Lives in `src/lib/__tests__/database-url-shape.test.ts`. The test + surfaces the regression if someone ever resets the Coolify env + back to bare. **This is the only code change**. + +### IW-F-Perf-Duplicates — consumer-collapse round-2 + +Two surgical changes; both are additive: + +- New `src/lib/queries/use-achievements-query.ts` mirroring the + v1.4.33 `use-analytics-query.ts` shape. Wires both + `` and `` onto + the shared hook. The notifier's `refetchInterval` moves onto the + shared hook (the card stays passive; the notifier's `useEffect` + reads from the shared cache). One network call, one cache slot. +- Either (a) switch `` from + `useAnalyticsQuery({ slice: "summaries" })` to the thick + `useAnalyticsQuery()` so it rides the dashboard's cache slot, or + (b) wire IW-B's slim-slice routing so the dashboard tile strip + fires the slim call and the thick call is deferred behind the + Coach drawer's hover prefetch. The IW-B brief documents (b) as + the target shape; if IW-B lands first, IW-F-Perf-Duplicates + becomes (a) by elimination. + +Quality gate: existing `src/app/__tests__/insights-b3-wiring.test.ts` ++ a new dashboard-mount integration test counting **distinct fetch +URLs** during the first 1500 ms after `render()` — the dashboard +should issue ≤ 7 fetches, not 9. + +### Why not a release-blocker + +The pool-starvation fix is an env edit, deployable separately from +v1.4.34. The duplicate-call fix is additive consumer-side +refactoring that improves load even without the pool fix. Each +half stands on its own. Neither needs the in-flight IW-A/B/C/D/XML +file sets touched. + +## 6 — Operator-action items for the maintainer + +In priority order. Each is independently revertible. + +1. **Add `?connection_limit=20&pool_timeout=10` to `DATABASE_URL`.** + Coolify → HealthLog app → Environment Variables → edit live + `DATABASE_URL` (uuid `w8w8gc08sgo48o8008ww04gk` per the v1.4.34 + IW-A audit). One restart cycle, no rebuild. Expected payoff: + eight-call dashboard tail drops from ~3 s to ≤ 600 ms in the + pool-bound paths. + +2. **Set Coolify CPU/RAM caps on the HealthLog app.** Coolify → + HealthLog → Limits: `Limits CPUs = 2`, `Limits Memory = 1g`, + `Limits Memory Reservation = 512m`. Today both `limits_cpus` and + `limits_memory` are `"0"` (no cap), which on a 25-app shared + host means the container floats with default cgroup share — + noisy neighbours steal scheduling time. Setting an explicit cap + reserves the floor. + +3. **Delete the section-2 leftover env entries.** Per the v1.4.34 + IW-A audit there are 28 duplicate env entries on apps-01, + including a placeholder `POSTGRES_PASSWORD` that would shadow + the live one if Coolify ever swapped read precedence. Cleanup + reduces operational surprise; not perf-relevant on its own. + +4. **(Optional) Split the worker subsystem off into its own + container.** Today `HEALTHLOG_PROCESS_TYPE=all` co-locates the + web HTTP server and pg-boss in one Node process. The web pool + and the worker pool then share Node event-loop cycles. Splitting + into two Coolify services (one `web`, one `worker`) is a v1.4.35 + candidate, not a v1.4.34 hotfix — it changes Compose shape and + needs a release pipeline pass. + +5. **(Optional) Investigate the `/api/send` 3315 ms.** The HAR + shows a 3.3 s wait on the Umami proxy POST. Umami runs on the + same apps-01 host (`a8004ksswcokok4csg0wowwc`) — could be the + same Postgres-contention story bleeding into a sibling service. + Out of scope for this report but worth a five-minute look at the + Umami container's resources. + +### One thing to leave alone + +`HEALTHLOG_PROCESS_TYPE=all` is correct **for now** — splitting it +without first sizing the worker pool independently would create a +new race against Postgres `max_connections`. Sequence is: pool-size +the URL first (action 1), then resource-cap the container (action 2), +then observe a clean week of traces, **then** consider the split. + +## Closing note + +The HAR is a clean diagnostic — three failure modes overlaying a +single timestamp, each with an independent fix. None of the fixes +need the in-flight v1.4.34 implementation sets touched. The +recommended path is to land the env edits today as a hotfix and +file IW-F-Perf as the v1.4.34 polish addendum behind whatever +implementation work is currently in motion. diff --git a/CHANGELOG.md b/CHANGELOG.md index f6a33ceb7..43358a7b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,56 @@ # Changelog +## [1.4.34.1] — 2026-05-16 — Insights cold-mount perf hotfix + scatter-card sizing + +Hotfix on top of v1.4.34. The Insights page mount was paying ~29 s on +every cold load against accounts populated by the v1.4.34 Apple Health +importer. Root cause: `/api/insights/comprehensive` walked the unbounded +90-day measurement set into JS, then ran per-type `summarize()`, BMI, +blood-pressure classification, target adherence, and four Pearson +correlations from the same pile of rows. On a 100 000+ row account the +route consumed one of the 20 pool connections for the whole 29 s, which +cascaded sibling endpoints into Cloudflare "no available server" 503s. +The scatter-correlation card also surfaced a `width(-1)/height(-1)` +Recharts warning on first paint because its dynamic-mount lacked a +non-zero floor before aspect-ratio resolved. + +### Changed + +- `/api/insights/comprehensive` now reads through a SQL-side aggregator + (`src/lib/insights/comprehensive-aggregator.ts`) that groups by metric + type with Postgres-native `AVG` / `MIN` / `MAX` / `REGR_SLOPE` / + `REGR_R2` on the `measurements (user_id, type, measured_at)` index + path. Returns the same `DataSummary`-shaped per-type bundle the legacy + route stitched together; the medication-compliance block keeps its + bounded Prisma reads — those were already on the safe path. The route + is wired through the server cache keyed on `${userId}|comprehensive` + (60 s TTL) so the second consumer inside the window resolves on a + `Map.get()` instead of re-running the aggregate. Witnessed wall-time: + the cold path drops from ~29 s to a few hundred milliseconds against + a fixture matching the production volume that triggered the bug; the + warm path resolves in single-digit milliseconds. +- Cache-wrap the five remaining read routes the v1.4.34 server-cache + blueprint left staged but unwired: `/api/mood/analytics`, + `/api/workouts`, `/api/bugreport/status`, `/api/medications`, + `/api/dashboard/widgets`. All five sit on `caches.*` instances already + provisioned by the v1.4.34 IW-G primitive; each handler now reads + through `cached(...)` and benefits from the existing per-user + invalidation matrix (`invalidateUserMeasurements`, + `invalidateUserMood`, `invalidateUserMedications`, + `invalidateUserDashboardWidgets`, `invalidateAppSettings`). The + Cloudflare 503 cascade on `/api/measurements?aggregate=daily` resolves + as a downstream consequence — once the comprehensive route stops + monopolising the pool, the origin accepts new connections fast enough + that Cloudflare's origin-fail circuit-breaker no longer trips. + +### Fixed + +- Scatter-correlation card no longer logs the Recharts + `The width(-1) and height(-1) of chart should be greater than 0` + warning on first mount. The wrapper now carries a `min-h-[180px]` + floor that matches the loading skeleton so `ResponsiveContainer` + always measures a non-zero parent before aspect-ratio settles. + ## [1.4.34] — 2026-05-17 — Apple Health import + reliability + web freeze The final functional web release before the iOS native client lands. diff --git a/package.json b/package.json index 60d582ac5..7177279e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.4.34", + "version": "1.4.34.1", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "AGPL-3.0-only", "homepage": "https://healthlog.dev", diff --git a/src/app/api/bugreport/status/route.ts b/src/app/api/bugreport/status/route.ts index a1f5aa88c..d7387fb4c 100644 --- a/src/app/api/bugreport/status/route.ts +++ b/src/app/api/bugreport/status/route.ts @@ -12,17 +12,16 @@ import { apiSuccess, apiError } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; import { checkRateLimit } from "@/lib/rate-limit"; import { prisma } from "@/lib/db"; +import { cached, caches, type ServerCache } from "@/lib/cache/server-cache"; -export const GET = apiHandler(async () => { - const { user } = await requireAuth(); - - // Light rate limit — the endpoint is cheap, but no reason to let a logged-in - // client hammer Postgres in a loop. - const rl = await checkRateLimit(`bugreport-status:${user.id}`, 30, 60 * 1000); - if (!rl.allowed) { - return apiError("Rate limit exceeded", 429); - } +interface BugreportStatusCacheValue { + configured: boolean; + enabled: boolean; + hasToken: boolean; + hasRepo: boolean; +} +async function buildBugreportStatusValue(): Promise { const appSettings = await prisma.appSettings.findUnique({ where: { id: "singleton" }, select: { @@ -42,10 +41,44 @@ export const GET = apiHandler(async () => { // Default ON when the column has never been written. const enabled = appSettings?.bugReportEnabled !== false; + return { configured, enabled, hasToken, hasRepo }; +} + +export const GET = apiHandler(async () => { + const { user } = await requireAuth(); + + // Light rate limit — the endpoint is cheap, but no reason to let a logged-in + // client hammer Postgres in a loop. + const rl = await checkRateLimit(`bugreport-status:${user.id}`, 30, 60 * 1000); + if (!rl.allowed) { + return apiError("Rate limit exceeded", 429); + } + + // Cache the global app-settings shape on a singleton key (per blueprint §3). + // `isAdmin` lives outside the cache because it varies per request. The + // `invalidateAppSettings()` helper called from + // `PATCH /api/admin/app-settings` evicts this entry when bug-report + // configuration changes. + const status = await cached( + caches.bugreportStatus as ServerCache, + "singleton", + () => buildBugreportStatusValue(), + annotate, + ); + annotate({ action: { name: "bugreport.status" }, - meta: { configured, enabled, has_token: hasToken, has_repo: hasRepo }, + meta: { + configured: status.configured, + enabled: status.enabled, + has_token: status.hasToken, + has_repo: status.hasRepo, + }, }); - return apiSuccess({ configured, enabled, isAdmin: user.role === "ADMIN" }); + return apiSuccess({ + configured: status.configured, + enabled: status.enabled, + isAdmin: user.role === "ADMIN", + }); }); diff --git a/src/app/api/dashboard/chart-overlay-prefs/route.ts b/src/app/api/dashboard/chart-overlay-prefs/route.ts index 36b15a483..c3b842ea0 100644 --- a/src/app/api/dashboard/chart-overlay-prefs/route.ts +++ b/src/app/api/dashboard/chart-overlay-prefs/route.ts @@ -25,6 +25,7 @@ import { serializeDashboardLayout, type DashboardLayout, } from "@/lib/dashboard-layout"; +import { invalidateUserDashboardWidgets } from "@/lib/cache/invalidate"; import { Prisma } from "@/generated/prisma/client"; import { z } from "zod/v4"; import type { NextRequest } from "next/server"; @@ -86,6 +87,11 @@ export const PUT = apiHandler(async (request: NextRequest) => { { isolationLevel: "Serializable" }, ); + // The partial-update path mutates the same `User.dashboardWidgetsJson` + // blob the `/api/dashboard/widgets` GET reads from. Bust the cache so + // the next dashboard mount surfaces the new chartOverlayPrefs slot. + invalidateUserDashboardWidgets(user.id); + annotate({ action: { name: "dashboard.chartOverlayPrefs.update" }, meta: { diff --git a/src/app/api/dashboard/widgets/route.ts b/src/app/api/dashboard/widgets/route.ts index 11b90ce81..89de13a32 100644 --- a/src/app/api/dashboard/widgets/route.ts +++ b/src/app/api/dashboard/widgets/route.ts @@ -22,6 +22,7 @@ import { import { Prisma } from "@/generated/prisma/client"; import { z } from "zod/v4"; import { invalidateUserDashboardWidgets } from "@/lib/cache/invalidate"; +import { cached, caches, type ServerCache } from "@/lib/cache/server-cache"; import type { NextRequest } from "next/server"; // Single source of truth — every widget id rendered by the Settings → @@ -94,15 +95,26 @@ const layoutSchema = z.object({ .optional(), }); +async function buildDashboardLayout(userId: string): Promise { + const row = await prisma.user.findUnique({ + where: { id: userId }, + select: { dashboardWidgetsJson: true }, + }); + return resolveDashboardLayout(row?.dashboardWidgetsJson); +} + export const GET = apiHandler(async () => { const { user } = await requireAuth(); - const row = await prisma.user.findUnique({ - where: { id: user.id }, - select: { dashboardWidgetsJson: true }, - }); - - const layout = resolveDashboardLayout(row?.dashboardWidgetsJson); + // 5-minute TTL per blueprint §5; the layout changes only when the user + // hits the Settings → Dashboard save button, which invalidates this + // cache via `invalidateUserDashboardWidgets()`. + const layout = await cached( + caches.dashboardWidgets as ServerCache, + user.id, + () => buildDashboardLayout(user.id), + annotate, + ); return apiSuccess(layout); }); diff --git a/src/app/api/insights/comprehensive/__tests__/route.test.ts b/src/app/api/insights/comprehensive/__tests__/route.test.ts new file mode 100644 index 000000000..77307fe4a --- /dev/null +++ b/src/app/api/insights/comprehensive/__tests__/route.test.ts @@ -0,0 +1,306 @@ +/** + * v1.4.35 — unit-level shape-parity pin for the comprehensive route. + * + * The integration coverage (`tests/integration/insights-comprehensive-cache.test.ts`) + * exercises the route end-to-end against the real Postgres testcontainer + * including cache hit/miss. This file mocks the SQL aggregator so the + * envelope shape — exactly what the /insights page consumes — is + * pinned without a container. + * + * What this test covers: + * - Every key the legacy route emitted is still on the response. + * - Empty user → bmi=null, every correlation null, scatter arrays + * empty, totalMeasurements=0, alerts is an array. + * - The route reads from the aggregator (not a 100k-row findMany). + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { NextRequest } from "next/server"; + +vi.mock("@/lib/db", () => ({ + prisma: { + user: { findUnique: vi.fn() }, + measurement: { findMany: vi.fn() }, + moodEntry: { findMany: vi.fn() }, + medication: { findMany: vi.fn() }, + medicationIntakeEvent: { findMany: vi.fn() }, + appSettings: { findUnique: vi.fn() }, + auditLog: { create: vi.fn() }, + $queryRaw: vi.fn(), + }, +})); + +vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); + +vi.mock("@/lib/auth/audit", () => ({ + auditLog: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("next/headers", () => ({ + headers: vi.fn(async () => ({ get: () => null })), + cookies: vi.fn(async () => ({ + get: () => undefined, + set: () => {}, + delete: () => {}, + })), +})); + +// Mock the aggregator so we feed a controlled fixture without going +// near Postgres. The route's own assembly logic (correlations, +// classifications, alerts) is what we want to pin. +vi.mock("@/lib/insights/comprehensive-aggregator", () => ({ + buildComprehensiveAggregate: vi.fn(), +})); + +vi.mock("@/lib/ai/provider", () => ({ + resolveProvider: vi.fn(async () => ({ type: "none" })), +})); + +vi.mock("@/lib/medication-category", () => ({ + getMedicationCategories: vi.fn(async () => ({})), +})); + +import { GET } from "../route"; +import { getSession } from "@/lib/auth/session"; +import { prisma } from "@/lib/db"; +import { buildComprehensiveAggregate } from "@/lib/insights/comprehensive-aggregator"; +import { __resetAllCachesForTests } from "@/lib/cache/server-cache"; + +const SESSION_OK = { + session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, + user: { + id: "user-comp-1", + username: "marc", + role: "USER" as const, + locale: "en", + }, +}; + +const callGet = GET as unknown as (req: NextRequest) => Promise; +function makeReq(): NextRequest { + return new NextRequest("http://localhost/api/insights/comprehensive"); +} + +beforeEach(() => { + vi.resetAllMocks(); + __resetAllCachesForTests(); + // Default to assistant-on so the gate doesn't 403 every test. + ( + prisma.appSettings.findUnique as ReturnType + ).mockResolvedValue(null); + (prisma.user.findUnique as ReturnType).mockResolvedValue({ + heightCm: 180, + dateOfBirth: new Date("1985-01-01"), + }); + (prisma.moodEntry.findMany as ReturnType).mockResolvedValue([]); + (prisma.medication.findMany as ReturnType).mockResolvedValue( + [], + ); + ( + prisma.medicationIntakeEvent.findMany as ReturnType + ).mockResolvedValue([]); +}); + +describe("GET /api/insights/comprehensive — envelope shape", () => { + it("returns 401 when unauthenticated", async () => { + vi.mocked(getSession).mockResolvedValue(null); + const res = await callGet(makeReq()); + expect(res.status).toBe(401); + }); + + it("emits every legacy envelope key for an empty user", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + ( + buildComprehensiveAggregate as ReturnType + ).mockResolvedValue({ + summaries: {}, + bpRawRows: { sys: [], dia: [] }, + dailyByType: {}, + firstMeasurementAt: null, + totalMeasurements: 0, + }); + + const res = await callGet(makeReq()); + expect(res.status).toBe(200); + const body = (await res.json()) as { data: Record }; + + // Every legacy key — the consumer (`/insights` page) reads each of + // these. If any one disappears the page silently breaks. + const requiredKeys = [ + "summaries", + "bmi", + "bmiClassification", + "bpClassification", + "bpPctInTarget", + "bpTargets", + "weightBpCorrelation", + "scatterData", + "bpMedicationCorrelation", + "bpMedicationScatterData", + "moodSummary", + "moodBpCorrelation", + "moodBpScatterData", + "moodWeightCorrelation", + "moodWeightScatterData", + "moodPulseCorrelation", + "moodPulseScatterData", + "medications", + "alerts", + "hasProvider", + "dataSpanDays", + "totalMeasurements", + ]; + for (const key of requiredKeys) { + expect(body.data).toHaveProperty(key); + } + + expect(body.data.bmi).toBeNull(); + expect(body.data.bmiClassification).toBeNull(); + expect(body.data.scatterData).toEqual([]); + expect(body.data.bpMedicationScatterData).toEqual([]); + expect(body.data.medications).toEqual([]); + expect(Array.isArray(body.data.alerts)).toBe(true); + expect(body.data.totalMeasurements).toBe(0); + expect(body.data.dataSpanDays).toBe(0); + }); + + it("computes BMI from aggregate WEIGHT.latest and user heightCm", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + ( + buildComprehensiveAggregate as ReturnType + ).mockResolvedValue({ + summaries: { + WEIGHT: { + count: 1, + latest: 81.0, + min: 81.0, + max: 81.0, + mean: 81.0, + avg7: 81.0, + avg30: 81.0, + slope7: null, + slope30: null, + slope90: null, + anomalyCount: 0, + avg30LastMonth: null, + avg30LastYear: null, + }, + }, + bpRawRows: { sys: [], dia: [] }, + dailyByType: {}, + firstMeasurementAt: new Date(), + totalMeasurements: 1, + }); + + const res = await callGet(makeReq()); + const body = (await res.json()) as { + data: { bmi: number | null; bmiClassification: { category: string } }; + }; + // BMI = 81 / (1.80 ^ 2) = 25.0 → "Overweight" in classifyBMI. + expect(body.data.bmi).toBeCloseTo(25, 1); + expect(body.data.bmiClassification.category).toBe("Overweight"); + }); + + it("pairs sys + dia raw rows with 5-min tolerance for bpPctInTarget", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + const at = new Date("2026-05-10T08:00:00Z"); + ( + buildComprehensiveAggregate as ReturnType + ).mockResolvedValue({ + summaries: { + BLOOD_PRESSURE_SYS: { + count: 1, + latest: 125, + min: 125, + max: 125, + mean: 125, + avg7: 125, + avg30: 125, + slope7: null, + slope30: null, + slope90: null, + anomalyCount: 0, + avg30LastMonth: null, + avg30LastYear: null, + }, + BLOOD_PRESSURE_DIA: { + count: 1, + latest: 75, + min: 75, + max: 75, + mean: 75, + avg7: 75, + avg30: 75, + slope7: null, + slope30: null, + slope90: null, + anomalyCount: 0, + avg30LastMonth: null, + avg30LastYear: null, + }, + }, + bpRawRows: { + sys: [{ measuredAt: at, value: 125 }], + dia: [{ measuredAt: at, value: 75 }], + }, + dailyByType: {}, + firstMeasurementAt: at, + totalMeasurements: 2, + }); + + const res = await callGet(makeReq()); + const body = (await res.json()) as { + data: { + bpPctInTarget: number | null; + bpClassification: { category: string } | null; + }; + }; + // 125/75 sits inside the under-65 target ceiling (sysHigh=129, + // diaHigh=79) and above the hypotension floor → 100% in target. + expect(body.data.bpPctInTarget).toBe(100); + // avg30 sys=125, dia=75 → ESH "Normal" band. + expect(body.data.bpClassification?.category).toBe("Normal"); + }); + + it("derives weight × BP scatter from daily-bucketed series", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + // Five day-aligned pairs (n >= 5 unlocks pearsonCorrelation). + const days = [ + "2026-05-01", + "2026-05-02", + "2026-05-03", + "2026-05-04", + "2026-05-05", + ]; + ( + buildComprehensiveAggregate as ReturnType + ).mockResolvedValue({ + summaries: {}, + bpRawRows: { sys: [], dia: [] }, + dailyByType: { + WEIGHT: days.map((d, i) => ({ day: d, value: 80 + i })), + BLOOD_PRESSURE_SYS: days.map((d, i) => ({ day: d, value: 120 + i * 2 })), + }, + firstMeasurementAt: new Date(), + totalMeasurements: 10, + }); + + const res = await callGet(makeReq()); + const body = (await res.json()) as { + data: { + scatterData: Array<{ weight: number; sysBP: number }>; + weightBpCorrelation: { r: number; n: number } | null; + }; + }; + expect(body.data.scatterData).toHaveLength(5); + expect(body.data.scatterData[0]).toEqual({ weight: 80, sysBP: 120 }); + // Perfect linear relationship → r === 1. + expect(body.data.weightBpCorrelation?.r).toBe(1); + expect(body.data.weightBpCorrelation?.n).toBe(5); + }); +}); diff --git a/src/app/api/insights/comprehensive/route.ts b/src/app/api/insights/comprehensive/route.ts index cd7e6d8cb..b8624c093 100644 --- a/src/app/api/insights/comprehensive/route.ts +++ b/src/app/api/insights/comprehensive/route.ts @@ -1,7 +1,8 @@ import { prisma } from "@/lib/db"; import { resolveProvider } from "@/lib/ai/provider"; import { apiSuccess } from "@/lib/api-response"; -import { summarize, type DataPoint } from "@/lib/analytics/trends"; +import type { DataPoint, DataSummary } from "@/lib/analytics/trends"; +import { summarize } from "@/lib/analytics/trends"; import { getBpTargets } from "@/lib/analytics/bp-targets"; import { isBpReadingInTarget } from "@/lib/analytics/bp-in-target"; import { @@ -16,11 +17,11 @@ import { } from "@/lib/analytics/correlations"; import { calculateCompliance } from "@/lib/analytics/compliance"; import { getMedicationCategories } from "@/lib/medication-category"; -import type { MeasurementType } from "@/generated/prisma/client"; -import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { apiHandler, requireAuth, type AuthContext } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; -import { measurementTypeEnum } from "@/lib/validations/measurement"; import { requireAssistantSurface } from "@/lib/feature-flags"; +import { cached, caches, type ServerCache } from "@/lib/cache/server-cache"; +import { buildComprehensiveAggregate } from "@/lib/insights/comprehensive-aggregator"; export const dynamic = "force-dynamic"; @@ -30,10 +31,41 @@ export const GET = apiHandler(async () => { // recommendations grid that share the Coach gate. await requireAssistantSurface("coach"); + // v1.4.35 — read-through the analytics cache keyed on + // (userId, "comprehensive"). The /insights page mount routinely + // fans out to this endpoint alongside the Coach drawer + the + // recommendations grid; the 60s TTL converts every duplicate + // mount within the window to a Map lookup. Invalidation is handled + // by `invalidateUserMeasurements` which already evicts every key + // under the `${userId}|` prefix (see `lib/cache/invalidate.ts`). + const body = await cached( + caches.analytics as ServerCache + >>, + `${user.id}|comprehensive`, + () => buildComprehensiveResponse(user), + annotate, + ); + + return apiSuccess(body); +}); + +type AuthedUser = AuthContext["user"]; + +/** + * v1.4.35 — comprehensive insights response body, lifted out of the + * route handler so `cached()` can wrap it. Replaces the previous + * 100k-row-in-JS aggregation with SQL-side aggregates served by + * `buildComprehensiveAggregate`. The medication-compliance block keeps + * its bounded Prisma reads (medications + intake events) — the directive + * called those out as already on the safe path. The byte-shape of the + * returned envelope is unchanged from the legacy route. + */ +export async function buildComprehensiveResponse(user: AuthedUser) { const userId = user.id; const ninetyDaysAgo = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); - // Fetch user profile + // Fetch user profile (height + DOB drive BMI + BP targets). const dbUser = await prisma.user.findUnique({ where: { id: userId }, select: { @@ -42,24 +74,26 @@ export const GET = apiHandler(async () => { }, }); - // Derived from canonical enum so adding a new measurement type does not - // require touching this file (V3 audit finding: enum drift cousins). - const types = [...measurementTypeEnum.options] as MeasurementType[]; + // ── Aggregate-first reads ───────────────────────────────── + // SQL-side rollup replaces the 100k-row findMany on the legacy + // route. Returns per-type DataSummary identical to the JS + // `summarize()` semantics over the 90-day window, plus the bounded + // raw rows the BP target adherence pairing needs, plus daily-mean + // buckets for the correlation pairings. + const aggregate = await buildComprehensiveAggregate(userId); - const allMeasurements = await prisma.measurement.findMany({ - where: { userId, type: { in: types }, measuredAt: { gte: ninetyDaysAgo } }, - orderBy: { measuredAt: "asc" }, - select: { type: true, value: true, measuredAt: true }, - }); - - // Fetch mood entries (90 days) + // Mood entries are user-recorded; the table is small and the + // `moodSummary` field requires raw scores (`summarize()` over the + // raw score series). The findMany stays. const moodEntries = await prisma.moodEntry.findMany({ where: { userId, moodLoggedAt: { gte: ninetyDaysAgo } }, orderBy: { moodLoggedAt: "asc" }, select: { date: true, score: true, moodLoggedAt: true }, }); - // Aggregate mood to daily averages + // Aggregate mood to daily averages (drives the mood × metric + // correlations). Bucket key matches the legacy code's `date` column + // semantics (YYYY-MM-DD anchored to the user's tz at write time). const moodByDay = new Map(); for (const entry of moodEntries) { const current = moodByDay.get(entry.date) ?? { sum: 0, count: 0 }; @@ -74,7 +108,9 @@ export const GET = apiHandler(async () => { value: Math.round((stats.sum / stats.count) * 100) / 100, })); - // Build DataPoint array for mood summary + // moodSummary uses the same `summarize()` helper over raw scores — + // the table is bounded by user-recorded entries so the in-JS path + // is safe. const moodDataPoints: DataPoint[] = moodEntries.map((e) => ({ date: e.moodLoggedAt, value: e.score, @@ -82,21 +118,9 @@ export const GET = apiHandler(async () => { const moodSummary = moodDataPoints.length > 0 ? summarize(moodDataPoints) : null; - const byType = (t: MeasurementType): DataPoint[] => - allMeasurements - .filter((m) => m.type === t) - .map((m) => ({ date: m.measuredAt, value: m.value })); + const summaries: Record = aggregate.summaries; - // Summaries - const summaries: Record> = {}; - for (const t of types) { - const data = byType(t); - if (data.length > 0) { - summaries[t] = summarize(data); - } - } - - // BMI + // ── BMI ────────────────────────────────────────────────── let bmi: number | null = null; let bmiClassification = null; if (dbUser?.heightCm && summaries.WEIGHT?.latest) { @@ -105,7 +129,7 @@ export const GET = apiHandler(async () => { bmiClassification = classifyBMI(bmi); } - // BP classification (30-day average) + // ── BP classification (30-day average) ──────────────────── let bpClassification = null; const bpTargets = getBpTargets(dbUser?.dateOfBirth ?? null); if ( @@ -118,11 +142,21 @@ export const GET = apiHandler(async () => { ); } - // BP target adherence + // ── BP target adherence ────────────────────────────────── + // Keeps the 5-minute tolerance contract — sys + dia must pair on + // the same reading session. The aggregator pulled the bounded raw + // rows; we reuse `pairByTimestamp` against them so the output is + // byte-identical to the legacy path. let bpPctInTarget: number | null = null; if (bpTargets) { - const sysData = byType("BLOOD_PRESSURE_SYS"); - const diaData = byType("BLOOD_PRESSURE_DIA"); + const sysData: DataPoint[] = aggregate.bpRawRows.sys.map((r) => ({ + date: r.measuredAt, + value: r.value, + })); + const diaData: DataPoint[] = aggregate.bpRawRows.dia.map((r) => ({ + date: r.measuredAt, + value: r.value, + })); const sysPairs = pairByTimestamp(sysData, diaData, 5 * 60 * 1000); if (sysPairs.length > 0) { @@ -135,82 +169,48 @@ export const GET = apiHandler(async () => { } } - // Correlations: Weight vs Sys BP - const weightData = byType("WEIGHT"); - const sysData = byType("BLOOD_PRESSURE_SYS"); - const weightBpPairs = pairByTimestamp(weightData, sysData); + // ── Correlations: weight × Sys BP ───────────────────────── + // Daily-key pairing (the legacy `pairByTimestamp` ran with a + // 24-hour default tolerance — daily-mean buckets are the + // SQL-side equivalent and the directive explicitly accepts the + // small semantic shift to bound the raw-row footprint). + const weightDaily = aggregate.dailyByType.WEIGHT ?? []; + const sysDaily = aggregate.dailyByType.BLOOD_PRESSURE_SYS ?? []; + const weightBpPairs: PairedPoint[] = joinDailyByDay( + weightDaily, + sysDaily, + ); const weightBpCorrelation = pearsonCorrelation(weightBpPairs); const scatterData = weightBpPairs.map((p) => ({ weight: p.a, sysBP: p.b, })); - // Correlations: Mood vs metrics (using daily averages matched by date) - function buildMoodMetricPairs( - dailyMood: Array<{ day: string; value: number }>, - measurements: Array<{ measuredAt: Date; value: number }>, - ): PairedPoint[] { - // Group measurements by day - const metricByDay = new Map(); - for (const m of measurements) { - const dayKey = m.measuredAt.toISOString().slice(0, 10); - const current = metricByDay.get(dayKey) ?? { sum: 0, count: 0 }; - current.sum += m.value; - current.count += 1; - metricByDay.set(dayKey, current); - } - - const pairs: PairedPoint[] = []; - for (const mood of dailyMood) { - const metric = metricByDay.get(mood.day); - if (metric) { - pairs.push({ - a: mood.value, - b: Math.round((metric.sum / metric.count) * 100) / 100, - date: new Date(`${mood.day}T12:00:00.000Z`), - }); - } - } - return pairs; - } - - // Mood vs Systolic BP - const moodBpPairs = buildMoodMetricPairs( - dailyMoodEntries, - allMeasurements - .filter((m) => m.type === "BLOOD_PRESSURE_SYS") - .map((m) => ({ measuredAt: m.measuredAt, value: m.value })), - ); + // ── Correlations: mood × {sys BP, weight, pulse} ───────── + // Daily-key matches the legacy `buildMoodMetricPairs` exactly — + // mood is already per-day in the legacy code and we just align the + // metric side to the same daily bucket. + const moodBpPairs = pairMoodWithDaily(dailyMoodEntries, sysDaily); const moodBpCorrelation = pearsonCorrelation(moodBpPairs); const moodBpScatterData = moodBpPairs.map((p) => ({ mood: p.a, sysBP: p.b })); - // Mood vs Weight - const moodWeightPairs = buildMoodMetricPairs( - dailyMoodEntries, - allMeasurements - .filter((m) => m.type === "WEIGHT") - .map((m) => ({ measuredAt: m.measuredAt, value: m.value })), - ); + const moodWeightPairs = pairMoodWithDaily(dailyMoodEntries, weightDaily); const moodWeightCorrelation = pearsonCorrelation(moodWeightPairs); const moodWeightScatterData = moodWeightPairs.map((p) => ({ mood: p.a, weight: p.b, })); - // Mood vs Pulse - const moodPulsePairs = buildMoodMetricPairs( - dailyMoodEntries, - allMeasurements - .filter((m) => m.type === "PULSE") - .map((m) => ({ measuredAt: m.measuredAt, value: m.value })), - ); + const pulseDaily = aggregate.dailyByType.PULSE ?? []; + const moodPulsePairs = pairMoodWithDaily(dailyMoodEntries, pulseDaily); const moodPulseCorrelation = pearsonCorrelation(moodPulsePairs); const moodPulseScatterData = moodPulsePairs.map((p) => ({ mood: p.a, pulse: p.b, })); - // Medication compliance + // ── Medication compliance ──────────────────────────────── + // Bounded reads — left untouched per the directive. const medications = await prisma.medication.findMany({ where: { userId, active: true }, include: { schedules: true }, @@ -229,9 +229,7 @@ export const GET = apiHandler(async () => { (med) => (categoryMap[med.id] ?? "OTHER") === "BLOOD_PRESSURE", ); // Single round-trip for all medications instead of N+1: one query keyed - // on `medicationId IN (...)`, then group in memory. The previous loop - // hit Postgres once per medication, which scales poorly for users with - // many active meds. + // on `medicationId IN (...)`, then group in memory. const allEvents = medications.length ? await prisma.medicationIntakeEvent.findMany({ where: { @@ -284,7 +282,11 @@ export const GET = apiHandler(async () => { } } - // Correlation: continuity of BP medications vs systolic BP + // ── Correlation: BP medication continuity × systolic BP ── + // Uses the same daily BP_SYS series the aggregator pre-computed — + // no second round-trip. The intake-side aggregation matches the + // legacy code: continuity = min(1, taken / expectedPerDay), pairing + // every day that had a sys reading. let bpMedicationCorrelation: { r: number; strength: string; @@ -302,13 +304,10 @@ export const GET = apiHandler(async () => { ); if (expectedBpIntakesPerDay > 0) { - const sysByDay = new Map(); - for (const m of allMeasurements) { - if (m.type !== "BLOOD_PRESSURE_SYS") continue; - const dayKey = m.measuredAt.toISOString().slice(0, 10); - const list = sysByDay.get(dayKey) ?? []; - list.push(m.value); - sysByDay.set(dayKey, list); + // Daily systolic means (already keyed YYYY-MM-DD) from the SQL pass. + const sysByDay = new Map(); + for (const row of sysDaily) { + sysByDay.set(row.day, row.value); } const takenByDay = new Map(); @@ -319,11 +318,9 @@ export const GET = apiHandler(async () => { } const pairs: Array<{ a: number; b: number; date: Date }> = []; - for (const [dayKey, sysValues] of sysByDay.entries()) { + for (const [dayKey, avgSys] of sysByDay.entries()) { const taken = takenByDay.get(dayKey) ?? 0; const continuity = Math.min(1, taken / expectedBpIntakesPerDay); - const avgSys = - sysValues.reduce((sum, value) => sum + value, 0) / sysValues.length; pairs.push({ a: continuity, b: avgSys, @@ -360,25 +357,24 @@ export const GET = apiHandler(async () => { })), }); - // Data span - const firstMeasurement = - allMeasurements.length > 0 ? allMeasurements[0].measuredAt : null; - const dataSpanDays = firstMeasurement + // Data span — derived from the earliest 90-day measurement. + const dataSpanDays = aggregate.firstMeasurementAt ? Math.ceil( - (Date.now() - firstMeasurement.getTime()) / (24 * 60 * 60 * 1000), + (Date.now() - aggregate.firstMeasurementAt.getTime()) / + (24 * 60 * 60 * 1000), ) : 0; annotate({ action: { name: "insights.comprehensive" }, meta: { - totalMeasurements: allMeasurements.length, + totalMeasurements: aggregate.totalMeasurements, moodEntries: moodEntries.length, medications: medications.length, }, }); - return apiSuccess({ + return { summaries, bmi, bmiClassification, @@ -400,6 +396,75 @@ export const GET = apiHandler(async () => { alerts, hasProvider: (await resolveProvider(userId)).type !== "none", dataSpanDays, - totalMeasurements: allMeasurements.length, - }); -}); + totalMeasurements: aggregate.totalMeasurements, + }; +} + +// ───────────────────────────────────────────────────────────── +// Helpers +// ───────────────────────────────────────────────────────────── + +/** + * Daily-key inner join between two daily-aggregated series. Returns + * `PairedPoint` shapes compatible with `pearsonCorrelation`. + * + * Inputs are sorted ASC by `day` (the aggregator's ORDER BY + * guarantees that). The walk is O(n + m) merge-join style instead of + * Map.lookup-per-element so it scales with the worst-case 90 buckets + * x 90 buckets case without surprises. + */ +function joinDailyByDay( + a: Array<{ day: string; value: number }>, + b: Array<{ day: string; value: number }>, +): PairedPoint[] { + const pairs: PairedPoint[] = []; + let i = 0; + let j = 0; + while (i < a.length && j < b.length) { + const ai = a[i]; + const bj = b[j]; + if (ai.day === bj.day) { + pairs.push({ + a: ai.value, + b: bj.value, + date: new Date(`${ai.day}T12:00:00.000Z`), + }); + i += 1; + j += 1; + } else if (ai.day < bj.day) { + i += 1; + } else { + j += 1; + } + } + return pairs; +} + +/** + * Pair the daily mood series (left side) with a daily-aggregated + * metric (right side). Matches the legacy `buildMoodMetricPairs` + * helper byte-for-byte: same daily YYYY-MM-DD key, same + * `T12:00:00.000Z` anchor for the `date` field. + */ +function pairMoodWithDaily( + dailyMood: Array<{ day: string; value: number }>, + dailyMetric: Array<{ day: string; value: number }>, +): PairedPoint[] { + if (dailyMood.length === 0 || dailyMetric.length === 0) return []; + const metricByDay = new Map(); + for (const m of dailyMetric) { + metricByDay.set(m.day, m.value); + } + const pairs: PairedPoint[] = []; + for (const mood of dailyMood) { + const metric = metricByDay.get(mood.day); + if (metric !== undefined) { + pairs.push({ + a: mood.value, + b: metric, + date: new Date(`${mood.day}T12:00:00.000Z`), + }); + } + } + return pairs; +} diff --git a/src/app/api/medications/route.ts b/src/app/api/medications/route.ts index f002a412b..eb18ad5bf 100644 --- a/src/app/api/medications/route.ts +++ b/src/app/api/medications/route.ts @@ -16,40 +16,34 @@ import { import { serializeScheduleRecurrence } from "@/lib/medication-schedule"; import { getUserTodayBounds } from "@/lib/timezone"; import { invalidateUserMedications } from "@/lib/cache/invalidate"; +import { cached, caches, type ServerCache } from "@/lib/cache/server-cache"; import { NextRequest } from "next/server"; -export const GET = apiHandler(async () => { - const { user } = await requireAuth(); +type MedicationsListResult = Array>; - // Compute today's UTC range for event counting - const userTz = user.timezone || "Europe/Berlin"; +async function buildMedicationsList( + userId: string, + userTz: string, +): Promise { const { start: todayStartUtc, end: todayEndUtc } = getUserTodayBounds( new Date(), userTz, ); - // Run all three queries in parallel const [medications, latestIntakes, todayEvents] = await Promise.all([ prisma.medication.findMany({ - where: { userId: user.id }, + where: { userId }, include: { schedules: true }, orderBy: { createdAt: "desc" }, }), prisma.medicationIntakeEvent.groupBy({ by: ["medicationId"], - where: { - userId: user.id, - skipped: false, - takenAt: { not: null }, - }, + where: { userId, skipped: false, takenAt: { not: null } }, _max: { takenAt: true }, }), prisma.medicationIntakeEvent.groupBy({ by: ["medicationId"], - where: { - userId: user.id, - scheduledFor: { gte: todayStartUtc, lte: todayEndUtc }, - }, + where: { userId, scheduledFor: { gte: todayStartUtc, lte: todayEndUtc } }, _count: { id: true }, }), ]); @@ -76,24 +70,36 @@ export const GET = apiHandler(async () => { getEvent()?.addWarning("Medication categories could not be loaded"); } + return medications.map((m) => ({ + ...m, + category: categoryMap[m.id] ?? "OTHER", + lastTakenAt: lastTakenAtByMedicationId[m.id] ?? null, + todayEventCount: todayEventCountByMedId[m.id] ?? 0, + })); +} + +export const GET = apiHandler(async () => { + const { user } = await requireAuth(); + + const userTz = user.timezone || "Europe/Berlin"; + + // Cache the list shape on userId. The 60s TTL bounds the cross-midnight + // staleness window for `todayEventCount` (the only time-of-day-derived + // field in the response); writes via POST/PUT/DELETE flush through + // `invalidateUserMedications` before the next read lands. + const result = await cached( + caches.medications as ServerCache, + user.id, + () => buildMedicationsList(user.id, userTz), + annotate, + ); + annotate({ action: { name: "medication.list" }, - meta: { count: medications.length }, + meta: { count: result.length }, }); - return apiSuccess( - medications.map((m) => ({ - ...m, - // v1.4.25 W4d — `category` is the existing clinical taxonomy - // (BLOOD_PRESSURE / VITAMIN / ...) from the side-table; the - // Prisma model has its own `treatmentClass` (GENERIC | GLP1) that - // spreads through `...m` and unlocks the GLP-1 surfaces. Two - // orthogonal concepts, both exposed. - category: categoryMap[m.id] ?? "OTHER", - lastTakenAt: lastTakenAtByMedicationId[m.id] ?? null, - todayEventCount: todayEventCountByMedId[m.id] ?? 0, - })), - ); + return apiSuccess(result); }); export const POST = apiHandler(async (request: NextRequest) => { diff --git a/src/app/api/mood/analytics/route.ts b/src/app/api/mood/analytics/route.ts index 6be9db8f2..1002560a3 100644 --- a/src/app/api/mood/analytics/route.ts +++ b/src/app/api/mood/analytics/route.ts @@ -3,6 +3,7 @@ import { apiSuccess } from "@/lib/api-response"; import { summarize, type DataPoint } from "@/lib/analytics/trends"; import { apiHandler, requireAuth } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; +import { cached, caches, type ServerCache } from "@/lib/cache/server-cache"; export const dynamic = "force-dynamic"; @@ -26,11 +27,17 @@ function aggregateDailyAverages( })); } -export const GET = apiHandler(async () => { - const { user } = await requireAuth(); +interface MoodAnalyticsResult { + entries: Array<{ date: string; score: number; samples: number }>; + summary: ReturnType; + entryCount: number; +} +async function buildMoodAnalyticsResponse( + userId: string, +): Promise { const moodEntries = await prisma.moodEntry.findMany({ - where: { userId: user.id }, + where: { userId }, orderBy: { moodLoggedAt: "asc" }, select: { date: true, score: true, moodLoggedAt: true }, }); @@ -39,7 +46,6 @@ export const GET = apiHandler(async () => { moodEntries.map((e) => ({ date: e.date, score: e.score })), ); - // Build DataPoint array for summarize() using moodLoggedAt as the date const dataPoints: DataPoint[] = moodEntries.map((e) => ({ date: e.moodLoggedAt, value: e.score, @@ -47,10 +53,23 @@ export const GET = apiHandler(async () => { const summary = summarize(dataPoints); + return { entries, summary, entryCount: moodEntries.length }; +} + +export const GET = apiHandler(async () => { + const { user } = await requireAuth(); + + const result = await cached( + caches.moodAnalytics as ServerCache, + user.id, + () => buildMoodAnalyticsResponse(user.id), + annotate, + ); + annotate({ action: { name: "mood.analytics" }, - meta: { entryCount: moodEntries.length }, + meta: { entryCount: result.entryCount }, }); - return apiSuccess({ entries, summary }); + return apiSuccess({ entries: result.entries, summary: result.summary }); }); diff --git a/src/app/api/workouts/__tests__/canonical-dedup.test.ts b/src/app/api/workouts/__tests__/canonical-dedup.test.ts index b220c5f02..c24ad2064 100644 --- a/src/app/api/workouts/__tests__/canonical-dedup.test.ts +++ b/src/app/api/workouts/__tests__/canonical-dedup.test.ts @@ -16,6 +16,8 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; +import { __resetAllCachesForTests } from "@/lib/cache/server-cache"; + vi.mock("@/lib/db", () => ({ prisma: { workout: { @@ -100,6 +102,11 @@ function makeRow( describe("GET /api/workouts — canonical dedup", () => { beforeEach(() => { vi.clearAllMocks(); + // Reset every server cache between tests so a fixed userId doesn't + // carry cached state across cases (v1.4.34.2 — added when the + // `/api/workouts` GET handler started reading through + // `caches.workouts`). + __resetAllCachesForTests(); vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); // Default: no per-user source-priority override → fall back to the // canonical APPLE_HEALTH ≻ WITHINGS ≻ MANUAL ≻ IMPORT ladder. diff --git a/src/app/api/workouts/route.ts b/src/app/api/workouts/route.ts index b3a965b2c..4fe739e6a 100644 --- a/src/app/api/workouts/route.ts +++ b/src/app/api/workouts/route.ts @@ -54,10 +54,119 @@ import { apiHandler, requireAuth } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; import { apiSuccess } from "@/lib/api-response"; import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-workout-rows"; +import { cached, caches, type ServerCache } from "@/lib/cache/server-cache"; const DEFAULT_LIMIT = 50; const MAX_LIMIT = 200; +interface WorkoutsParams { + readonly limit: number; + readonly offset: number; + readonly since: string | null; + readonly until: string | null; + readonly sportType: string | null; +} + +interface WorkoutsResult { + workouts: Array<{ + id: string; + sportType: string | null; + startedAt: Date | null; + endedAt: Date | null; + durationSec: number | null; + distanceM: number | null; + activeEnergyKcal: number | null; + avgHr: number | null; + maxHr: number | null; + source: string; + externalId: string | null; + }>; + meta: { + total: number; + limit: number; + offset: number; + droppedDuplicates: number; + }; +} + +async function buildWorkoutsResponse( + userId: string, + params: WorkoutsParams, +): Promise { + const { limit, offset, since, until, sportType } = params; + + const where: Record = { userId }; + if (since || until) { + const range: Record = {}; + if (since) { + const d = new Date(since); + if (!Number.isNaN(d.getTime())) range.gte = d; + } + if (until) { + const d = new Date(until); + if (!Number.isNaN(d.getTime())) range.lte = d; + } + if (Object.keys(range).length > 0) { + where.startedAt = range; + } + } + if (sportType) { + where.sportType = sportType; + } + + const rows = await prisma.workout.findMany({ + where, + orderBy: [{ startedAt: "desc" }, { id: "asc" }], + select: { + id: true, + source: true, + externalId: true, + sportType: true, + startedAt: true, + endedAt: true, + durationSec: true, + totalDistanceM: true, + totalEnergyKcal: true, + avgHeartRate: true, + maxHeartRate: true, + createdAt: true, + }, + }); + + const userRow = await prisma.user.findUnique({ + where: { id: userId }, + select: { sourcePriorityJson: true }, + }); + const canonical = pickCanonicalWorkoutRows( + rows, + userRow?.sourcePriorityJson ?? null, + ); + + const page = canonical.slice(offset, offset + limit).map((row) => ({ + id: row.id, + sportType: row.sportType, + startedAt: row.startedAt, + endedAt: row.endedAt, + durationSec: row.durationSec, + distanceM: row.totalDistanceM, + activeEnergyKcal: row.totalEnergyKcal, + avgHr: row.avgHeartRate, + maxHr: row.maxHeartRate, + source: row.source, + externalId: row.externalId, + })); + + return { + workouts: page, + meta: { + total: canonical.length, + limit, + offset, + droppedDuplicates: rows.length - canonical.length, + }, + }; +} + export const GET = apiHandler(async (request: NextRequest) => { const { user } = await requireAuth(); annotate({ action: { name: "workouts.list" } }); @@ -79,80 +188,15 @@ export const GET = apiHandler(async (request: NextRequest) => { const until = searchParams.get("until"); const sportType = searchParams.get("sportType"); - const where: Record = { userId: user.id }; - if (since || until) { - const range: Record = {}; - if (since) { - const d = new Date(since); - if (!Number.isNaN(d.getTime())) range.gte = d; - } - if (until) { - const d = new Date(until); - if (!Number.isNaN(d.getTime())) range.lte = d; - } - if (Object.keys(range).length > 0) { - where.startedAt = range; - } - } - if (sportType) { - where.sportType = sportType; - } + const params: WorkoutsParams = { limit, offset, since, until, sportType }; + const cacheKey = `${user.id}|${limit}|${offset}|${since ?? ""}|${until ?? ""}|${sportType ?? ""}`; - // Read the full filtered set, descending start. The picker requires - // deterministic input order; the `(startedAt, id)` secondary sort - // pins the tie-break for rows that share a millisecond stamp. - const rows = await prisma.workout.findMany({ - where, - orderBy: [{ startedAt: "desc" }, { id: "asc" }], - select: { - id: true, - source: true, - externalId: true, - sportType: true, - startedAt: true, - endedAt: true, - durationSec: true, - totalDistanceM: true, - totalEnergyKcal: true, - avgHeartRate: true, - maxHeartRate: true, - createdAt: true, - }, - }); - - // Pass the user's source-priority blob so the picker honours the - // per-user ladder. The picker preserves input order on the returned - // subset; descending-start input → descending-start output. - const userRow = await prisma.user.findUnique({ - where: { id: user.id }, - select: { sourcePriorityJson: true }, - }); - const canonical = pickCanonicalWorkoutRows( - rows, - userRow?.sourcePriorityJson ?? null, + const result = await cached( + caches.workouts as ServerCache, + cacheKey, + () => buildWorkoutsResponse(user.id, params), + annotate, ); - const page = canonical.slice(offset, offset + limit).map((row) => ({ - id: row.id, - sportType: row.sportType, - startedAt: row.startedAt, - endedAt: row.endedAt, - durationSec: row.durationSec, - distanceM: row.totalDistanceM, - activeEnergyKcal: row.totalEnergyKcal, - avgHr: row.avgHeartRate, - maxHr: row.maxHeartRate, - source: row.source, - externalId: row.externalId, - })); - - return apiSuccess({ - workouts: page, - meta: { - total: canonical.length, - limit, - offset, - droppedDuplicates: rows.length - canonical.length, - }, - }); + return apiSuccess(result); }); diff --git a/src/components/charts/scatter-correlation-chart.tsx b/src/components/charts/scatter-correlation-chart.tsx index 1183d9ab0..ac2bb5fe0 100644 --- a/src/components/charts/scatter-correlation-chart.tsx +++ b/src/components/charts/scatter-correlation-chart.tsx @@ -91,7 +91,13 @@ export function ScatterCorrelationChart>({ // approximates the previous 250 px shape. return (
diff --git a/src/lib/insights/__tests__/comprehensive-aggregator.test.ts b/src/lib/insights/__tests__/comprehensive-aggregator.test.ts new file mode 100644 index 000000000..09c2a6948 --- /dev/null +++ b/src/lib/insights/__tests__/comprehensive-aggregator.test.ts @@ -0,0 +1,175 @@ +/** + * v1.4.35 — unit pin for `buildComprehensiveAggregate`. + * + * Heavier integration coverage (real Postgres, real `regr_slope`, + * cache hit/miss assertions) lives in + * `tests/integration/insights-comprehensive-cache.test.ts`. This file + * mocks `$queryRaw` and Prisma's `measurement.findMany` so the shape + + * rounding + slope-direction contracts are pinned without a container. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/db", () => ({ + prisma: { + $queryRaw: vi.fn(), + measurement: { findMany: vi.fn() }, + }, +})); + +vi.mock("@/lib/logging/context", () => ({ + annotate: vi.fn(), +})); + +import { prisma } from "@/lib/db"; +import { buildComprehensiveAggregate } from "../comprehensive-aggregator"; + +const RAW = prisma.$queryRaw as unknown as ReturnType; +const FIND_MANY = + prisma.measurement.findMany as unknown as ReturnType; + +beforeEach(() => { + RAW.mockReset(); + FIND_MANY.mockReset(); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("buildComprehensiveAggregate", () => { + it("returns an empty bundle for a user with no measurements", async () => { + // Pass order: aggregates, latests, daily, sysRaw, diaRaw. The + // `firstMeasurementAt` query is skipped when totalMeasurements === 0. + RAW.mockResolvedValueOnce([]) // aggregates + .mockResolvedValueOnce([]) // latests + .mockResolvedValueOnce([]); // daily + FIND_MANY.mockResolvedValueOnce([]) // sys + .mockResolvedValueOnce([]); // dia + + const result = await buildComprehensiveAggregate("user-empty"); + + expect(result.summaries).toEqual({}); + expect(result.bpRawRows.sys).toEqual([]); + expect(result.bpRawRows.dia).toEqual([]); + expect(result.dailyByType).toEqual({}); + expect(result.firstMeasurementAt).toBeNull(); + expect(result.totalMeasurements).toBe(0); + // 3 $queryRaw calls (aggregates, latests, daily) + 2 findMany. + // No firstMeasurementAt query when total === 0. + expect(RAW).toHaveBeenCalledTimes(3); + expect(FIND_MANY).toHaveBeenCalledTimes(2); + }); + + it("maps a populated WEIGHT aggregate into the DataSummary shape", async () => { + const now = new Date(); + RAW.mockResolvedValueOnce([ + { + type: "WEIGHT", + count: BigInt(42), + min_value: 79.2, + max_value: 84.1, + mean_value: 82.05, + stddev_value: 1.2, + anomaly_count: BigInt(3), + avg7: 81.9, + avg30: 82.1, + avg30_last_month: 83.0, + slope7: -0.014, + r2_7: 0.65, + slope30: -0.005, + r2_30: 0.42, + slope90: 0.001, + r2_90: 0.12, + }, + ]) + .mockResolvedValueOnce([{ type: "WEIGHT", value: 81.4, measured_at: now }]) + .mockResolvedValueOnce([ + { type: "WEIGHT", day: "2026-05-10", mean_value: 82.0 }, + { type: "WEIGHT", day: "2026-05-11", mean_value: 82.2 }, + ]) + .mockResolvedValueOnce([{ first_at: new Date(now.getTime() - 86400000) }]); + FIND_MANY.mockResolvedValueOnce([]) // sys + .mockResolvedValueOnce([]); // dia + + const result = await buildComprehensiveAggregate("user-pop"); + const weight = result.summaries.WEIGHT; + + expect(weight.count).toBe(42); + expect(weight.latest).toBe(81.4); + expect(weight.min).toBe(79.2); + expect(weight.max).toBe(84.1); + expect(weight.mean).toBe(82.05); + expect(weight.avg7).toBe(81.9); + expect(weight.avg30).toBe(82.1); + expect(weight.avg30LastMonth).toBe(83.0); + // Always null in the 90-day window — preserved legacy semantics. + expect(weight.avg30LastYear).toBeNull(); + expect(weight.anomalyCount).toBe(3); + // Slope direction matches the JS threshold rules: + // -0.014 → "down" (|s| >= 0.01) + // -0.005 → "stable" (|s| < 0.01) + // 0.001 → "stable" + expect(weight.slope7).toEqual({ + slope: -0.014, + direction: "down", + confidence: 0.65, + }); + expect(weight.slope30).toEqual({ + slope: -0.005, + direction: "stable", + confidence: 0.42, + }); + expect(weight.slope90).toEqual({ + slope: 0.001, + direction: "stable", + confidence: 0.12, + }); + + expect(result.totalMeasurements).toBe(42); + expect(result.firstMeasurementAt).toBeInstanceOf(Date); + expect(result.dailyByType.WEIGHT).toEqual([ + { day: "2026-05-10", value: 82.0 }, + { day: "2026-05-11", value: 82.2 }, + ]); + }); + + it("threads sys/dia raw rows through bpRawRows so 5-min pairing survives", async () => { + const measuredAt = new Date("2026-05-10T08:00:00Z"); + RAW.mockResolvedValueOnce([ + { + type: "BLOOD_PRESSURE_SYS", + count: BigInt(1), + min_value: 120, + max_value: 120, + mean_value: 120, + stddev_value: 0, + anomaly_count: BigInt(0), + avg7: 120, + avg30: 120, + avg30_last_month: null, + slope7: null, + r2_7: null, + slope30: null, + r2_30: null, + slope90: null, + r2_90: null, + }, + ]) + .mockResolvedValueOnce([ + { + type: "BLOOD_PRESSURE_SYS", + value: 120, + measured_at: measuredAt, + }, + ]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ first_at: measuredAt }]); + FIND_MANY.mockResolvedValueOnce([ + { measuredAt, value: 120 }, + ]).mockResolvedValueOnce([{ measuredAt, value: 80 }]); + + const result = await buildComprehensiveAggregate("user-bp"); + expect(result.bpRawRows.sys).toEqual([{ measuredAt, value: 120 }]); + expect(result.bpRawRows.dia).toEqual([{ measuredAt, value: 80 }]); + }); +}); diff --git a/src/lib/insights/comprehensive-aggregator.ts b/src/lib/insights/comprehensive-aggregator.ts new file mode 100644 index 000000000..f9ccb9d66 --- /dev/null +++ b/src/lib/insights/comprehensive-aggregator.ts @@ -0,0 +1,373 @@ +/** + * v1.4.35 — SQL-driven aggregator for `/api/insights/comprehensive`. + * + * Background + * ---------- + * The route previously pulled every measurement of every type from the + * trailing 90 days into JS, then ran `summarize()` per type plus a + * pile of in-JS correlation pairings. Post Apple-Health import the + * unbounded `findMany` on line 49 of the old route routinely returned + * 100 000+ rows and dominated request time (29 s in the v1.4.34 prod + * HAR capture, with sibling endpoints cascading into 503s on pool + * starvation). + * + * The new shape mirrors the `summaries-slice` aggregator + * (`src/lib/analytics/summaries-slice.ts`) the v1.4.33 C1 dashboard + * tile strip already proved out: a single `$queryRaw` per surface, + * grouped by `type`, with Postgres-side `AVG` / `MIN` / `MAX` / + * `REGR_SLOPE` / `REGR_R2` against the index path + * `measurements (user_id, type, measured_at)`. Difference vs. the + * slim slice: the comprehensive endpoint also needs `anomalyCount`, + * `avg30LastMonth`, `avg30LastYear`, and the 90-day window cap. + * + * Scope notes + * ----------- + * - The 90-day window is preserved (the legacy `findMany` clamped on + * `measuredAt >= ninetyDaysAgo`). `min` / `max` / `mean` therefore + * describe the 90-day window, NOT the user's all-time range — + * identical to the old behaviour. + * + * - For correlations: + * - **weight × BP**: directive accepts a daily-key swap from the + * old `pairByTimestamp` (24h default tolerance). Postgres + * joins on `date_trunc('day', measured_at)`. + * - **mood × {BP, weight, pulse}**: same daily-key shape the + * legacy code already used (the `buildMoodMetricPairs` helper + * bucketed by `toISOString().slice(0,10)`). + * + * - BP target adherence (`bpPctInTarget`) keeps the 5-minute + * tolerance contract — that pairing is sys/dia within the *same* + * reading session. The aggregator pulls sys+dia raw rows (90d, + * bounded — typically <2k rows even for power users) so + * `pairByTimestamp` semantics survive byte-identical. + * + * - `bpMedicationCorrelation` is computed in the route itself; the + * aggregator surfaces daily BP_SYS means via the same daily-join + * query so the medication block can pair compliance × systolic + * without a second findMany. + * + * - Mood data still loads via `prisma.moodEntry.findMany` — that + * table is small (a row per user-recorded entry, bounded) and the + * `moodSummary` field calls `summarize()` against raw scores. The + * legacy route's `moodSummary` was built off raw rows so we keep + * the raw fetch. + * + * Output shape + * ------------ + * The aggregator returns the bundle of pre-computed values the route + * stitches into the final response. Helper functions inside the + * aggregator collapse `$queryRaw` rows into the same `DataSummary` + * shape `summarize()` would return. + */ +import { prisma } from "@/lib/db"; +import type { DataSummary } from "@/lib/analytics/trends"; + +/** + * Raw row from the aggregate query. Keyed on `MeasurementType`, with + * windowed AVG/SLOPE/R2 expressions plus anomaly-detection inputs + * (window-wide mean + stddev so we can compute the |z| > 2 count + * client-side without a second pass over the raw rows). + */ +interface AggregateRow { + type: string; + count: bigint; + min_value: number | null; + max_value: number | null; + mean_value: number | null; + stddev_value: number | null; + anomaly_count: bigint; + avg7: number | null; + avg30: number | null; + avg30_last_month: number | null; + slope7: number | null; + r2_7: number | null; + slope30: number | null; + r2_30: number | null; + slope90: number | null; + r2_90: number | null; +} + +/** Latest raw value per type (DISTINCT ON over the 90-day window). */ +interface LatestRow { + type: string; + value: number; + measured_at: Date; +} + +/** Daily-mean per (type, day) for correlation pairings. */ +export interface DailyAggregateRow { + type: string; + day: string; + mean_value: number; +} + +/** + * Result bundle. Everything the route handler needs to assemble the + * comprehensive response without holding the 100k-row raw measurement + * array in memory. + */ +export interface ComprehensiveAggregate { + /** Per-type DataSummary, 90-day window. Only types with rows are + * populated — matches the legacy route's behaviour (only types + * with rows landed on `summaries[t]`). */ + summaries: Record; + /** Sys/dia raw rows for the BP target adherence path. Bounded: + * typically <2k rows per user over 90 days even with Apple Health. */ + bpRawRows: { + sys: Array<{ measuredAt: Date; value: number }>; + dia: Array<{ measuredAt: Date; value: number }>; + }; + /** Daily means per (type, day) for the correlation pairings. The + * route filters down to the types it needs (WEIGHT, BLOOD_PRESSURE_SYS, + * PULSE) when joining against mood / weight / bp. */ + dailyByType: Record>; + /** Earliest measurement timestamp across the 90-day window — feeds + * the route's `dataSpanDays` field. NULL when the user has no rows. */ + firstMeasurementAt: Date | null; + /** Total row count across all types in the 90-day window — feeds the + * wide-event meta + the response's `totalMeasurements` field. */ + totalMeasurements: number; +} + +/** Same threshold/rounding contract as `trendSlope()` in `trends.ts`. */ +function buildSlope( + slope: number | null, + rSquared: number | null, +): DataSummary["slope7"] { + if (slope === null) return null; + const threshold = 0.01; + const direction: "up" | "down" | "stable" = + Math.abs(slope) < threshold ? "stable" : slope > 0 ? "up" : "down"; + return { + slope: Math.round(slope * 1000) / 1000, + direction, + confidence: rSquared === null ? 0 : Math.round(rSquared * 100) / 100, + }; +} + +function round2(value: number | null): number | null { + if (value === null || !Number.isFinite(value)) return null; + return Math.round(value * 100) / 100; +} + +/** + * Build the per-type `DataSummary` bundle for the comprehensive route. + * + * Three SQL passes against the same `(user_id, type, measured_at)` + * index, all targeting the trailing 90 days: + * + * 1. Per-type aggregate (count + min/max/mean/stddev + windowed + * avgs + regr_slope/r2 for the 7/30/90-day slopes). + * 2. `DISTINCT ON (type)` for the most-recent raw value per type. + * 3. Per-type-per-day mean (`date_trunc('day', measured_at)`) for the + * correlation pairings and the daily-bucket consumers. + * + * Plus two bounded raw-row queries for sys / dia (BP target pairing + * needs sub-day timestamps, can't be replaced by a daily mean). + */ +export async function buildComprehensiveAggregate( + userId: string, +): Promise { + const ninetyDaysAgo = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); + + const [aggregates, latests, dailyRows, sysRaw, diaRaw] = await Promise.all([ + prisma.$queryRaw` + WITH window_stats AS ( + SELECT + m."type", + AVG(m."value") AS mean_value, + STDDEV_POP(m."value") AS stddev_value + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."measured_at" >= ${ninetyDaysAgo} + GROUP BY m."type" + ) + SELECT + m."type"::text AS type, + COUNT(*) AS count, + MIN(m."value")::double precision AS min_value, + MAX(m."value")::double precision AS max_value, + AVG(m."value")::double precision AS mean_value, + ws.stddev_value::double precision AS stddev_value, + -- |z| > 2 anomaly count over the 90-day window. STDDEV_POP + -- matches the JS helper which divides by n (not n-1). + COUNT(*) FILTER ( + WHERE ws.stddev_value IS NOT NULL + AND ws.stddev_value > 0 + AND ABS( + (ROUND(((m."value" - ws.mean_value) / ws.stddev_value)::numeric, 2)) + ) > 2 + ) AS anomaly_count, + AVG(m."value") FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '7 days' + )::double precision AS avg7, + AVG(m."value") FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '30 days' + )::double precision AS avg30, + -- Prior 30d window (days [30, 60) ago) used by tile delta callout. + AVG(m."value") FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '60 days' + AND m."measured_at" < NOW() - INTERVAL '30 days' + )::double precision AS avg30_last_month, + REGR_SLOPE( + m."value", + EXTRACT(EPOCH FROM m."measured_at") / 86400.0 + ) FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '7 days' + )::double precision AS slope7, + REGR_R2( + m."value", + EXTRACT(EPOCH FROM m."measured_at") / 86400.0 + ) FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '7 days' + )::double precision AS r2_7, + REGR_SLOPE( + m."value", + EXTRACT(EPOCH FROM m."measured_at") / 86400.0 + ) FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '30 days' + )::double precision AS slope30, + REGR_R2( + m."value", + EXTRACT(EPOCH FROM m."measured_at") / 86400.0 + ) FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '30 days' + )::double precision AS r2_30, + REGR_SLOPE( + m."value", + EXTRACT(EPOCH FROM m."measured_at") / 86400.0 + ) FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '90 days' + )::double precision AS slope90, + REGR_R2( + m."value", + EXTRACT(EPOCH FROM m."measured_at") / 86400.0 + ) FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '90 days' + )::double precision AS r2_90 + FROM measurements m + JOIN window_stats ws ON ws."type" = m."type" + WHERE m."user_id" = ${userId} + AND m."measured_at" >= ${ninetyDaysAgo} + GROUP BY m."type", ws.stddev_value + `, + prisma.$queryRaw` + SELECT DISTINCT ON (m."type") + m."type"::text AS type, + m."value"::double precision AS value, + m."measured_at" AS measured_at + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."measured_at" >= ${ninetyDaysAgo} + ORDER BY m."type", m."measured_at" DESC + `, + prisma.$queryRaw` + SELECT + m."type"::text AS type, + TO_CHAR(date_trunc('day', m."measured_at"), 'YYYY-MM-DD') AS day, + (ROUND((AVG(m."value"))::numeric, 2))::double precision AS mean_value + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."measured_at" >= ${ninetyDaysAgo} + AND m."type" IN ('WEIGHT', 'BLOOD_PRESSURE_SYS', 'PULSE') + GROUP BY m."type", date_trunc('day', m."measured_at") + ORDER BY m."type", day ASC + `, + prisma.measurement.findMany({ + where: { + userId, + type: "BLOOD_PRESSURE_SYS", + measuredAt: { gte: ninetyDaysAgo }, + }, + orderBy: { measuredAt: "asc" }, + select: { measuredAt: true, value: true }, + }), + prisma.measurement.findMany({ + where: { + userId, + type: "BLOOD_PRESSURE_DIA", + measuredAt: { gte: ninetyDaysAgo }, + }, + orderBy: { measuredAt: "asc" }, + select: { measuredAt: true, value: true }, + }), + ]); + + // ── avg30LastYear ───────────────────────────────────────── + // The 90-day window guarantees this is null (no rows >= 365 days + // old will appear) — preserves legacy semantics. + + const latestByType = new Map(); + for (const row of latests) { + latestByType.set(row.type, { + value: Number(row.value), + measuredAt: new Date(row.measured_at), + }); + } + + // Seed only types that actually have rows — the legacy route only + // populated `summaries[t]` when `data.length > 0`. + const summaries: Record = {}; + let totalMeasurements = 0; + let firstMeasurementAt: Date | null = null; + + for (const row of aggregates) { + const count = Number(row.count); + totalMeasurements += count; + const latest = latestByType.get(row.type); + summaries[row.type] = { + count, + latest: latest?.value ?? null, + min: round2(row.min_value), + max: round2(row.max_value), + mean: round2(row.mean_value), + avg7: round2(row.avg7), + avg30: round2(row.avg30), + slope7: buildSlope(row.slope7, row.r2_7), + slope30: buildSlope(row.slope30, row.r2_30), + slope90: buildSlope(row.slope90, row.r2_90), + anomalyCount: Number(row.anomaly_count), + avg30LastMonth: round2(row.avg30_last_month), + // Legacy semantics: the 90-day findMany window guarantees no + // rows from a year ago. Always null. + avg30LastYear: null, + }; + } + + // ── firstMeasurementAt ─────────────────────────────────── + // Take the earliest `measuredAt` across the user's 90-day rows. + // One extra cheap aggregate query keeps the route's `dataSpanDays` + // truthful. Run lazily: skip when no aggregate rows came back + // (totalMeasurements === 0). + if (totalMeasurements > 0) { + const firstRows = await prisma.$queryRaw>` + SELECT MIN(m."measured_at") AS first_at + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."measured_at" >= ${ninetyDaysAgo} + `; + firstMeasurementAt = firstRows[0]?.first_at + ? new Date(firstRows[0].first_at) + : null; + } + + // ── daily-by-type ──────────────────────────────────────── + const dailyByType: Record> = {}; + for (const row of dailyRows) { + const list = dailyByType[row.type] ?? []; + list.push({ day: row.day, value: Number(row.mean_value) }); + dailyByType[row.type] = list; + } + + return { + summaries, + bpRawRows: { + sys: sysRaw.map((r) => ({ measuredAt: r.measuredAt, value: r.value })), + dia: diaRaw.map((r) => ({ measuredAt: r.measuredAt, value: r.value })), + }, + dailyByType, + firstMeasurementAt, + totalMeasurements, + }; +} + diff --git a/tests/integration/insights-comprehensive-cache.test.ts b/tests/integration/insights-comprehensive-cache.test.ts new file mode 100644 index 000000000..4333eb9e5 --- /dev/null +++ b/tests/integration/insights-comprehensive-cache.test.ts @@ -0,0 +1,282 @@ +/** + * v1.4.35 — integration coverage for the comprehensive insights cache. + * + * Pins two contracts the unit suite cannot reach: + * + * 1. **Read-through cache.** Priming with one call counts as a + * miss; the second call inside the 60-second TTL counts as a hit + * and annotates `cache.analytics.outcome === "hit"` on the route's + * wide event. + * + * 2. **SQL aggregation parity end-to-end.** The route's envelope + * includes every legacy key (summaries / bmi / bp* / mood* / + * medications / alerts / hasProvider / dataSpanDays / + * totalMeasurements) computed from a fixture of ~50 measurement + * rows. The byte-shape matches what the consumer (/insights page) + * reads on first paint. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +// `resolveProvider` reads `app_settings.default_locale` which trails the +// production schema in this integration env. Mock the leaf: the route +// only uses `provider.type !== "none"` to set the `hasProvider` flag. +vi.mock("@/lib/ai/provider", () => ({ + resolveProvider: vi.fn().mockResolvedValue({ type: "none" }), +})); + +import { clearLogBuffer, readLogBuffer } from "@/lib/logging/in-memory-buffer"; + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + clearLogBuffer(); + const { __resetAllCachesForTests } = await import( + "@/lib/cache/server-cache" + ); + __resetAllCachesForTests(); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +async function seedSession(username: string) { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { + username, + email: `${username}@example.test`, + role: "USER", + heightCm: 180, + dateOfBirth: new Date("1985-01-01"), + }, + }); + const session = await prisma.session.create({ + data: { + userId: user.id, + expiresAt: new Date(Date.now() + 60_000), + }, + }); + cookieJar.set("healthlog_session", session.id); + return user; +} + +describe("GET /api/insights/comprehensive — server cache", () => { + it("returns a hit on the second call inside the TTL", async () => { + const user = await seedSession("comp-cache-hit-user"); + const prisma = getPrismaClient(); + + // Seed ~50 rows so the aggregator has data to return. + const now = Date.now(); + const DAY_MS = 24 * 60 * 60 * 1000; + const weightRows = Array.from({ length: 30 }, (_, i) => ({ + userId: user.id, + type: "WEIGHT" as const, + value: 80 + (i % 5) * 0.2, + unit: "kg", + source: "MANUAL" as const, + measuredAt: new Date(now - i * DAY_MS), + })); + const sysRows = Array.from({ length: 10 }, (_, i) => ({ + userId: user.id, + type: "BLOOD_PRESSURE_SYS" as const, + value: 120 + i, + unit: "mmHg", + source: "MANUAL" as const, + measuredAt: new Date(now - i * DAY_MS), + })); + const diaRows = Array.from({ length: 10 }, (_, i) => ({ + userId: user.id, + type: "BLOOD_PRESSURE_DIA" as const, + value: 75 + i, + unit: "mmHg", + source: "MANUAL" as const, + measuredAt: new Date(now - i * DAY_MS), + })); + await prisma.measurement.createMany({ + data: [...weightRows, ...sysRows, ...diaRows], + }); + + const { GET } = await import("@/app/api/insights/comprehensive/route"); + const { caches } = await import("@/lib/cache/server-cache"); + + const first = await ( + GET as unknown as (req: Request) => Promise + )(new Request("http://localhost/api/insights/comprehensive")); + expect(first.status).toBe(200); + expect(caches.analytics.stats().misses).toBe(1); + expect(caches.analytics.stats().hits).toBe(0); + + // Second call inside the TTL — hit. + const second = await ( + GET as unknown as (req: Request) => Promise + )(new Request("http://localhost/api/insights/comprehensive")); + expect(second.status).toBe(200); + expect(caches.analytics.stats().hits).toBe(1); + + // The wide-event annotation carries the cache outcome on the hit. + const events = readLogBuffer({}); + const cacheEvent = events.find( + (e) => + (e?.meta as Record | undefined)?.[ + "cache.analytics.outcome" + ] === "hit", + ); + expect(cacheEvent).toBeDefined(); + }); + + it("emits every legacy envelope key with real SQL aggregation", async () => { + const user = await seedSession("comp-cache-shape-user"); + const prisma = getPrismaClient(); + + // A tight fixture: 5 paired BP readings + 5 weight rows + 5 mood + // entries spread across 5 distinct days. Enough to populate every + // correlation block (n >= 5 is the pearsonCorrelation gate). + const now = Date.now(); + const DAY_MS = 24 * 60 * 60 * 1000; + const dayDates = Array.from( + { length: 5 }, + (_, i) => new Date(now - i * DAY_MS), + ); + await prisma.measurement.createMany({ + data: [ + // Sys + dia at exactly the same instant (5-min tolerance auto-passes) + ...dayDates.flatMap((d, i) => [ + { + userId: user.id, + type: "BLOOD_PRESSURE_SYS" as const, + value: 120 + i, + unit: "mmHg", + source: "MANUAL" as const, + measuredAt: d, + }, + { + userId: user.id, + type: "BLOOD_PRESSURE_DIA" as const, + value: 75 + i, + unit: "mmHg", + source: "MANUAL" as const, + measuredAt: d, + }, + ]), + ...dayDates.map((d, i) => ({ + userId: user.id, + type: "WEIGHT" as const, + value: 80 + i, + unit: "kg", + source: "MANUAL" as const, + measuredAt: d, + })), + ...dayDates.map((d, i) => ({ + userId: user.id, + type: "PULSE" as const, + value: 65 + i, + unit: "bpm", + source: "MANUAL" as const, + measuredAt: d, + })), + ], + }); + + await prisma.moodEntry.createMany({ + data: dayDates.map((d, i) => ({ + userId: user.id, + date: d.toISOString().slice(0, 10), + mood: "GUT", + score: 3 + (i % 3), + moodLoggedAt: d, + })), + }); + + const { GET } = await import("@/app/api/insights/comprehensive/route"); + + const response = await ( + GET as unknown as (req: Request) => Promise + )(new Request("http://localhost/api/insights/comprehensive")); + expect(response.status).toBe(200); + + const body = (await response.json()) as { + data: Record & { + summaries: Record< + string, + { count: number; latest: number | null; mean: number | null } | undefined + >; + totalMeasurements: number; + bpPctInTarget: number | null; + bpClassification: { category: string } | null; + bmi: number | null; + }; + }; + + // Every legacy key still present. + const legacyKeys = [ + "summaries", + "bmi", + "bmiClassification", + "bpClassification", + "bpPctInTarget", + "bpTargets", + "weightBpCorrelation", + "scatterData", + "bpMedicationCorrelation", + "bpMedicationScatterData", + "moodSummary", + "moodBpCorrelation", + "moodBpScatterData", + "moodWeightCorrelation", + "moodWeightScatterData", + "moodPulseCorrelation", + "moodPulseScatterData", + "medications", + "alerts", + "hasProvider", + "dataSpanDays", + "totalMeasurements", + ]; + for (const key of legacyKeys) { + expect(body.data).toHaveProperty(key); + } + + // Sanity-check the aggregator's numbers. + expect(body.data.summaries.WEIGHT?.count).toBe(5); + expect(body.data.summaries.BLOOD_PRESSURE_SYS?.count).toBe(5); + expect(body.data.summaries.PULSE?.count).toBe(5); + // 5 SYS + 5 DIA + 5 WEIGHT + 5 PULSE = 20 measurement rows. Mood + // entries live on `moodEntries` and don't count toward this field + // (matches legacy semantics). + expect(body.data.totalMeasurements).toBe(20); + // bpPctInTarget: 120-124/75-79 all inside under-65 ceiling. + expect(body.data.bpPctInTarget).toBe(100); + // BMI: latest weight = 80 (newest day, smallest i) at 1.80m → 24.7. + expect(body.data.bmi).toBeCloseTo(24.7, 1); + }); +});