diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index ae4fc16ec..c17830d99 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -77130,7 +77130,9 @@ components: glucose while their entry is `bidirectional` or `writeOnly` and enabled): when the newest reading of the type was entered here rather than arriving from Apple Health, or null when none was. Such readings go to the phone and never come back as an arrival, so `stale` already counts them: the type is quiet only when both - this and `lastSeenAt` are. Show it as the server sends it; do not recompute `stale`." + are. Since v1.43.9 `lastSeenAt` on such a type is the newer of this and the newest reading from Apple + Health, because a reading entered here is in Apple Health too; show both as the server sends them and do not + recompute `stale`." anyOf: - type: string format: date-time diff --git a/docs/ops/heap-diagnostics.md b/docs/ops/heap-diagnostics.md index 87b12052f..7652079a3 100644 --- a/docs/ops/heap-diagnostics.md +++ b/docs/ops/heap-diagnostics.md @@ -6,7 +6,7 @@ A Node process that reaches its heap limit stops with `JavaScript heap out of me Two background wide events, no configuration needed: -- `runtime.heap.sampled` (info), every five minutes, and every minute while the heap is at or above 70 % of its limit. It goes to stdout and Loki but not to Admin > App logs, so it cannot push other events out of that page's 500-entry buffer. It carries `heap_used_mb`, `heap_total_mb`, `heap_limit_mb`, `heap_used_ratio`, `old_space_mb`, `rss_mb`, `external_mb`, `array_buffers_mb`, `active_requests`, `active_jobs`, `in_flight_dropped` and `uptime_s`. +- `runtime.heap.sampled` (info), every five minutes, and every minute while the heap is at or above 70 % of its limit. It goes to stdout and Loki but not to Admin > App logs, so it cannot push other events out of that page's 500-entry buffer. It carries `heap_used_mb`, `heap_total_mb`, `heap_limit_mb`, `heap_used_ratio`, the used size of each heap space (`new_space_mb`, `old_space_mb`, `code_space_mb`, `large_object_space_mb`, `new_large_object_space_mb`, `code_large_object_space_mb`, `trusted_space_mb`, `read_only_space_mb`, and `other_spaces_mb` for any space not named), `rss_mb`, `external_mb`, `array_buffers_mb`, `active_requests`, `active_jobs`, `in_flight_dropped` and `uptime_s`. The spaces add up to `heap_used_mb`: a large buffer or string (a profile picture held in memory, a long response body) shows in `large_object_space_mb`, not in `old_space_mb`, and memory outside the V8 heap shows in `external_mb` and `array_buffers_mb`. - `runtime.heap.pressure` (warn), the first time the heap reaches 85 % of its limit. Besides the figures above it lists the 50 oldest requests and jobs in flight (`in_flight_oldest`: kind, method, route, age in seconds, whether it was a server action) and the routes most of them share (`in_flight_by_route`). Routes are reduced to their shape: no query string, secret path segments replaced by `[REDACTED]`, ids by `:id`, anything unusual by `:x`. It writes again only after the heap has dropped below 70 % and thirty minutes have passed. One exception: when the heap reaches 95 %, the same report is written again at once, with `pressure_ratio: 0.95`, whatever the cooldown; that tier re-arms as soon as the heap is back under 85 %. A heap that climbs to its limit shortly after an 85 % report therefore still leaves a report naming the work that pushed it over. The heap is read every five seconds, a single call that takes microseconds. Tracking requests costs one map insert and one delete per request. diff --git a/next.config.ts b/next.config.ts index 6658934f0..0af3eb7aa 100644 --- a/next.config.ts +++ b/next.config.ts @@ -329,9 +329,20 @@ const nextConfig: NextConfig = { // their own body after authenticating (`src/lib/http/proxy-bypass-routes.ts`), // and every other route's own cap is at or below this. Large exports still // want a matching reverse-proxy body limit (docs/self-hosting/reverse-proxy.md). + // + // `preloadEntriesOnStart` is off. Next turns it on by default and then + // requires every route and page module (about 670 here) right after the + // server starts. That load runs on the event loop: a stall of several + // seconds in the first minute of every container, the first requests + // queued behind it, and a heap of about 430 MB before anyone has signed in, + // most of it code for routes a given instance rarely or never serves. Off, + // each module loads on its route's first request, which costs that one + // request a few tens of milliseconds; the same image measured locally + // started at 185 MB of heap instead of 428 and without the stall. experimental: { optimizePackageImports: ["recharts", "lucide-react"], proxyClientMaxBodySize: "1mb", + preloadEntriesOnStart: false, }, }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6a9f052b4..4a51aab68 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,7 +6,7 @@ settings: overrides: js-yaml@<4.3.2: ^4.3.2 - hono@<4.13.5: ^4.13.5 + hono@<4.13.11: ^4.13.11 baseline-browser-mapping@<2.11.0: ^2.11.0 sharp@<0.35.4: ^0.35.4 dompurify@<3.4.16: ^3.4.16 @@ -669,7 +669,7 @@ packages: resolution: {integrity: sha512-dSneS5qhiauZWGDCeK4o695Xd9nUNjviSZCMQrj10eetr8Uln1ucn6bbphOM6UynAMMtNIzZNSpL9vnASJwrPQ==} engines: {node: '>=18.14.1'} peerDependencies: - hono: ^4.13.5 + hono: ^4.13.11 '@humanfs/core@0.19.2': resolution: {integrity: sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==} @@ -4436,8 +4436,8 @@ packages: hermes-parser@0.25.1: resolution: {integrity: sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==} - hono@4.13.7: - resolution: {integrity: sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==} + hono@4.13.13: + resolution: {integrity: sha512-CQ46U0ZkAGmbT/4UxdzzGJpacP2IeKgY4a5/tOI9AABbpOMfK739wfDXmv1usCk+3RkKj1hQy4/fjhiwa2xlrA==} engines: {node: '>=16.9.0'} html-escaper@2.0.2: @@ -7382,9 +7382,9 @@ snapshots: '@hexagon/base64@1.1.28': {} - '@hono/node-server@1.19.17(hono@4.13.7)': + '@hono/node-server@1.19.17(hono@4.13.13)': dependencies: - hono: 4.13.7 + hono: 4.13.13 '@humanfs/core@0.19.2': dependencies: @@ -7581,7 +7581,7 @@ snapshots: '@modelcontextprotocol/sdk@1.32.1(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@3.25.76)': dependencies: - '@hono/node-server': 1.19.17(hono@4.13.7) + '@hono/node-server': 1.19.17(hono@4.13.13) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -7591,7 +7591,7 @@ snapshots: eventsource-parser: 3.1.1 express: 5.2.1(supports-color@7.2.0) express-rate-limit: 8.6.2(express@5.2.1(supports-color@7.2.0))(supports-color@7.2.0) - hono: 4.13.7 + hono: 4.13.13 jose: 6.2.12 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -7605,7 +7605,7 @@ snapshots: '@modelcontextprotocol/sdk@1.32.1(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@4.6.7)': dependencies: - '@hono/node-server': 1.19.17(hono@4.13.7) + '@hono/node-server': 1.19.17(hono@4.13.13) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -7615,7 +7615,7 @@ snapshots: eventsource-parser: 3.1.1 express: 5.2.1(supports-color@7.2.0) express-rate-limit: 8.6.2(express@5.2.1(supports-color@7.2.0))(supports-color@7.2.0) - hono: 4.13.7 + hono: 4.13.13 jose: 6.2.12 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -11247,7 +11247,7 @@ snapshots: dependencies: hermes-estree: 0.25.1 - hono@4.13.7: {} + hono@4.13.13: {} html-escaper@2.0.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e98b2596f..a699c5cd7 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,11 +1,12 @@ overrides: # CVE-2026-84375 (js-yaml < 4.3.2, HIGH), CVE-2026-84363/84364/84365 (hono - # < 4.13.5, via @hono/node-server under the MCP SDK), CVE-2026-45819 + # < 4.13.5, via @hono/node-server under the MCP SDK; GHSA-5r4p-p66f-jhc7 raised the + # floor to 4.13.11), CVE-2026-45819 # (baseline-browser-mapping < 2.11.0), GHSA-rgj7-g3m4-5g8c (sharp < 0.35.4, # pulled by next). None is a direct dependency; the bounds move the copies # the scanner flags and nothing else. "js-yaml@<4.3.2": "^4.3.2" - "hono@<4.13.5": "^4.13.5" + "hono@<4.13.11": "^4.13.11" "baseline-browser-mapping@<2.11.0": "^2.11.0" "sharp@<0.35.4": "^0.35.4" # GHSA-55q2-fjhq-7xh7: a crafted document gets past the sanitiser and its diff --git a/prisma/migrations/0417_measurement_rollup_reading_extremes/migration.sql b/prisma/migrations/0417_measurement_rollup_reading_extremes/migration.sql new file mode 100644 index 000000000..a42916f48 --- /dev/null +++ b/prisma/migrations/0417_measurement_rollup_reading_extremes/migration.sql @@ -0,0 +1,24 @@ +-- The lowest and highest reading value per rollup bucket. +-- +-- The full-history extremes the insight features carry (`allTimeMin`, +-- `allTimeMax`, `allTimeAvg` for weight, blood pressure and pulse) were read +-- from the raw `measurements` rows on every call: two passes over every +-- reading of those types the account ever recorded, the largest read on the +-- database. The DAY rollups already hold everything that fold needs except +-- the extremes of the readings themselves: `min_value` / `max_value` widen to +-- a row's own `value_min` / `value_max`, the intra-hour range an hourly +-- heart-rate statistic carries, so they are not the same figure. +-- +-- Two nullable columns. Existing rows read NULL until the boot re-fold +-- (`rollup-full-backfill`, discovered by the NULL arm beside `sum_xy`) +-- refills them; new writes fill them from the same aggregate as every other +-- column. A reader treats a type with a NULL in its window as uncovered and +-- reads the raw rows, as before. +-- +-- Reversibility: +-- ALTER TABLE "measurement_rollups" DROP COLUMN IF EXISTS "reading_min_value"; +-- ALTER TABLE "measurement_rollups" DROP COLUMN IF EXISTS "reading_max_value"; + +ALTER TABLE "measurement_rollups" ADD COLUMN IF NOT EXISTS "reading_min_value" DOUBLE PRECISION; + +ALTER TABLE "measurement_rollups" ADD COLUMN IF NOT EXISTS "reading_max_value" DOUBLE PRECISION; diff --git a/prisma/migrations/0418_audit_log_geo_attempt/migration.sql b/prisma/migrations/0418_audit_log_geo_attempt/migration.sql new file mode 100644 index 000000000..cd5cd5d84 --- /dev/null +++ b/prisma/migrations/0418_audit_log_geo_attempt/migration.sql @@ -0,0 +1,30 @@ +-- The geo backfill notes when it last tried a row, and finds its candidates +-- through an index that holds only them. +-- +-- The hourly backfill (`src/lib/jobs/geo-backfill.ts`) re-resolves audit rows +-- whose location or carrier is missing. An address no database will ever +-- place (a private range that reached the log through a proxy) stays missing, +-- so the same few rows were looked up again on every pass, and finding them +-- meant walking thirty days of `audit_logs` through `created_at`, filtering +-- every row that had long been resolved. +-- +-- `geo_attempted_at` records the try; the backfill skips a row it tried less +-- than `GEO_BACKFILL_RETRY_DAYS` ago. NULL for every existing row: each is +-- tried once more, then waits like any other. +-- +-- `audit_logs_geo_backfill_idx` is partial on the rows still missing +-- something, so it holds the backlog and nothing else, and shrinks as rows +-- resolve. Prisma cannot express the predicate, so the index lives here only +-- (noted at the model in `schema.prisma`). A plain build: it takes SHARE on +-- `audit_logs` while it reads the table once, which `migrate deploy` does at +-- boot before the app serves. +-- +-- Reversibility: +-- DROP INDEX IF EXISTS "audit_logs_geo_backfill_idx"; +-- ALTER TABLE "audit_logs" DROP COLUMN IF EXISTS "geo_attempted_at"; + +ALTER TABLE "audit_logs" ADD COLUMN IF NOT EXISTS "geo_attempted_at" TIMESTAMP(3); + +CREATE INDEX IF NOT EXISTS "audit_logs_geo_backfill_idx" + ON "audit_logs" ("created_at") + WHERE "ip_address" IS NOT NULL AND ("location" IS NULL OR "carrier" IS NULL); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c38d79ad9..c802f3fd4 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -2253,31 +2253,31 @@ enum RollupGranularity { } model MeasurementRollup { - userId String @map("user_id") - type MeasurementType - granularity RollupGranularity - bucketStart DateTime @map("bucket_start") @db.Timestamptz(3) + userId String @map("user_id") + type MeasurementType + granularity RollupGranularity + bucketStart DateTime @map("bucket_start") @db.Timestamptz(3) /// v1.11.1 — source-aware grain. Rollup rows are minted per source so the /// read path can collapse overlapping sources (e.g. WHOOP + Apple Watch /// resting heart rate) to the ladder-canonical reading, while cumulative /// metrics (steps, energy) sum per source. Always non-null; migration 0115 /// purged the legacy source-blind aggregates and the boot backfill re-mints /// per-source rows. - source MeasurementSource - count Int - mean Float - minValue Float @map("min_value") - maxValue Float @map("max_value") - sd Float? - slope Float? - r2 Float? + source MeasurementSource + count Int + mean Float + minValue Float @map("min_value") + maxValue Float @map("max_value") + sd Float? + slope Float? + r2 Float? /// v1.4.39 W-SUM — cumulative-metric sum for the bucket. Populated /// alongside `mean / count` for every type; cumulative read paths /// (steps, flights, distance, daylight, active-energy) consume this /// directly instead of re-deriving from raw measurements. Nullable /// to keep the migration additive — the boot backfill fills NULLs /// for existing rows; new rows always carry the sum. - sumValue Float? @map("sum_value") + sumValue Float? @map("sum_value") /// v1.20.0 F6 — per-bucket OLS regression accumulators over the same /// canonical-source raw rows the live REGR_* query reads. With /// `n = count` and `Σy = mean·count` these close the additive set so a @@ -2291,11 +2291,21 @@ model MeasurementRollup { /// parity-matches the cross-bucket compose. Nullable to keep migration 0190 /// additive — the boot backfill fills NULLs for pre-migration rows; new /// rows always carry them (they ride the same aggregate as mean / sum). - sumX Float? @map("sum_x") - sumXy Float? @map("sum_xy") - sumXx Float? @map("sum_xx") - sumYy Float? @map("sum_yy") - computedAt DateTime @default(now()) @map("computed_at") @db.Timestamptz(3) + sumX Float? @map("sum_x") + sumXy Float? @map("sum_xy") + sumXx Float? @map("sum_xx") + sumYy Float? @map("sum_yy") + /// The lowest and highest reading `value` of the bucket. `minValue` / + /// `maxValue` widen to a row's own range (`Measurement.valueMin` / + /// `valueMax`, the intra-hour range an hourly heart-rate statistic + /// carries); these do not, so a reader whose figure is over the readings + /// themselves (the full-history extremes in `feature-blocks.ts`) can fold + /// the DAY tier instead of the raw rows. Nullable to keep migration 0417 + /// additive: the boot re-fold fills existing rows, new rows always carry + /// them, and a reader treats a type with a NULL in its range as uncovered. + readingMinValue Float? @map("reading_min_value") + readingMaxValue Float? @map("reading_max_value") + computedAt DateTime @default(now()) @map("computed_at") @db.Timestamptz(3) user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@ -5405,18 +5415,26 @@ model AuditLog { /// lives. actorUserId String? @map("actor_user_id") - action String // e.g. "auth.login", "measurement.create", "export.csv" - details String? // JSON string, no sensitive data - ipAddress String? @map("ip_address") - location String? // e.g. "Berlin, DE" — resolved from IP at log time + action String // e.g. "auth.login", "measurement.create", "export.csv" + details String? // JSON string, no sensitive data + ipAddress String? @map("ip_address") + location String? // e.g. "Berlin, DE" — resolved from IP at log time // v1.4.27 B3 — resolved at audit-creation time from the bundled // GeoLite2-ASN MMDB. `asn` is the autonomous-system number, // `carrier` is the organisation string (e.g. "Deutsche Telekom AG"). // Both nullable so older rows, private-IP rows, and offline-miss // rows stay valid. - asn Int? // autonomous-system number - carrier String? // autonomous-system organisation - createdAt DateTime @default(now()) @map("created_at") + asn Int? // autonomous-system number + carrier String? // autonomous-system organisation + /// When the geo backfill (`src/lib/jobs/geo-backfill.ts`) last tried this + /// row. A row it could not complete waits `GEO_BACKFILL_RETRY_DAYS` before + /// the next try, so an address no database will ever place (a private + /// range behind a proxy) is not resolved again on every hourly pass. NULL: + /// never tried. The backfill reads its candidates through the partial index + /// `audit_logs_geo_backfill_idx` (migration 0418, raw SQL only: Prisma + /// cannot express its predicate). + geoAttemptedAt DateTime? @map("geo_attempted_at") + createdAt DateTime @default(now()) @map("created_at") user User? @relation(fields: [userId], references: [id], onDelete: SetNull) diff --git a/src/__tests__/session-user-projection-guard.test.ts b/src/__tests__/session-user-projection-guard.test.ts new file mode 100644 index 000000000..36a64c618 --- /dev/null +++ b/src/__tests__/session-user-projection-guard.test.ts @@ -0,0 +1,137 @@ +/** + * The caller lookups load a projection of `User`, never the whole row. + * + * Every authenticated request resolves its caller through the session lookup + * (`lib/auth/session.ts`), the Bearer resolver (`lib/auth/bearer.ts`) or, for a + * switched record, the owner load in `lib/api-handler.ts`. They used to + * `include: { user: true }` / `findUnique({ where })`, which copied the profile + * picture (megabytes), the cached briefing text and every stored credential + * into memory on each request. They now name `SESSION_USER_SELECT`. + * + * Two halves: + * 1. each of those lookups carries the projection, and no file anywhere loads + * a related user row whole with `include: { user: true }`; + * 2. the projection itself leaves out every `Bytes` column of the model and + * every secret-shaped column, derived from `schema.prisma` so a new + * column of either kind is covered without editing this file. + */ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { walkSourceFiles } from "./helpers/source-files"; +import { SESSION_USER_SELECT } from "@/lib/auth/session-user"; + +const SRC = join(process.cwd(), "src"); + +function read(rel: string): string { + return readFileSync(join(SRC, rel), "utf8"); +} + +function stripComments(src: string): string { + return src + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/(^|[^:])\/\/.*$/gm, "$1"); +} + +/** The `{ ... }` argument of every call matching `callee`, braces balanced. */ +function callObjects(src: string, callee: RegExp): string[] { + const code = stripComments(src); + const out: string[] = []; + const re = new RegExp(callee.source + /\s*\(\s*\{/.source, "g"); + for (let m = re.exec(code); m; m = re.exec(code)) { + const open = m.index + m[0].length - 1; + let depth = 0; + let i = open; + for (; i < code.length; i++) { + if (code[i] === "{") depth++; + else if (code[i] === "}" && --depth === 0) break; + } + out.push(code.slice(open, i + 1)); + } + return out; +} + +const USER_LOOKUP = /prisma\.user\.find(?:Unique|First)(?:OrThrow)?/; +const SESSION_LOOKUP = /prisma\.session\.find(?:Unique|First)(?:OrThrow)?/; + +describe("caller lookups load the session projection", () => { + it("the session lookups include the user through SESSION_USER_SELECT", () => { + const calls = callObjects( + read("lib/auth/session.ts"), + SESSION_LOOKUP, + ).filter((body) => /\binclude\s*:/.test(body)); + expect(calls.length).toBeGreaterThanOrEqual(2); + for (const body of calls) { + expect(body).toMatch( + /include\s*:\s*\{\s*user\s*:\s*\{\s*select\s*:\s*SESSION_USER_SELECT\s*\}\s*\}/, + ); + } + }); + + it("the Bearer resolver and the switched-owner load select SESSION_USER_SELECT", () => { + for (const rel of ["lib/auth/bearer.ts", "lib/api-handler.ts"]) { + const calls = callObjects(read(rel), USER_LOOKUP); + expect(calls.length, rel).toBeGreaterThan(0); + for (const body of calls) { + expect(body, rel).toMatch(/\bselect\s*:/); + } + expect( + calls.some((body) => /select\s*:\s*SESSION_USER_SELECT\b/.test(body)), + rel, + ).toBe(true); + } + }); + + it("no file loads a related user row whole", () => { + const files = walkSourceFiles(SRC, { floor: 3000 }) + .filter((p) => !p.startsWith("generated/")) + .filter((p) => !p.includes("__tests__")) + .filter((p) => !/\.test\.tsx?$/.test(p)); + expect(files.length).toBeGreaterThan(0); + const offenders = files.filter((rel) => + /\buser\s*:\s*true\b/.test( + stripComments(read(rel)).replace(/select\s*:\s*\{[^{}]*\}/g, ""), + ), + ); + expect(offenders).toEqual([]); + }); +}); + +describe("SESSION_USER_SELECT leaves out bytes and secrets", () => { + const schema = readFileSync( + join(process.cwd(), "prisma", "schema.prisma"), + "utf8", + ); + const model = schema.match(/^model User \{([\s\S]*?)^\}/m)?.[1] ?? ""; + const columns = [ + ...model.matchAll(/^\s+(\w+)\s+([A-Z]\w*)(\??)(?:\s|$)/gm), + ].map(([, name, type]) => ({ name, type })); + const selected = Object.keys(SESSION_USER_SELECT); + + it("reads the User model", () => { + expect(columns.length).toBeGreaterThan(50); + expect(selected.length).toBeGreaterThan(0); + }); + + it("names only columns the model has", () => { + const names = new Set(columns.map((c) => c.name)); + expect(selected.filter((key) => !names.has(key))).toEqual([]); + }); + + it("selects no Bytes column", () => { + const bytes = columns.filter((c) => c.type === "Bytes").map((c) => c.name); + expect(bytes).toContain("avatarBytes"); + expect(selected.filter((key) => bytes.includes(key))).toEqual([]); + }); + + it("selects no credential, secret or large cached text", () => { + const secretShaped = + /Encrypted$|^passwordHash$|^totpSecret|^totpLastStep$|^telegramBotToken$|^insightsCachedText$|^environmentAqHistoryJson$/; + const secrets = columns + .map((c) => c.name) + .filter((name) => secretShaped.test(name)); + expect(secrets.length).toBeGreaterThan(10); + expect(selected.filter((key) => secrets.includes(key))).toEqual([]); + }); +}); diff --git a/src/app/api/account/grants/__tests__/invite-recent-proof.test.ts b/src/app/api/account/grants/__tests__/invite-recent-proof.test.ts index e4c987074..f450f87e6 100644 --- a/src/app/api/account/grants/__tests__/invite-recent-proof.test.ts +++ b/src/app/api/account/grants/__tests__/invite-recent-proof.test.ts @@ -14,7 +14,11 @@ vi.mock("@/lib/db", () => ({ session: { findUnique: vi.fn() }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, - user: { findFirst: vi.fn() }, + user: { + findFirst: vi.fn(), + // The stored row: the password hash is not on the session user. + findUnique: vi.fn(async () => ({ passwordHash: "argon" })), + }, accountGrant: { create: vi.fn(), findFirst: vi.fn() }, }, })); @@ -50,7 +54,6 @@ const OWNER = { role: "USER", username: "owner", totpConfirmedAt: null, - passwordHash: "argon", }; function invite(): NextRequest { diff --git a/src/app/api/admin/users/[id]/reset-password/route.ts b/src/app/api/admin/users/[id]/reset-password/route.ts index ae913dddc..257102aeb 100644 --- a/src/app/api/admin/users/[id]/reset-password/route.ts +++ b/src/app/api/admin/users/[id]/reset-password/route.ts @@ -60,7 +60,10 @@ export const POST = apiHandler( return apiError("Password required", 422); } - const target = await prisma.user.findUnique({ where: { id } }); + const target = await prisma.user.findUnique({ + where: { id }, + select: { locale: true, username: true }, + }); if (!target) return apiError("User not found", 404); const locale = await resolveServerLocale({ diff --git a/src/app/api/admin/users/[id]/route.ts b/src/app/api/admin/users/[id]/route.ts index 8a954c7dd..f55cb71fc 100644 --- a/src/app/api/admin/users/[id]/route.ts +++ b/src/app/api/admin/users/[id]/route.ts @@ -60,7 +60,10 @@ export const PUT = apiHandler( return returnAllZodIssues(parsed.error, 422); } - const target = await prisma.user.findUnique({ where: { id } }); + const target = await prisma.user.findUnique({ + where: { id }, + select: { role: true }, + }); if (!target) return apiError("User not found", 404); // Prevent removing the last admin diff --git a/src/app/api/auth/me/mfa/disable/__tests__/route.test.ts b/src/app/api/auth/me/mfa/disable/__tests__/route.test.ts index a819d6a08..96134f00b 100644 --- a/src/app/api/auth/me/mfa/disable/__tests__/route.test.ts +++ b/src/app/api/auth/me/mfa/disable/__tests__/route.test.ts @@ -34,6 +34,7 @@ vi.mock("@/lib/rate-limit", () => ({ vi.mock("@/lib/auth/mfa/verify-factor", () => ({ verifyMfaFactor: vi.fn(), + loadFactorUser: vi.fn(), })); vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); @@ -42,7 +43,7 @@ import { POST } from "../route"; import { requireMfaManagementAuth } from "@/lib/api-handler"; import { prisma } from "@/lib/db"; import { destroyOtherSessions } from "@/lib/auth/session"; -import { verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; +import { loadFactorUser, verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; function req() { return new Request("http://localhost/api/auth/me/mfa/disable", { @@ -54,6 +55,12 @@ function req() { beforeEach(() => { vi.clearAllMocks(); + vi.mocked(loadFactorUser).mockResolvedValue({ + id: "u1", + totpSecretEncrypted: "enc", + totpLastStep: null, + totpConfirmedAt: new Date("2026-01-01"), + }); vi.mocked(requireMfaManagementAuth).mockResolvedValue({ transport: "cookie", user: { id: "u1" }, diff --git a/src/app/api/auth/me/mfa/disable/route.ts b/src/app/api/auth/me/mfa/disable/route.ts index 1493d61e4..201f4c61f 100644 --- a/src/app/api/auth/me/mfa/disable/route.ts +++ b/src/app/api/auth/me/mfa/disable/route.ts @@ -30,7 +30,7 @@ import { auditLog } from "@/lib/auth/audit"; import { prisma } from "@/lib/db"; import { destroyOtherSessions } from "@/lib/auth/session"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; -import { verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; +import { loadFactorUser, verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; import { mfaDisableSchema } from "@/lib/validations/mfa"; export const dynamic = "force-dynamic"; @@ -72,11 +72,10 @@ export const POST = apiHandler(async (req: Request) => { ); } - const factor = await verifyMfaFactor( - user, - parsed.data.method, - parsed.data.code, - ); + const factorUser = await loadFactorUser(user.id); + const factor = factorUser + ? await verifyMfaFactor(factorUser, parsed.data.method, parsed.data.code) + : { ok: false, replay: false }; if (!factor.ok) { await auditLog("auth.mfa.failed", { userId: user.id, diff --git a/src/app/api/auth/me/mfa/totp/confirm/route.ts b/src/app/api/auth/me/mfa/totp/confirm/route.ts index 6f30c6c6d..ec21b4bec 100644 --- a/src/app/api/auth/me/mfa/totp/confirm/route.ts +++ b/src/app/api/auth/me/mfa/totp/confirm/route.ts @@ -37,6 +37,7 @@ import { destroyOtherSessions } from "@/lib/auth/session"; import { decrypt } from "@/lib/crypto"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; import { verifyTotp } from "@/lib/auth/mfa/totp"; +import { loadFactorUser } from "@/lib/auth/mfa/verify-factor"; import { generateRecoveryCodes, persistRecoveryCodes, @@ -69,7 +70,9 @@ export const POST = apiHandler(async (req: Request) => { if (user.totpConfirmedAt) { throw new HttpError(409, "A second factor is already active"); } - if (!user.totpSecretEncrypted) { + // The pending secret and its replay floor are not on the per-request user. + const pending = await loadFactorUser(user.id); + if (!pending?.totpSecretEncrypted) { throw new HttpError(409, "Start enrollment before confirming"); } @@ -90,11 +93,11 @@ export const POST = apiHandler(async (req: Request) => { // Decrypt is fail-closed — a missing/rotated key throws rather than // confirming against junk. - const secret = decrypt(user.totpSecretEncrypted); + const secret = decrypt(pending.totpSecretEncrypted); const result = verifyTotp( secret, parsed.data.code, - user.totpLastStep === null ? null : Number(user.totpLastStep), + pending.totpLastStep === null ? null : Number(pending.totpLastStep), ); if (!result.valid) { diff --git a/src/app/api/auth/me/route.ts b/src/app/api/auth/me/route.ts index 41a819538..09cebfdc0 100644 --- a/src/app/api/auth/me/route.ts +++ b/src/app/api/auth/me/route.ts @@ -162,6 +162,7 @@ export const GET = apiHandler(async () => { onboardingRow, ai, coachReasoning, + identity, ] = await Promise.all([ // The actor's own row is already in hand; a switched session needs the // record's `gender`, which is the column the cycle gate derives from. @@ -235,6 +236,13 @@ export const GET = apiHandler(async () => { recordId, authority: aiAuthority, }), + // The encrypted insurance number is not on the per-request user + // (`SESSION_USER_SELECT`); it is read here, for the one response that + // shows it. + prisma.user.findUnique({ + where: { id: user.id }, + select: { insuranceNumberEncrypted: true }, + }), ]); // The masking step, and the one place the record scoping is narrowed rather @@ -286,9 +294,9 @@ export const GET = apiHandler(async () => { // one row never 500s the whole profile fetch (the field just reads // null and the user re-enters it). let insuranceNumber: string | null = null; - if (user.insuranceNumberEncrypted) { + if (identity?.insuranceNumberEncrypted) { try { - insuranceNumber = decrypt(user.insuranceNumberEncrypted); + insuranceNumber = decrypt(identity.insuranceNumberEncrypted); } catch { insuranceNumber = null; } diff --git a/src/app/api/auth/passkeys/[id]/__tests__/route.test.ts b/src/app/api/auth/passkeys/[id]/__tests__/route.test.ts index c4e4adde5..2d8f7039c 100644 --- a/src/app/api/auth/passkeys/[id]/__tests__/route.test.ts +++ b/src/app/api/auth/passkeys/[id]/__tests__/route.test.ts @@ -21,6 +21,7 @@ vi.mock("@/lib/api-handler", () => ({ vi.mock("@/lib/db", () => ({ prisma: { + user: { findUnique: vi.fn() }, passkey: { findUnique: vi.fn(), update: vi.fn(), @@ -40,7 +41,9 @@ import { DELETE, PATCH } from "../route"; import { requireAuth, requireMfaManagementAuth } from "@/lib/api-handler"; import { prisma } from "@/lib/db"; -const USER = { id: "user-1", username: "u", passwordHash: "argon2id$…" }; +const USER = { id: "user-1", username: "u" }; +/** The stored credential row; the session user carries no hash. */ +const STORED = { passwordHash: "argon2id$…" }; const ROW = { id: "pk-1", @@ -75,6 +78,7 @@ const commitElevation = vi.fn().mockResolvedValue(undefined); beforeEach(() => { vi.resetAllMocks(); + vi.mocked(prisma.user.findUnique).mockResolvedValue(STORED as never); vi.mocked(requireAuth).mockResolvedValue({ user: USER } as never); commitElevation.mockClear().mockResolvedValue(undefined); vi.mocked(requireMfaManagementAuth).mockResolvedValue({ @@ -256,9 +260,12 @@ describe("DELETE /api/auth/passkeys/{id} — the removal's gate", () => { name: "Only one", } as never); vi.mocked(prisma.passkey.count).mockResolvedValue(1 as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + passwordHash: null, + } as never); vi.mocked(requireMfaManagementAuth).mockResolvedValue({ transport: "cookie", - user: { ...USER, passwordHash: null }, + user: USER, session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, commitElevation, } as never); diff --git a/src/app/api/auth/passkeys/[id]/route.ts b/src/app/api/auth/passkeys/[id]/route.ts index 87cd24b77..38fb8ed38 100644 --- a/src/app/api/auth/passkeys/[id]/route.ts +++ b/src/app/api/auth/passkeys/[id]/route.ts @@ -128,10 +128,14 @@ export const DELETE = apiHandler( } // Check: at least 1 auth method must remain - const passkeyCount = await prisma.passkey.count({ - where: { userId: user.id }, - }); - const hasPassword = !!user.passwordHash; + const [passkeyCount, credential] = await Promise.all([ + prisma.passkey.count({ where: { userId: user.id } }), + prisma.user.findUnique({ + where: { id: user.id }, + select: { passwordHash: true }, + }), + ]); + const hasPassword = !!credential?.passwordHash; if (passkeyCount <= 1 && !hasPassword) { return apiError( diff --git a/src/app/api/auth/password/__tests__/route.test.ts b/src/app/api/auth/password/__tests__/route.test.ts index 1ebb1bf31..5cee4d7cd 100644 --- a/src/app/api/auth/password/__tests__/route.test.ts +++ b/src/app/api/auth/password/__tests__/route.test.ts @@ -20,7 +20,7 @@ vi.mock("@/lib/api-handler", async () => { vi.mock("@/lib/db", () => ({ prisma: { - user: { update: vi.fn() }, + user: { update: vi.fn(), findUnique: vi.fn() }, }, })); @@ -92,7 +92,6 @@ const SESSION_OK = { username: "tester", email: "t@example.com", role: "USER" as const, - passwordHash: "h", onboardingCompletedAt: new Date(), locale: "en", }, @@ -109,6 +108,10 @@ function postReq(body: unknown): NextRequest { beforeEach(() => { vi.resetAllMocks(); vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + // The hash is read off the stored row, not the session user. + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + passwordHash: "h", + } as never); vi.mocked(checkRateLimit).mockResolvedValue({ allowed: true } as never); // Default: account clears the step-up (no MFA enrolled, or a fresh factor). vi.mocked(requireFreshMfaIfEnrolled).mockResolvedValue(SESSION_OK as never); diff --git a/src/app/api/auth/password/route.ts b/src/app/api/auth/password/route.ts index 629473990..db421e863 100644 --- a/src/app/api/auth/password/route.ts +++ b/src/app/api/auth/password/route.ts @@ -58,11 +58,18 @@ export const POST = apiHandler(async (request: NextRequest) => { } const { currentPassword, newPassword } = parsed.data; - if (!user.passwordHash) { + const credential = await prisma.user.findUnique({ + where: { id: user.id }, + select: { passwordHash: true }, + }); + if (!credential?.passwordHash) { return apiError("No password set for this account", 400); } - const currentValid = await verifyPassword(user.passwordHash, currentPassword); + const currentValid = await verifyPassword( + credential.passwordHash, + currentPassword, + ); if (!currentValid) { await recordReproofFailure( user.id, diff --git a/src/app/api/dashboard/snapshot/__tests__/route.test.ts b/src/app/api/dashboard/snapshot/__tests__/route.test.ts index a2226125c..e161146af 100644 --- a/src/app/api/dashboard/snapshot/__tests__/route.test.ts +++ b/src/app/api/dashboard/snapshot/__tests__/route.test.ts @@ -17,7 +17,8 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; vi.mock("@/lib/db", () => ({ - prisma: {}, + // The cached briefing text is read off its own row, not the session user. + prisma: { user: { findUnique: async () => null } }, })); vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn(), @@ -71,7 +72,6 @@ const SESSION_OK = { glucoseUnit: "mg/dL", onboardingTourCompleted: true, disableCoach: false, - insightsCachedText: null, insightsCachedAt: null, dashboardWidgetsJson: null, }, diff --git a/src/app/api/fhir/__tests__/capability-and-everything.test.ts b/src/app/api/fhir/__tests__/capability-and-everything.test.ts index c402cf539..d3709b9dd 100644 --- a/src/app/api/fhir/__tests__/capability-and-everything.test.ts +++ b/src/app/api/fhir/__tests__/capability-and-everything.test.ts @@ -28,6 +28,8 @@ vi.mock("@/lib/db", () => ({ }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, + // The password hash is read off the stored row; none here. + user: { findUnique: vi.fn(async () => null) }, }, })); vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: vi.fn() })); diff --git a/src/app/api/integrations/healthkit/__tests__/route.test.ts b/src/app/api/integrations/healthkit/__tests__/route.test.ts index d8f44da69..52b77eb46 100644 --- a/src/app/api/integrations/healthkit/__tests__/route.test.ts +++ b/src/app/api/integrations/healthkit/__tests__/route.test.ts @@ -406,6 +406,9 @@ describe("GET /api/integrations/healthkit — sync health", () => { expect(entries.WEIGHT.lastEnteredHereAt).toBe( enteredHereAt.toISOString(), ); + // The reading entered here is in Apple Health too, so it is the type's + // newest reading: the row and the verdict read the same instant. + expect(entries.WEIGHT.lastSeenAt).toBe(enteredHereAt.toISOString()); // A read-only type keeps its own verdict and carries no such field. expect(entries.PULSE.stale).toBe(true); expect(entries.PULSE).not.toHaveProperty("lastEnteredHereAt"); diff --git a/src/app/api/mcp/oauth/__tests__/authorize.test.ts b/src/app/api/mcp/oauth/__tests__/authorize.test.ts index 164da251e..24f12bb88 100644 --- a/src/app/api/mcp/oauth/__tests__/authorize.test.ts +++ b/src/app/api/mcp/oauth/__tests__/authorize.test.ts @@ -35,6 +35,8 @@ vi.mock("@/lib/db", () => ({ }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, + // The password hash is read off the stored row; none here. + user: { findUnique: vi.fn(async () => null) }, }, })); vi.mock("@/lib/app-settings", () => ({ diff --git a/src/app/api/mcp/tokens/__tests__/route.test.ts b/src/app/api/mcp/tokens/__tests__/route.test.ts index e389130e3..98c3e136e 100644 --- a/src/app/api/mcp/tokens/__tests__/route.test.ts +++ b/src/app/api/mcp/tokens/__tests__/route.test.ts @@ -19,6 +19,8 @@ vi.mock("@/lib/db", () => ({ }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, + // The password hash is read off the stored row; none here. + user: { findUnique: vi.fn(async () => null) }, }, })); vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); diff --git a/src/app/api/record-settings/integrations/route.ts b/src/app/api/record-settings/integrations/route.ts index ced499e0b..027e26b17 100644 --- a/src/app/api/record-settings/integrations/route.ts +++ b/src/app/api/record-settings/integrations/route.ts @@ -38,38 +38,50 @@ export const GET = apiHandler(async () => { const context = await requireGuardianAuth(); const access = resolveGuardianRecordSettingsAccess(context); if (!access) throw new SharingAccessDeniedError(); - const [statuses, withings, whoop, fitbit, googleHealth] = await Promise.all([ - Promise.all( - MANAGED_INTEGRATIONS.map((integration) => - getIntegrationStatus(access.recordId, integration), + const [statuses, withings, whoop, fitbit, googleHealth, credentials] = + await Promise.all([ + Promise.all( + MANAGED_INTEGRATIONS.map((integration) => + getIntegrationStatus(access.recordId, integration), + ), ), - ), - prisma.withingsConnection.findUnique({ - where: { userId: access.recordId }, - select: { id: true }, - }), - prisma.whoopConnection.findUnique({ - where: { userId: access.recordId }, - select: { whoopUserId: true }, - }), - prisma.fitbitConnection.findUnique({ - where: { userId: access.recordId }, - select: { id: true }, - }), - prisma.googleHealthConnection.findUnique({ - where: { userId: access.recordId }, - select: { id: true }, - }), - ]); + prisma.withingsConnection.findUnique({ + where: { userId: access.recordId }, + select: { id: true }, + }), + prisma.whoopConnection.findUnique({ + where: { userId: access.recordId }, + select: { whoopUserId: true }, + }), + prisma.fitbitConnection.findUnique({ + where: { userId: access.recordId }, + select: { id: true }, + }), + prisma.googleHealthConnection.findUnique({ + where: { userId: access.recordId }, + select: { id: true }, + }), + // Presence only; the credentials themselves are not on the per-request + // user and are not read past this null check. + prisma.user.findUnique({ + where: { id: context.user.id }, + select: { + nightscoutUrlEncrypted: true, + polarAccessTokenEncrypted: true, + ouraAccessTokenEncrypted: true, + stravaAccessTokenEncrypted: true, + }, + }), + ]); const connected: Record = { withings: withings !== null, whoop: whoop?.whoopUserId != null, fitbit: fitbit !== null, "google-health": googleHealth !== null, - nightscout: context.user.nightscoutUrlEncrypted !== null, - polar: context.user.polarAccessTokenEncrypted !== null, - oura: context.user.ouraAccessTokenEncrypted !== null, - strava: context.user.stravaAccessTokenEncrypted !== null, + nightscout: (credentials?.nightscoutUrlEncrypted ?? null) !== null, + polar: (credentials?.polarAccessTokenEncrypted ?? null) !== null, + oura: (credentials?.ouraAccessTokenEncrypted ?? null) !== null, + strava: (credentials?.stravaAccessTokenEncrypted ?? null) !== null, // Not offered on a managed profile (no managed integrations before // v1.44); listed so the map stays total. dawarich: false, diff --git a/src/app/api/share-links/__tests__/route.test.ts b/src/app/api/share-links/__tests__/route.test.ts index 33509c3c9..bc5e3ad24 100644 --- a/src/app/api/share-links/__tests__/route.test.ts +++ b/src/app/api/share-links/__tests__/route.test.ts @@ -28,6 +28,8 @@ vi.mock("@/lib/db", () => ({ }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, + // The password hash is read off the stored row; none here. + user: { findUnique: vi.fn(async () => null) }, }, })); diff --git a/src/lib/__tests__/require-fresh-mfa.test.ts b/src/lib/__tests__/require-fresh-mfa.test.ts index 28104c009..90001aea4 100644 --- a/src/lib/__tests__/require-fresh-mfa.test.ts +++ b/src/lib/__tests__/require-fresh-mfa.test.ts @@ -5,6 +5,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; vi.mock("@/lib/db", () => ({ prisma: { session: { findUnique: vi.fn() }, + user: { findUnique: vi.fn() }, webauthnMfaCredential: { count: vi.fn() }, passkey: { count: vi.fn() }, }, @@ -50,7 +51,15 @@ beforeEach(() => { vi.resetAllMocks(); }); -function mockSession(user: unknown) { +function mockSession(user: { + id: string; + totpConfirmedAt: Date | null; + passwordHash?: string | null; +}) { + // The password hash is read off the stored row, not the session user. + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + passwordHash: user.passwordHash ?? null, + } as never); vi.mocked(getSession).mockResolvedValue({ session: { id: "sess-1", expiresAt: new Date(Date.now() + 1e6) }, user, diff --git a/src/lib/__tests__/require-recent-proof.test.ts b/src/lib/__tests__/require-recent-proof.test.ts index 8c0d09fd1..136797db7 100644 --- a/src/lib/__tests__/require-recent-proof.test.ts +++ b/src/lib/__tests__/require-recent-proof.test.ts @@ -72,13 +72,17 @@ beforeEach(() => { }); function cookie( - user: unknown, + user: { id: string; passwordHash?: string | null }, stamps: { mfaVerifiedAt?: Date | null; createdAt?: Date; reproofAt?: Date | null; }, ) { + // The password hash is read off the stored row, not the session user. + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + passwordHash: user.passwordHash ?? null, + } as never); vi.mocked(getSession).mockResolvedValue({ session: { id: "sess-1", expiresAt: new Date(Date.now() + 1e6) }, user, diff --git a/src/lib/api-handler.ts b/src/lib/api-handler.ts index 5f0f9c9fa..35f907691 100644 --- a/src/lib/api-handler.ts +++ b/src/lib/api-handler.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import { headers } from "next/headers"; -import type { AccountGrant, User } from "@/generated/prisma/client"; +import type { AccountGrant } from "@/generated/prisma/client"; +import { SESSION_USER_SELECT, type SessionUser } from "./auth/session-user"; import { applyBypassRouteHeaders, bypassRouteRefusal, @@ -472,7 +473,7 @@ export type AuthContext = { */ readonly recordEpoch?: number; }; - user: User; + user: SessionUser; /** * Transport the caller authenticated with, derived from the SAME branch that * sets the wide event's `auth_method`: the cookie-session path resolves @@ -886,7 +887,7 @@ export async function requireActorAuth(): Promise { */ export interface RecordAuthContext extends AuthContext { /** The authenticated caller. Equals `user` when the caller acts as themselves. */ - readonly actor: User; + readonly actor: SessionUser; /** The grant being exercised, or null when the caller acts as themselves. */ readonly grantId: string | null; } @@ -1125,7 +1126,7 @@ export async function requireGuardianAuth(): Promise { async function resolveSwitchedRecord( auth: AuthContext, carrier: Extract, - admit: (grant: AccountGrant, owner: User) => string | null, + admit: (grant: AccountGrant, owner: SessionUser) => string | null, ): Promise { /** Every refusal below is this one error, with the reason kept off the wire. */ const denied = (reason: string): SharingAccessDeniedError => { @@ -1155,6 +1156,7 @@ async function resolveSwitchedRecord( const owner = await prisma.user.findUnique({ where: { id: grant.grantorId }, + select: SESSION_USER_SELECT, }); // Unreachable while the FK cascade holds — a grant naming a deleted account // is deleted with it. Fail closed anyway: an authorization we cannot resolve @@ -1584,7 +1586,7 @@ export async function requireRecentProof(options: { * caller themselves (the admin routes, which must stay on `requireAdmin`). */ export async function assertRecentCookieProof( - user: User, + user: SessionUser, sessionId: string, ): Promise { const enrolled = await hasSecondFactorEnrolled(user); @@ -1713,7 +1715,7 @@ export const STEP_UP_ELEVATION_HEADER = "x-step-up"; * own device login. A comment would not have caught that; a type does. */ export type MfaManagementContext = { - user: User; + user: SessionUser; /** * Spend the elevation. Call it immediately BEFORE the mutation, once every * cheap validation has passed — a 429, a 422, or a wrong factor code must not @@ -1837,7 +1839,7 @@ export async function requireMfaManagementAuth( * the wire. */ async function resolveBearerElevation( - auth: { user: User; apiTokenId: string }, + auth: { user: SessionUser; apiTokenId: string }, options: { freshFactor: boolean; proofSource: FreshFactorProofSource }, ): Promise<() => Promise> { const { freshFactor, proofSource } = options; diff --git a/src/lib/auth/__tests__/enrollment-proof-second-factor.test.ts b/src/lib/auth/__tests__/enrollment-proof-second-factor.test.ts index 0caecb261..a6dac826c 100644 --- a/src/lib/auth/__tests__/enrollment-proof-second-factor.test.ts +++ b/src/lib/auth/__tests__/enrollment-proof-second-factor.test.ts @@ -14,6 +14,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; vi.mock("@/lib/db", () => ({ prisma: { session: { findUnique: vi.fn(), update: vi.fn() }, + user: { findUnique: vi.fn() }, webauthnMfaCredential: { count: vi.fn() }, passkey: { count: vi.fn() }, authChallenge: { findUnique: vi.fn() }, @@ -38,7 +39,10 @@ vi.mock("@/lib/auth/mfa/webauthn", () => ({ verifyMfaAuthentication: vi.fn(), createMfaAuthenticationOptions: vi.fn(), })); -vi.mock("@/lib/auth/mfa/verify-factor", () => ({ verifyMfaFactor: vi.fn() })); +vi.mock("@/lib/auth/mfa/verify-factor", () => ({ + verifyMfaFactor: vi.fn(), + loadFactorUser: vi.fn(), +})); vi.mock("@/lib/api-handler", () => ({ apiHandler: (fn: unknown) => fn, requireCookieAuth: vi.fn(), @@ -47,7 +51,7 @@ vi.mock("@/lib/api-handler", () => ({ import { prisma } from "@/lib/db"; import { checkRateLimit, refundRateLimit } from "@/lib/rate-limit"; import { verifyPassword } from "@/lib/auth/password"; -import { verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; +import { loadFactorUser, verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; import { createRegistrationOptions } from "@/lib/auth/passkey"; import { requireCookieAuth } from "@/lib/api-handler"; import { @@ -79,6 +83,21 @@ const ENROLLED: { }; const PLAIN = { ...ENROLLED, totpSecretEncrypted: null, totpConfirmedAt: null }; +/** + * The account as the request sees it: its secrets live in the stored row, + * which the verifiers read for themselves, and not on the session user. + */ +function stored(user: typeof ENROLLED | typeof PLAIN) { + vi.mocked(prisma.user.findUnique).mockResolvedValue(user as never); + vi.mocked(loadFactorUser).mockResolvedValue(user as never); + return { + id: user.id, + username: user.username, + email: user.email, + totpConfirmedAt: user.totpConfirmedAt, + }; +} + function req(body?: unknown): Request { return new Request("http://localhost/api/auth/me/mfa/totp/setup", { method: "POST", @@ -101,7 +120,7 @@ function session(stamps: { function gate(user: typeof ENROLLED | typeof PLAIN, request: Request) { return checkCookieEnrollmentProof({ - user: user as never, + user: stored(user), sessionId: "sess-1", request, ipAddress: "203.0.113.9", @@ -198,7 +217,7 @@ describe("POST /api/auth/passkey/register-options", () => { it("refuses a password on an account with a second factor, before verifying it", async () => { vi.mocked(requireCookieAuth).mockResolvedValue({ - user: ENROLLED, + user: stored(ENROLLED), session: { id: "sess-1" }, } as never); const res = await PASSKEY_REGISTER_OPTIONS( @@ -214,7 +233,7 @@ describe("POST /api/auth/passkey/register-options", () => { it("still takes a password on an account without one", async () => { vi.mocked(requireCookieAuth).mockResolvedValue({ - user: PLAIN, + user: stored(PLAIN), session: { id: "sess-1" }, } as never); vi.mocked(verifyPassword).mockResolvedValue(true); @@ -236,7 +255,7 @@ describe("changing the email address", () => { opts: { cookie?: boolean; password?: string | null } = {}, ) { return authorizeSensitiveChange({ - user: user as never, + user: stored(user), cookieSessionId: opts.cookie === false ? null : "sess-1", currentPassword: opts.password ?? null, ipAddress: "203.0.113.9", diff --git a/src/lib/auth/bearer.ts b/src/lib/auth/bearer.ts index 8abb7997f..3af3317fe 100644 --- a/src/lib/auth/bearer.ts +++ b/src/lib/auth/bearer.ts @@ -14,9 +14,9 @@ * HTTP path keeps its existing `auth.bearer.*` audit trail), so this layer adds * no I/O beyond the two reads the validation needs. */ -import type { User } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import { hashToken } from "@/lib/auth/hmac"; +import { SESSION_USER_SELECT, type SessionUser } from "@/lib/auth/session-user"; /** Why a Bearer token was rejected. Stable handles for audit details. */ export type BearerAuthReason = @@ -67,7 +67,7 @@ export class BearerAuthError extends Error { /** The validated identity behind a Bearer token. */ export interface BearerResolution { - user: User; + user: SessionUser; /** The `ApiToken` row id (the token-id half of the `:` binding). */ tokenId: string; /** The token's granted scopes (`["*"]` = wildcard). */ @@ -147,6 +147,7 @@ export async function resolveBearerToken( const user = await prisma.user.findUnique({ where: { id: apiToken.userId }, + select: SESSION_USER_SELECT, }); if (!user) { diff --git a/src/lib/auth/existing-factor-proof.ts b/src/lib/auth/existing-factor-proof.ts index d4d46ad08..a2e5d5fe5 100644 --- a/src/lib/auth/existing-factor-proof.ts +++ b/src/lib/auth/existing-factor-proof.ts @@ -16,7 +16,6 @@ * (same key, so its counter carries straight over). A shared bucket means a * guesser cannot multiply the budget by rotating between routes. */ -import type { User } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import { apiError } from "@/lib/api-response"; import { REPROOF_REQUIRED_CODE } from "@/lib/api-errors"; @@ -34,7 +33,7 @@ import { import { verifyPassword } from "@/lib/auth/password"; import { verifyAuthentication } from "@/lib/auth/passkey"; import { verifyMfaAuthentication } from "@/lib/auth/mfa/webauthn"; -import { verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; +import { loadFactorUser, verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; import { stepUpMintSchema, type StepUpMintRequest, @@ -93,6 +92,9 @@ export async function refundReproof(userId: string): Promise { export type ProofOutcome = { ok: true } | { ok: false; reason: string }; +/** The caller a proof is checked for: who they are and whether TOTP is live. */ +type ProofUser = { id: string; totpConfirmedAt: Date | null }; + /** * Verify a proof against the account. Never throws for a bad proof: a * malformed or expired WebAuthn challenge is a failed attempt, not a 500. @@ -102,24 +104,25 @@ export type ProofOutcome = { ok: true } | { ok: false; reason: string }; * A TOTP code is accepted only from a confirmed secret (`verifyMfaFactor`). */ export async function verifyExistingFactorProof( - user: Pick< - User, - | "id" - | "passwordHash" - | "totpSecretEncrypted" - | "totpLastStep" - | "totpConfirmedAt" - >, + user: ProofUser, proof: ExistingFactorProof, ): Promise { + // The secrets are read here, for this one check: the caller's row is the + // per-request projection, which carries none of them. if (proof.method === "password") { - if (!user.passwordHash) return { ok: false, reason: "no_password" }; - const ok = await verifyPassword(user.passwordHash, proof.password); + const row = await prisma.user.findUnique({ + where: { id: user.id }, + select: { passwordHash: true }, + }); + if (!row?.passwordHash) return { ok: false, reason: "no_password" }; + const ok = await verifyPassword(row.passwordHash, proof.password); return ok ? { ok } : { ok: false, reason: "bad_password" }; } if (proof.method === "totp") { - const result = await verifyMfaFactor(user, "totp", proof.code); + const factorUser = await loadFactorUser(user.id); + if (!factorUser) return { ok: false, reason: "bad_totp" }; + const result = await verifyMfaFactor(factorUser, "totp", proof.code); if (result.ok) return { ok: true }; return { ok: false, reason: result.replay ? "totp_replay" : "bad_totp" }; } @@ -230,7 +233,7 @@ export const RECENT_PROOF_MAX_AGE_MS = 5 * 60 * 1000; * throttled and audited through the shared re-proof bucket. */ export async function checkCookieEnrollmentProof(args: { - user: User; + user: ProofUser; sessionId: string; request: Request; ipAddress: string | null; @@ -373,14 +376,7 @@ export type SensitiveChangeProof = * of such an account; the shipped app does not change it. */ export async function authorizeSensitiveChange(args: { - user: Pick< - User, - | "id" - | "passwordHash" - | "totpSecretEncrypted" - | "totpLastStep" - | "totpConfirmedAt" - >; + user: ProofUser; /** The session row id on a cookie request; null on a Bearer request. */ cookieSessionId: string | null; currentPassword: string | null; diff --git a/src/lib/auth/mfa/verify-factor.ts b/src/lib/auth/mfa/verify-factor.ts index 36536af2c..eea2f834e 100644 --- a/src/lib/auth/mfa/verify-factor.ts +++ b/src/lib/auth/mfa/verify-factor.ts @@ -36,6 +36,28 @@ export interface FactorUser { totpConfirmedAt: Date | null; } +/** + * Load the factor columns for one check, at the moment of the check. + * + * The per-request user (`SESSION_USER_SELECT`) carries no secret material, so + * a route that verifies a factor for an already-authenticated caller reads the + * secret and its replay floor here. Reading them fresh is also the right + * answer for the floor: it is the value the guarded update below compares. + */ +export async function loadFactorUser( + userId: string, +): Promise { + return prisma.user.findUnique({ + where: { id: userId }, + select: { + id: true, + totpSecretEncrypted: true, + totpLastStep: true, + totpConfirmedAt: true, + }, + }); +} + export interface FactorResult { ok: boolean; /** TOTP only: the code matched the secret but its step was already used. */ diff --git a/src/lib/auth/second-factor.ts b/src/lib/auth/second-factor.ts index 78d5ea8e6..b28935755 100644 --- a/src/lib/auth/second-factor.ts +++ b/src/lib/auth/second-factor.ts @@ -1,4 +1,3 @@ -import type { User } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; /** @@ -53,19 +52,25 @@ export type RecentProofMethod = "password" | "totp" | "webauthn" | "passkey"; * endpoint cannot disagree. */ export async function recentProofMethods( - user: Pick, + user: { id: string; totpConfirmedAt: Date | null }, enrolled?: boolean, ): Promise { const hasSecond = enrolled ?? (await hasSecondFactorEnrolled(user)); - const [keys, passkeys] = await Promise.all([ + // Whether a password exists is read here rather than off the caller's row: + // the hash is not part of the per-request user (`SESSION_USER_SELECT`). + const [keys, passkeys, credential] = await Promise.all([ prisma.webauthnMfaCredential.count({ where: { userId: user.id } }), prisma.passkey.count({ where: { userId: user.id } }), + prisma.user.findUnique({ + where: { id: user.id }, + select: { passwordHash: true }, + }), ]); const methods: RecentProofMethod[] = []; if (hasSecond) { if (user.totpConfirmedAt) methods.push("totp"); if (keys > 0) methods.push("webauthn"); - } else if (user.passwordHash) { + } else if (credential?.passwordHash) { methods.push("password"); } if (passkeys > 0) methods.push("passkey"); diff --git a/src/lib/auth/session-user.ts b/src/lib/auth/session-user.ts new file mode 100644 index 000000000..d54cfeccd --- /dev/null +++ b/src/lib/auth/session-user.ts @@ -0,0 +1,126 @@ +import type { Prisma } from "@/generated/prisma/client"; + +/** + * The columns of `User` that ride on every authenticated request. + * + * Every cookie session and every Bearer token resolves its caller through one + * of the lookups in `session.ts` / `bearer.ts`, so whatever this projection + * names is read once per request, for every request the app serves. The row it + * replaced was the whole `users` record, and that record carries a profile + * picture of up to a few megabytes, the cached briefing text, the encrypted + * home location and air-quality history, and every stored integration + * credential. None of that is needed to answer "who is calling", and all of it + * was copied into a fresh buffer on each request. + * + * So the projection is an allowlist: small scalars that the request path reads + * (identity, role, locale and display preferences, the onboarding and MFA + * flags, the small preference documents). Anything else is read by the one + * route that needs it, with its own `select`, at the moment it needs it: + * + * - bytes and large text: `avatarBytes`, `insightsCachedText`, + * `insightsCachedTextEncrypted`, `homeLocationEncrypted`, + * `environmentAqHistoryJson`, `dawarichPrefsEncrypted`; + * - credentials and secrets: `passwordHash`, `totpSecretEncrypted`, + * `totpLastStep`, `telegramBotToken`, every `*Encrypted` integration and + * provider key, `insuranceNumberEncrypted`. + * + * `src/__tests__/session-user-projection-guard.test.ts` holds both halves: no + * caller lookup may load the full row, and none of the columns above may join + * this list. + */ +export const SESSION_USER_SELECT = { + id: true, + username: true, + email: true, + role: true, + oidcIssuer: true, + oidcSub: true, + createdAt: true, + updatedAt: true, + managedProfileAt: true, + heightCm: true, + dateOfBirth: true, + gender: true, + timezone: true, + homeTimezone: true, + homeSince: true, + environmentAirQualityEnabled: true, + codexTokenExpiresAt: true, + codexConnectedAt: true, + codexConnectionStatus: true, + useCentralCodex: true, + insightsPrivacyMode: true, + insightsCachedAt: true, + insightsCachedLocale: true, + insightsSnapshotHash: true, + insightsWarmFailedAt: true, + insightsBriefingRerollDate: true, + morningDigestRefreshedOn: true, + telegramChatId: true, + telegramEnabled: true, + nightscoutAllowPrivateHost: true, + dawarichAllowPrivateHost: true, + ouraScopeDenied: true, + stravaAthleteId: true, + stravaLastActivityAt: true, + stravaBackfillCompletedAt: true, + stravaBackfillResumeBefore: true, + locale: true, + thresholdsJson: true, + dashboardWidgetsJson: true, + insightsLayoutJson: true, + medicationListLayoutJson: true, + documentsLayoutJson: true, + moodTagLayoutJson: true, + glucoseUnit: true, + hasDiabetes: true, + labsLocalOcrEnabled: true, + unitPreference: true, + timeFormat: true, + dateFormat: true, + aiProvider: true, + aiModel: true, + aiBaseUrl: true, + aiCompatBaseUrl: true, + aiCompatModel: true, + aiProviderChain: true, + aiResponseTimeoutSeconds: true, + documentsAutoAiRead: true, + onboardingCompletedAt: true, + onboardingTourCompleted: true, + onboardingTourProgressJson: true, + disclaimerAcknowledgedAt: true, + disclaimerAcknowledgedVersion: true, + healthKitConfigJson: true, + healthKitLastSyncedAt: true, + healthKitLastSyncTrigger: true, + healthKitLastBackgroundSyncAt: true, + displayName: true, + lastReportPracticeName: true, + coachPrefsJson: true, + sourcePriorityJson: true, + lastSyncedAt: true, + syncResetAt: true, + moodReminderEnabled: true, + reportSelectionJson: true, + notificationPrefs: true, + disableCoach: true, + aiUsageLevel: true, + modulePreferencesJson: true, + healthScoreConfigJson: true, + documentQuotaBytes: true, + coachLastSeenAt: true, + avatarContentType: true, + avatarUpdatedAt: true, + totpConfirmedAt: true, + mfaEnforced: true, + passkeyUpgradeNudgeDismissed: true, + fullName: true, + insurerName: true, + insurerIkNumber: true, +} as const satisfies Prisma.UserSelect; + +/** The caller as every authenticated request sees them. */ +export type SessionUser = Prisma.UserGetPayload<{ + select: typeof SESSION_USER_SELECT; +}>; diff --git a/src/lib/auth/session.ts b/src/lib/auth/session.ts index 72dc0e0af..06ec6381b 100644 --- a/src/lib/auth/session.ts +++ b/src/lib/auth/session.ts @@ -1,8 +1,8 @@ import { randomBytes } from "node:crypto"; import { prisma } from "@/lib/db"; import { cookies } from "next/headers"; -import type { User } from "@/generated/prisma/client"; import { hashToken } from "@/lib/auth/hmac"; +import { SESSION_USER_SELECT, type SessionUser } from "@/lib/auth/session-user"; import { ensureDbCompatibility } from "@/lib/db-compat"; import { getEvent } from "@/lib/logging/context"; import { shouldEmitSecureCookie } from "@/lib/auth/secure-cookie"; @@ -99,13 +99,13 @@ async function findSessionByCookie(cookieValue: string) { if (cookieValue.startsWith(SESSION_TOKEN_PREFIX)) { return await prisma.session.findUnique({ where: { tokenHash: hashToken(cookieValue) }, - include: { user: true }, + include: { user: { select: SESSION_USER_SELECT } }, }); } const legacy = await prisma.session.findUnique({ where: { id: cookieValue }, - include: { user: true }, + include: { user: { select: SESSION_USER_SELECT } }, }); // A row that already holds a secret has retired its id as a credential. return legacy && legacy.tokenHash === null ? legacy : null; @@ -148,7 +148,10 @@ async function findSessionByCookie(cookieValue: string) { */ export async function validateSessionFromCookieValue( cookieValue: string | undefined | null, -): Promise<{ session: { id: string; expiresAt: Date }; user: User } | null> { +): Promise<{ + session: { id: string; expiresAt: Date }; + user: SessionUser; +} | null> { if (!cookieValue) return null; const session = await findSessionByCookie(cookieValue); if (!session || session.expiresAt < new Date()) return null; @@ -179,7 +182,7 @@ export async function validateSessionWithCreatedAt( cookieValue: string | undefined | null, ): Promise<{ session: { id: string; createdAt: Date; expiresAt: Date }; - user: User; + user: SessionUser; } | null> { if (!cookieValue) return null; const session = await findSessionByCookie(cookieValue); @@ -296,7 +299,7 @@ export async function getSession(): Promise<{ actingAsUserId: string | null; recordEpoch: number; }; - user: User; + user: SessionUser; } | null> { try { await ensureDbCompatibility(); diff --git a/src/lib/daily/daily-briefing-push.ts b/src/lib/daily/daily-briefing-push.ts index de858255f..1fa56dab9 100644 --- a/src/lib/daily/daily-briefing-push.ts +++ b/src/lib/daily/daily-briefing-push.ts @@ -26,7 +26,7 @@ * on `MEASUREMENT_ANOMALY`, never here. It fires ONLY inside a local morning * window, which is the quiet-hours guarantee for this nudge. */ -import type { User } from "@/generated/prisma/client"; +import { SESSION_USER_SELECT, type SessionUser } from "@/lib/auth/session-user"; import type { PrismaClient } from "@/generated/prisma/client"; import { annotate, getEvent } from "@/lib/logging/context"; @@ -82,11 +82,15 @@ export type DailyBriefingDispatchResult = export interface DailyBriefingDispatchDeps { dispatch?: typeof dispatchNotification; - loadDigest?: (user: User, now: Date, locale: Locale) => Promise; + loadDigest?: ( + user: SessionUser, + now: Date, + locale: Locale, + ) => Promise; } function loadDailyBriefingDigest( - user: User, + user: SessionUser, now: Date, locale: Locale, ): Promise { @@ -181,7 +185,10 @@ export async function maybeDispatchDailyBriefing( const loadDigest = deps.loadDigest ?? loadDailyBriefingDigest; try { - const user = await prisma.user.findUnique({ where: { id: userId } }); + const user = await prisma.user.findUnique({ + where: { id: userId }, + select: SESSION_USER_SELECT, + }); if (!user) { annotate({ action: { name: "daily.briefing_push.missing_user" } }); return "missing-user"; diff --git a/src/lib/daily/load-digest.ts b/src/lib/daily/load-digest.ts index d08043635..a79699426 100644 --- a/src/lib/daily/load-digest.ts +++ b/src/lib/daily/load-digest.ts @@ -12,7 +12,7 @@ * module-gated at the build layer) plus the resolved module map the item * builders consult. */ -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; @@ -547,7 +547,7 @@ export interface DailyDigestLoadOptions { } export async function loadDailyDigest( - user: User, + user: SessionUser, now: Date = new Date(), options: DailyDigestLoadOptions = {}, ): Promise { diff --git a/src/lib/dashboard/__tests__/snapshot-read-briefing-capability.test.ts b/src/lib/dashboard/__tests__/snapshot-read-briefing-capability.test.ts index 5c16fa762..ebe57a1d8 100644 --- a/src/lib/dashboard/__tests__/snapshot-read-briefing-capability.test.ts +++ b/src/lib/dashboard/__tests__/snapshot-read-briefing-capability.test.ts @@ -11,7 +11,7 @@ */ import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; import { AI_AVAILABLE, aiUnavailable, @@ -29,7 +29,19 @@ const briefingBody = { const buildDashboardSnapshot = vi.fn(async () => briefingBody); const aiCapabilityToServe = vi.fn(); -vi.mock("@/lib/db", () => ({ prisma: {} })); +// Generated a moment ago, so the today rule keeps it and these tests see +// only the capability decision. Read off its own row, not the session user. +vi.mock("@/lib/db", () => ({ + prisma: { + user: { + findUnique: async () => ({ + insightsCachedText: JSON.stringify({ + briefingGeneratedAt: new Date().toISOString(), + }), + }), + }, + }, +})); vi.mock("@/lib/dashboard/snapshot", async (importOriginal) => ({ ...(await importOriginal()), buildDashboardSnapshot: () => buildDashboardSnapshot(), @@ -44,16 +56,11 @@ vi.mock("@/lib/ai/capabilities/gate", () => ({ const { readDashboardSnapshotCached } = await import("../snapshot-read"); const { __resetAllCachesForTests } = await import("@/lib/cache/server-cache"); -// Generated a moment ago, so the today rule keeps it and these tests see -// only the capability decision. const USER = { id: "user-briefing-gate", locale: "en", timezone: "UTC", - insightsCachedText: JSON.stringify({ - briefingGeneratedAt: new Date().toISOString(), - }), -} as unknown as User; +} as unknown as SessionUser; beforeEach(() => { __resetAllCachesForTests(); diff --git a/src/lib/dashboard/__tests__/snapshot-read-briefing-today.test.ts b/src/lib/dashboard/__tests__/snapshot-read-briefing-today.test.ts index 9be475763..7cf124e4a 100644 --- a/src/lib/dashboard/__tests__/snapshot-read-briefing-today.test.ts +++ b/src/lib/dashboard/__tests__/snapshot-read-briefing-today.test.ts @@ -5,7 +5,7 @@ */ import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; import { AI_AVAILABLE, aiUnavailable, @@ -47,7 +47,13 @@ const body = (pulseLastSeen: string) => ({ let current = body("2026-10-04T06:50:00.000Z"); const buildDashboardSnapshot = vi.fn(async () => current); -vi.mock("@/lib/db", () => ({ prisma: {} })); +// The cached briefing text is read off its own row, not the session user. +const cacheRow = vi.hoisted(() => ({ + current: null as { insightsCachedText: string | null } | null, +})); +vi.mock("@/lib/db", () => ({ + prisma: { user: { findUnique: async () => cacheRow.current } }, +})); vi.mock("@/lib/dashboard/snapshot", async (importOriginal) => ({ ...(await importOriginal()), buildDashboardSnapshot: () => buildDashboardSnapshot(), @@ -72,16 +78,18 @@ const { __resetAllCachesForTests } = await import("@/lib/cache/server-cache"); /** 2026-10-05 21:00 in Berlin. */ const NOW = new Date("2026-10-05T19:00:00.000Z"); -function user(generatedAt: string | null): User { - return { - id: "user-briefing-today", - locale: "en", - timezone: "Europe/Berlin", +function user(generatedAt: string | null): SessionUser { + cacheRow.current = { insightsCachedText: JSON.stringify({ dailyBriefing: BRIEFING, ...(generatedAt ? { briefingGeneratedAt: generatedAt } : {}), }), - } as unknown as User; + }; + return { + id: "user-briefing-today", + locale: "en", + timezone: "Europe/Berlin", + } as unknown as SessionUser; } beforeEach(() => { diff --git a/src/lib/dashboard/__tests__/snapshot-read-cache-key.test.ts b/src/lib/dashboard/__tests__/snapshot-read-cache-key.test.ts index 317455e16..5abc61ed3 100644 --- a/src/lib/dashboard/__tests__/snapshot-read-cache-key.test.ts +++ b/src/lib/dashboard/__tests__/snapshot-read-cache-key.test.ts @@ -14,12 +14,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; const buildDashboardSnapshot = vi.fn(async () => ({ tiles: {} })); const resolveServerLocale = vi.fn(async () => "de"); -vi.mock("@/lib/db", () => ({ prisma: {} })); +vi.mock("@/lib/db", () => ({ + prisma: { user: { findUnique: async () => null } }, +})); vi.mock("@/lib/dashboard/snapshot", () => ({ buildDashboardSnapshot: (...args: unknown[]) => buildDashboardSnapshot(...(args as [])), @@ -46,7 +48,7 @@ const { invalidateUserMedications, } = await import("@/lib/cache/invalidate"); -const USER = { id: "user-key-guard", locale: "de" } as unknown as User; +const USER = { id: "user-key-guard", locale: "de" } as unknown as SessionUser; beforeEach(() => { __resetAllCachesForTests(); diff --git a/src/lib/dashboard/snapshot-read.ts b/src/lib/dashboard/snapshot-read.ts index 23857a904..cd98809e0 100644 --- a/src/lib/dashboard/snapshot-read.ts +++ b/src/lib/dashboard/snapshot-read.ts @@ -21,7 +21,7 @@ * that moves the registry back off the global breaks this paragraph, not * just a performance assumption. */ -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; @@ -41,7 +41,10 @@ import { briefingForToday } from "@/lib/daily/briefing-today"; import { requestTodayBriefingWarm } from "@/lib/daily/briefing-today-warm"; import { readBriefingGeneratedAt } from "@/lib/insights/briefing-generated-at"; import { userDayKey } from "@/lib/tz/format"; -import { openInsightsCachedText } from "@/lib/insights/cached-text-cipher"; +import { + INSIGHTS_CACHE_SELECT, + openInsightsCachedText, +} from "@/lib/insights/cached-text-cipher"; /** * Per-key TTL for the snapshot cache entry. Strictly greater than the @@ -118,12 +121,29 @@ export interface SnapshotReadOptions { * timing surface; the RSC prefetch passes none. */ export async function readDashboardSnapshotCached( - user: User, + user: SessionUser, time?: (label: string, builder: () => Promise) => Promise, options: SnapshotReadOptions = {}, ): Promise { - // `User` row is already in hand at both call sites — no extra round-trip. - const cachedText = openInsightsCachedText(user); + // The cached briefing text is not on the per-request user + // (`SESSION_USER_SELECT`), so it is read here, beside the locale: every + // read needs it for the generated-at stamp below, not only a cache miss. + const [cacheRow, locale] = await Promise.all([ + prisma.user.findUnique({ + where: { id: user.id }, + select: INSIGHTS_CACHE_SELECT, + }), + // v1.21.2 — the briefing recall + forward-look is locale-specific + // prose, so the snapshot cache key carries the resolved locale: an EN and + // a DE session never share a snapshot cell (and never see each other's + // memory wording). The `${user.id}|` prefix still covers the key under + // the measurement-write stale-sweep. + resolveServerLocale({ + userLocale: user.locale, + override: options.locale ?? null, + }), + ]); + const cachedText = openInsightsCachedText(cacheRow); const snapshotUser: SnapshotUserInput = { id: user.id, username: user.username, @@ -143,16 +163,6 @@ export async function readDashboardSnapshotCached( healthScoreConfigJson: user.healthScoreConfigJson, }; - // v1.21.2 — the briefing recall + forward-look is locale-specific - // prose, so the snapshot cache key carries the resolved locale: an EN and - // a DE session never share a snapshot cell (and never see each other's - // memory wording). The `${user.id}|` prefix still covers the key under - // the measurement-write stale-sweep. - const locale = await resolveServerLocale({ - userLocale: user.locale, - override: options.locale ?? null, - }); - // Stale-while-revalidate: a measurement write marks the analytics // bucket stale rather than hard-evicting the snapshot, so a busy iOS // sync serves the prior snapshot immediately (within the bucket's diff --git a/src/lib/insights/feature-blocks.ts b/src/lib/insights/feature-blocks.ts index c9b75c195..8e5af6078 100644 --- a/src/lib/insights/feature-blocks.ts +++ b/src/lib/insights/feature-blocks.ts @@ -17,11 +17,17 @@ import { foldMeanSql, isHourlyMeanTypeSql, } from "@/lib/measurements/day-mean"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { + loadUserSourcePriority, + utcBucketEnd, +} from "@/lib/rollups/measurement-read"; import { classifyAgainstEffectiveRange } from "@/lib/labs/reference-range"; import { calendarDaysUntil } from "@/lib/measurement-reminders/due-day"; import { resolveLabFields } from "@/lib/labs/serialise"; -import { readRollupBuckets } from "@/lib/rollups/measurement-rollups"; +import { + readRollupBuckets, + ROLLUP_FOLD_WINDOW_MS, +} from "@/lib/rollups/measurement-rollups"; import { deriveBucketedTypes } from "@/lib/signals/adapters/correlation"; import type { AggregatedFeatures, @@ -40,11 +46,10 @@ interface AllTimeExtremes { } /** - * v1.18.11 — full-history min / max / mean per measurement type via ONE - * grouped SQL aggregation, with NO row materialisation in JS. Fills the - * `allTime*` feature fields; the windowed `summarize()` covers trends + recent - * windows, and this covers the long-horizon figures the prompt labels - * "allTime". + * v1.18.11 — full-history min / max / mean per measurement type, folded in + * SQL with NO row materialisation in JS. Fills the `allTime*` feature fields; + * the windowed `summarize()` covers trends + recent windows, and this covers + * the long-horizon figures the prompt labels "allTime". * * One source per (type, day): each source is folded per day first, each day * keeps its ladder-canonical source, then the days fold into the figures — @@ -52,6 +57,26 @@ interface AllTimeExtremes { * row let a device that samples densely (a watch's pulse) outweigh the * readings the source ladder prefers on the same days. * + * Where the days come from. A day is the same per-source fold either way — + * count, sum, the extremes of the readings and, for an hourly-mean type, the + * day value — and the DAY rollup tier already holds exactly that per (type, + * source, day), recomputed on every write. Reading the raw rows instead meant + * two passes over every reading of these types the account ever recorded on + * every call, the largest read on the database. So the figures fold: + * + * - the DAY tier, for every whole UTC day from `tierFrom` on, for each type + * whose tier rows in that range all carry the reading extremes + * (migration 0417; the boot re-fold fills older rows) and account for + * the live record day by day (same days, same reading counts); + * - the raw rows for the rest: the days before `tierFrom` (older than the + * fold window, or the partial day the `floor` falls in), and every day of + * a type the tier does not cover yet or no longer matches. + * + * The split is at a UTC midnight, so no day is folded from both, and the + * canonical pick per day is the same on both sides. The tier's pulse day + * value follows the stored day-mean method; the raw fold follows this + * process's mirror of it, which trails a change by at most a minute. + * * Only the four types that expose `allTime*` fields are aggregated (weight, * systolic, diastolic, pulse). Returns a map keyed by `MeasurementType`; a type * with no rows is simply absent. @@ -67,15 +92,116 @@ export async function readAllTimeExtremes( ): Promise> { const out = new Map(); if (types.length === 0) return out; + // The first whole UTC day the tier may answer: after the fold window's + // edge (the fold writes nothing older, and the edge day may be partial) + // and after the floor's own day (only part of it is inside the floor). + const windowEdge = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); + const tierFrom = utcBucketEnd( + floor !== null && floor > windowEdge ? floor : windowEdge, + "DAY", + ); + const typeList = (list: readonly MeasurementType[]) => + Prisma.join(list.map((t) => Prisma.sql`${t}::measurement_type`)); + + const [priorityJson, tierRows] = await Promise.all([ + loadUserSourcePriority(userId), + // A type is read from the tier only when the tier accounts for the live + // record day by day: every day with a live reading has DAY rows whose + // counts add up to that day's live count, and every DAY row has its day + // of live readings. A bucket that outlived its rows (a hard delete with no + // rollup recompute) or a day the tier never got fails the comparison and + // the type is read raw. The figures from here are the anchor the Coach + // snapshot checks its coarse history against, so they must never repeat + // a stale bucket. The live side is a count per day off the partial + // `(user_id, type, measured_at)` live index, not a read of the rows. + prisma.$queryRaw>` + WITH live AS ( + SELECT m."type", date_trunc('day', m."measured_at") AS day, + COUNT(*)::bigint AS n + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList(types)}) + AND m."measured_at" >= ${tierFrom} + GROUP BY 1, 2 + ), + tier AS ( + SELECT r."type", r."bucket_start" AT TIME ZONE 'UTC' AS day, + SUM(r."count")::bigint AS n, + bool_and( + r."reading_min_value" IS NOT NULL + AND r."reading_max_value" IS NOT NULL + ) AS complete + FROM measurement_rollups r + WHERE r."user_id" = ${userId} + AND r."granularity" = 'DAY' + AND r."type" IN (${typeList(types)}) + AND r."bucket_start" >= ${tierFrom} + GROUP BY 1, 2 + ) + SELECT COALESCE(l."type", t."type")::text AS type + FROM live l + FULL JOIN tier t ON t."type" = l."type" AND t.day = l.day + GROUP BY 1 + HAVING bool_and(COALESCE(l.n = t.n AND t.complete, false)) + `, + ]); + const tierTypes = new Set(tierRows.map((r) => r.type)); + const fromTier = types.filter((t) => tierTypes.has(t)); + const rawOnly = types.filter((t) => !tierTypes.has(t)); + const since = floor === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${floor}`; - const priorityJson = await loadUserSourcePriority(userId); - const rank = Prisma.raw( - buildSourceRankCase(priorityJson, 'p."type"', 'p."source"'), - ); - const typeList = Prisma.join( - types.map((t) => Prisma.sql`${t}::measurement_type`), - ); + const rawArms: Prisma.Sql[] = []; + if (rawOnly.length > 0) { + rawArms.push(Prisma.sql` + SELECT m."type", m."source", m."measured_at", m."value" + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList(rawOnly)}) + ${since}`); + } + if (fromTier.length > 0) { + rawArms.push(Prisma.sql` + SELECT m."type", m."source", m."measured_at", m."value" + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList(fromTier)}) + ${since} + AND m."measured_at" < ${tierFrom}`); + } + const rank = (alias: string) => + Prisma.raw( + buildSourceRankCase(priorityJson, `${alias}."type"`, `${alias}."source"`), + ); + // The tier's canonical days, in the shape of the raw fold's: the day's + // reading-weighted sum and count, its reading extremes, and its day value + // with a weight of one (a DAY row's `mean` is the day value for an + // hourly-mean type). + const tierCanon = + fromTier.length === 0 + ? Prisma.empty + : Prisma.sql` + UNION ALL + SELECT * FROM ( + SELECT DISTINCT ON (r."type", r."bucket_start") + r."type", + r."count" AS cnt, + COALESCE(r."sum_value", r."count" * r."mean")::double precision + AS total, + r."reading_min_value" AS min_value, + r."reading_max_value" AS max_value, + r."mean" AS wsum, + 1.0::double precision AS wdays + FROM measurement_rollups r + WHERE r."user_id" = ${userId} + AND r."granularity" = 'DAY' + AND r."type" IN (${typeList(fromTier)}) + AND r."bucket_start" >= ${tierFrom} + ORDER BY r."type", r."bucket_start", (${rank("r")}), r."source"::text + ) tier`; // A day of pulse is the mean of its hours' means and the full history the // mean of its days (`day-mean.ts`). `weighted` carries, per source and day, // the day-weighted sum and the day's weight (one) for those types only; @@ -92,14 +218,10 @@ export async function readAllTimeExtremes( const rows = await prisma.$queryRaw< Array<{ type: string; mean: number; min: number; max: number }> >` - WITH hourly_src AS ( - SELECT m.* - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."deleted_at" IS NULL - AND m."type" IN (${typeList}) - ${since} - AND ${Prisma.raw(isHourlyMeanTypeSql('m."type"'))} + WITH live AS (${Prisma.join(rawArms, " UNION ALL ")}), + hourly_src AS ( + SELECT * FROM live l + WHERE ${Prisma.raw(isHourlyMeanTypeSql('l."type"'))} ), weighted AS ( SELECT @@ -113,34 +235,32 @@ export async function readAllTimeExtremes( ), per_source AS ( SELECT - m."type", - m."source", - date_trunc('day', m."measured_at") AS day, + l."type", + l."source", + date_trunc('day', l."measured_at") AS day, COUNT(*)::int AS cnt, - SUM(m."value")::double precision AS total, - MIN(m."value")::double precision AS min_value, - MAX(m."value")::double precision AS max_value - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."deleted_at" IS NULL - AND m."type" IN (${typeList}) - ${since} - GROUP BY m."type", m."source", 3 - ), - canon AS ( - SELECT DISTINCT ON (p."type", p.day) - p."type", p.cnt, p.total, p.min_value, p.max_value, w.wsum, w.wdays - FROM per_source p - LEFT JOIN weighted w - ON w."type" = p."type" AND w."source" = p."source" AND w.day = p.day - ORDER BY p."type", p.day, (${rank}), p."source"::text + SUM(l."value")::double precision AS total, + MIN(l."value")::double precision AS min_value, + MAX(l."value")::double precision AS max_value + FROM live l + GROUP BY l."type", l."source", 3 ) SELECT c."type"::text AS type, ${mean}::double precision AS mean, MIN(c.min_value)::double precision AS min, MAX(c.max_value)::double precision AS max - FROM canon c + FROM ( + SELECT * FROM ( + SELECT DISTINCT ON (p."type", p.day) + p."type", p.cnt, p.total, p.min_value, p.max_value, w.wsum, w.wdays + FROM per_source p + LEFT JOIN weighted w + ON w."type" = p."type" AND w."source" = p."source" AND w.day = p.day + ORDER BY p."type", p.day, (${rank("p")}), p."source"::text + ) raw + ${tierCanon} + ) c GROUP BY c."type" `; for (const r of rows) { diff --git a/src/lib/integrations/sync-verdict.ts b/src/lib/integrations/sync-verdict.ts index 5b911fc40..8a0863c19 100644 --- a/src/lib/integrations/sync-verdict.ts +++ b/src/lib/integrations/sync-verdict.ts @@ -342,21 +342,27 @@ export function classifyMetricFreshness( return samples.map((sample) => { // A type the app writes back to the phone (#1247): what is entered here // goes to Apple Health and never arrives back, so a person who keeps - // their readings here would otherwise watch the type go quiet. The newer - // of the two is what says whether the type is still in use. + // their readings here would otherwise watch the type go quiet. That + // reading is in Apple Health too, so the newer of the two is the type's + // newest reading: it is what the row shows and what the verdict reads. const written = enteredHere?.has(sample.type) ? { lastEnteredHereAt: enteredHere.get(sample.type) ?? null } : {}; - if (verdict !== "fresh" || SPARSE_METRIC_TYPES.has(sample.type)) { - return { ...sample, ...written, stale: false }; - } const seen = newest( toMillis(sample.lastSeenAt), toMillis(written.lastEnteredHereAt), ); - return { + const resolved = { ...sample, ...written, + lastSeenAt: + seen != null ? new Date(seen).toISOString() : sample.lastSeenAt, + }; + if (verdict !== "fresh" || SPARSE_METRIC_TYPES.has(sample.type)) { + return { ...resolved, stale: false }; + } + return { + ...resolved, stale: seen != null && nowMs - seen > METRIC_QUIET_AFTER_MS, }; }); diff --git a/src/lib/jobs/__tests__/geo-backfill.test.ts b/src/lib/jobs/__tests__/geo-backfill.test.ts index 0bd4bdec3..8a6b4a835 100644 --- a/src/lib/jobs/__tests__/geo-backfill.test.ts +++ b/src/lib/jobs/__tests__/geo-backfill.test.ts @@ -9,6 +9,7 @@ import { GEO_BACKFILL_BATCH_CAP, GEO_BACKFILL_CRON, GEO_BACKFILL_QUEUE, + GEO_BACKFILL_RETRY_DAYS, GEO_BACKFILL_WINDOW_DAYS, runGeoBackfill, } from "../geo-backfill"; @@ -65,6 +66,21 @@ describe("runGeoBackfill", () => { createdAt: { gt: new Date(now.getTime() - GEO_BACKFILL_WINDOW_DAYS * 86_400_000), }, + // A row tried less than the retry wait ago is not picked again. + AND: [ + { + OR: [ + { geoAttemptedAt: null }, + { + geoAttemptedAt: { + lt: new Date( + now.getTime() - GEO_BACKFILL_RETRY_DAYS * 86_400_000, + ), + }, + }, + ], + }, + ], }); expect(args?.take).toBe(GEO_BACKFILL_BATCH_CAP); }); @@ -91,7 +107,7 @@ describe("runGeoBackfill", () => { }); expect(prisma.auditLog.update).toHaveBeenCalledWith({ where: { id: "a1" }, - data: { location: "Berlin, DE" }, + data: { location: "Berlin, DE", geoAttemptedAt: expect.any(Date) }, }); }); @@ -117,6 +133,7 @@ describe("runGeoBackfill", () => { location: "München, DE", asn: 3320, carrier: "Deutsche Telekom AG", + geoAttemptedAt: expect.any(Date), }, }); }); @@ -136,7 +153,11 @@ describe("runGeoBackfill", () => { expect(summary.carrierResolved).toBe(1); expect(prisma.auditLog.update).toHaveBeenCalledWith({ where: { id: "a2b" }, - data: { location: "Bochum, DE", carrier: "Deutsche Telekom AG" }, + data: { + location: "Bochum, DE", + carrier: "Deutsche Telekom AG", + geoAttemptedAt: expect.any(Date), + }, }); }); @@ -158,15 +179,20 @@ describe("runGeoBackfill", () => { }); expect(prisma.auditLog.update).toHaveBeenCalledWith({ where: { id: "a3" }, - data: { carrier: "Vodafone GmbH", asn: 3209 }, + data: { + carrier: "Vodafone GmbH", + asn: 3209, + geoAttemptedAt: expect.any(Date), + }, }); }); - it("skips the update when the resolver misses entirely, counts as still-unresolved", async () => { + it("stamps only the attempt when the resolver misses entirely, counts as still-unresolved", async () => { const prisma = makePrismaMock([{ id: "a4", ipAddress: "192.0.2.1" }]); vi.mocked(lookupIpGeo).mockResolvedValueOnce(EMPTY_GEO); + const now = new Date("2026-05-15T12:00:00Z"); - const summary = await runGeoBackfill(prisma); + const summary = await runGeoBackfill(prisma, now); expect(summary).toEqual({ scanned: 1, @@ -174,7 +200,12 @@ describe("runGeoBackfill", () => { carrierResolved: 0, stillUnresolved: 1, }); - expect(prisma.auditLog.update).not.toHaveBeenCalled(); + // The try is recorded, so the next passes leave the row alone until + // the retry wait has passed. + expect(prisma.auditLog.update).toHaveBeenCalledWith({ + where: { id: "a4" }, + data: { geoAttemptedAt: now }, + }); }); it("handles a mixed batch of hits and misses", async () => { @@ -204,7 +235,7 @@ describe("runGeoBackfill", () => { carrierResolved: 2, stillUnresolved: 1, }); - expect(prisma.auditLog.update).toHaveBeenCalledTimes(2); + expect(prisma.auditLog.update).toHaveBeenCalledTimes(3); }); it("treats a Prisma update throw as still-unresolved without aborting the pass", async () => { diff --git a/src/lib/jobs/geo-backfill.ts b/src/lib/jobs/geo-backfill.ts index f0e7bd12d..919cb8a01 100644 --- a/src/lib/jobs/geo-backfill.ts +++ b/src/lib/jobs/geo-backfill.ts @@ -34,9 +34,16 @@ * * Idempotent: a row that the resolver still cannot match (private IP * sneaking through CF egress, freshly-allocated range the GeoLite2 - * release hasn't picked up yet) stays NULL and the next pass tries - * again. The cap is per-call, not per-row, so the helper can be - * scheduled at any cadence without doubling up. + * release hasn't picked up yet) stays NULL. Every row a pass looks at is + * stamped `geoAttemptedAt`, and a row stamped less than + * `GEO_BACKFILL_RETRY_DAYS` ago is not picked again: an address no + * database will ever place used to be looked up on every hourly pass for + * its whole 30-day window, and finding it walked every audit row of those + * 30 days. The wait still lets a range a later GeoLite2 release learns + * resolve on a later pass. The candidates are read through the partial + * index `audit_logs_geo_backfill_idx` (migration 0418), which holds only + * the rows still missing something. The cap is per-call, not per-row, so + * the helper can be scheduled at any cadence without doubling up. * * No notifications, no event ladder — the maintainer can read the * counts off the returned summary and decide whether to keep @@ -47,6 +54,8 @@ import { lookupIpGeo } from "@/lib/geo"; export const GEO_BACKFILL_BATCH_CAP = 500; export const GEO_BACKFILL_WINDOW_DAYS = 30; +/** How long a row the backfill could not complete waits for its next try. */ +export const GEO_BACKFILL_RETRY_DAYS = 7; /** * v1.4.37 — pg-boss queue name + cron expression for the recurring @@ -74,6 +83,9 @@ export async function runGeoBackfill( const cutoff = new Date( now.getTime() - GEO_BACKFILL_WINDOW_DAYS * 86_400_000, ); + const retryBefore = new Date( + now.getTime() - GEO_BACKFILL_RETRY_DAYS * 86_400_000, + ); const rows = await prisma.auditLog.findMany({ where: { @@ -85,6 +97,14 @@ export async function runGeoBackfill( OR: [{ location: null }, { carrier: null }], ipAddress: { not: null }, createdAt: { gt: cutoff }, + AND: [ + { + OR: [ + { geoAttemptedAt: null }, + { geoAttemptedAt: { lt: retryBefore } }, + ], + }, + ], }, select: { id: true, ipAddress: true }, orderBy: { createdAt: "desc" }, @@ -109,8 +129,14 @@ export async function runGeoBackfill( // ASN MMDB. The online fallback bounds itself with a request timeout. const { location, asn, carrier } = await lookupIpGeo(ip); - const data: { location?: string; asn?: number; carrier?: string | null } = - {}; + // Stamped whatever the outcome: a row that resolved leaves the candidate + // set anyway, and one that did not waits for its next try. + const data: { + location?: string; + asn?: number; + carrier?: string | null; + geoAttemptedAt: Date; + } = { geoAttemptedAt: now }; if (location) { data.location = location; summary.located += 1; @@ -121,10 +147,8 @@ export async function runGeoBackfill( summary.carrierResolved += 1; } - if (Object.keys(data).length === 0) { - summary.stillUnresolved += 1; - continue; - } + const resolved = data.location !== undefined || data.carrier !== undefined; + if (!resolved) summary.stillUnresolved += 1; try { await prisma.auditLog.update({ @@ -135,7 +159,7 @@ export async function runGeoBackfill( // The row may have been deleted by the retention sweeper between // findMany and update. Treat as unresolved; the next pass will // skip it because the row no longer exists. - summary.stillUnresolved += 1; + if (resolved) summary.stillUnresolved += 1; } } diff --git a/src/lib/jobs/reaction-line.ts b/src/lib/jobs/reaction-line.ts index 1051aae19..4ec108073 100644 --- a/src/lib/jobs/reaction-line.ts +++ b/src/lib/jobs/reaction-line.ts @@ -58,6 +58,7 @@ import { import { extractAssessmentSummary } from "@/lib/insights/status-shared"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { fenceUserText } from "@/lib/ai/coach/data-fence"; +import { SESSION_USER_SELECT } from "@/lib/auth/session-user"; import { singleUserTurn, type CompletionResult } from "@/lib/ai/types"; import type { Locale } from "@/lib/i18n/config"; import { resolveJobLocale } from "@/lib/i18n/job-locale"; @@ -500,7 +501,10 @@ export async function runReactionLine( }; } - const user = await prisma.user.findUnique({ where: { id: job.userId } }); + const user = await prisma.user.findUnique({ + where: { id: job.userId }, + select: SESSION_USER_SELECT, + }); if (!user) return { status: "skipped", reason: "no_user" }; const locale = await resolveJobLocale(user.locale); diff --git a/src/lib/observability/__tests__/heap-monitor.test.ts b/src/lib/observability/__tests__/heap-monitor.test.ts index 9e686a52d..9df0ae3bb 100644 --- a/src/lib/observability/__tests__/heap-monitor.test.ts +++ b/src/lib/observability/__tests__/heap-monitor.test.ts @@ -10,6 +10,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { getHeapSpaceStatistics } from "node:v8"; import { describe, expect, it, vi } from "vitest"; @@ -34,6 +35,7 @@ import { type HeapFigures, type HeapMonitorDeps, emitHeapLine, + heapSpaceFigures, isPrivateDirectory, writeDiagnosticHeapSnapshot, } from "../heap-monitor"; @@ -46,7 +48,15 @@ const FIGURE_KEYS: Array = [ "heap_total_mb", "heap_limit_mb", "heap_used_ratio", + "new_space_mb", "old_space_mb", + "code_space_mb", + "large_object_space_mb", + "new_large_object_space_mb", + "code_large_object_space_mb", + "trusted_space_mb", + "read_only_space_mb", + "other_spaces_mb", "rss_mb", "external_mb", "array_buffers_mb", @@ -125,6 +135,50 @@ describe("heap monitor sampling", () => { }); }); +describe("heap space breakdown", () => { + const MB = 1024 * 1024; + + it("reports each named space and sums the rest into other_spaces_mb", () => { + const figures = heapSpaceFigures([ + { space_name: "old_space", space_used_size: 300 * MB }, + { space_name: "large_object_space", space_used_size: 120 * MB }, + { space_name: "trusted_space", space_used_size: 40 * MB }, + { space_name: "shared_space", space_used_size: 2 * MB }, + { space_name: "a_space_v8_adds_later", space_used_size: 3 * MB }, + ]); + expect(figures).toEqual({ + new_space_mb: 0, + old_space_mb: 300, + code_space_mb: 0, + large_object_space_mb: 120, + new_large_object_space_mb: 0, + code_large_object_space_mb: 0, + trusted_space_mb: 40, + read_only_space_mb: 0, + other_spaces_mb: 5, + }); + }); + + it("accounts for every space this runtime reports", () => { + const spaces = getHeapSpaceStatistics(); + expect(spaces.length).toBeGreaterThan(0); + const figures = heapSpaceFigures(spaces); + expect(Object.keys(figures).sort()).toEqual( + FIGURE_KEYS.filter( + (k) => k.endsWith("space_mb") || k === "other_spaces_mb", + ) + .map(String) + .sort(), + ); + const total = spaces.reduce((sum, s) => sum + s.space_used_size, 0); + const reported = Object.values(figures).reduce((a, b) => a + b, 0); + // Each space is rounded to the megabyte on its own. + expect(Math.abs(reported - total / MB)).toBeLessThanOrEqual( + Object.keys(figures).length, + ); + }); +}); + describe("heap pressure report", () => { it("fires once at 85 % with the work in flight, at warn", () => { const h = harness(); diff --git a/src/lib/observability/heap-monitor.ts b/src/lib/observability/heap-monitor.ts index 362a9f7ae..5e049763d 100644 --- a/src/lib/observability/heap-monitor.ts +++ b/src/lib/observability/heap-monitor.ts @@ -72,12 +72,66 @@ export const HEAP_CRITICAL_RATIO = 0.95; export const HEAP_PRESSURE_COOLDOWN_MS = 30 * 60_000; export const HEAP_PRESSURE_LIST_MAX = 50; -export interface HeapFigures { +/** + * The used size of each V8 heap space, by the field it is reported under. + * + * `old_space` alone left most of a large heap unexplained: a buffer or string + * past V8's large-object threshold (a profile picture read into memory, a + * long JSON payload) lives in `large_object_space`, compiled code in the code + * spaces, and since V8 12 bytecode and other sandboxed objects in + * `trusted_space`. Every space a runtime reports that is not named here + * (the shared spaces, a space a later V8 adds) is summed into + * `other_spaces_mb`, so the fields account for the whole used heap and the + * line keeps one shape whatever the Node version. + */ +const HEAP_SPACE_FIELDS = { + new_space: "new_space_mb", + old_space: "old_space_mb", + code_space: "code_space_mb", + large_object_space: "large_object_space_mb", + new_large_object_space: "new_large_object_space_mb", + code_large_object_space: "code_large_object_space_mb", + trusted_space: "trusted_space_mb", + read_only_space: "read_only_space_mb", +} as const; + +type HeapSpaceField = + | (typeof HEAP_SPACE_FIELDS)[keyof typeof HEAP_SPACE_FIELDS] + | "other_spaces_mb"; + +export type HeapSpaceFigures = Record; + +/** The per-space breakdown of `getHeapSpaceStatistics()`, in megabytes. */ +export function heapSpaceFigures( + spaces: ReadonlyArray<{ space_name: string; space_used_size: number }>, +): HeapSpaceFigures { + const bytes: Record = { + new_space_mb: 0, + old_space_mb: 0, + code_space_mb: 0, + large_object_space_mb: 0, + new_large_object_space_mb: 0, + code_large_object_space_mb: 0, + trusted_space_mb: 0, + read_only_space_mb: 0, + other_spaces_mb: 0, + }; + for (const space of spaces) { + const field = + HEAP_SPACE_FIELDS[space.space_name as keyof typeof HEAP_SPACE_FIELDS] ?? + "other_spaces_mb"; + bytes[field] += space.space_used_size; + } + return Object.fromEntries( + Object.entries(bytes).map(([field, size]) => [field, megabytes(size)]), + ) as HeapSpaceFigures; +} + +export interface HeapFigures extends HeapSpaceFigures { heap_used_mb: number; heap_total_mb: number; heap_limit_mb: number; heap_used_ratio: number; - old_space_mb: number; rss_mb: number; external_mb: number; array_buffers_mb: number; @@ -115,9 +169,6 @@ function defaultReadHeap(): { used: number; limit: number } { function defaultReadFigures(registry: InFlightRegistry): HeapFigures { const memory = process.memoryUsage(); const limit = getHeapStatistics().heap_size_limit; - const oldSpace = getHeapSpaceStatistics().find( - (space) => space.space_name === "old_space", - ); const counts = registry.counts(); return { heap_used_mb: megabytes(memory.heapUsed), @@ -125,7 +176,7 @@ function defaultReadFigures(registry: InFlightRegistry): HeapFigures { heap_limit_mb: megabytes(limit), heap_used_ratio: limit > 0 ? Math.round((memory.heapUsed / limit) * 1000) / 1000 : 0, - old_space_mb: megabytes(oldSpace?.space_used_size ?? 0), + ...heapSpaceFigures(getHeapSpaceStatistics()), rss_mb: megabytes(memory.rss), external_mb: megabytes(memory.external), array_buffers_mb: megabytes(memory.arrayBuffers), diff --git a/src/lib/openapi/routes/integrations.ts b/src/lib/openapi/routes/integrations.ts index 9fc6ec356..9e36dfcb6 100644 --- a/src/lib/openapi/routes/integrations.ts +++ b/src/lib/openapi/routes/integrations.ts @@ -293,7 +293,7 @@ const appleHealthMetricFreshnessEntry = metricFreshnessEntry .nullable() .optional() .describe( - "v1.43.7 — present only on a type the app writes back to Apple Health (weight, blood pressure, blood glucose while their entry is `bidirectional` or `writeOnly` and enabled): when the newest reading of the type was entered here rather than arriving from Apple Health, or null when none was. Such readings go to the phone and never come back as an arrival, so `stale` already counts them: the type is quiet only when both this and `lastSeenAt` are. Show it as the server sends it; do not recompute `stale`.", + "v1.43.7 — present only on a type the app writes back to Apple Health (weight, blood pressure, blood glucose while their entry is `bidirectional` or `writeOnly` and enabled): when the newest reading of the type was entered here rather than arriving from Apple Health, or null when none was. Such readings go to the phone and never come back as an arrival, so `stale` already counts them: the type is quiet only when both are. Since v1.43.9 `lastSeenAt` on such a type is the newer of this and the newest reading from Apple Health, because a reading entered here is in Apple Health too; show both as the server sends them and do not recompute `stale`.", ), }) .meta({ diff --git a/src/lib/rollups/__tests__/measurement-rollups.test.ts b/src/lib/rollups/__tests__/measurement-rollups.test.ts index 3426575fa..69161ada6 100644 --- a/src/lib/rollups/__tests__/measurement-rollups.test.ts +++ b/src/lib/rollups/__tests__/measurement-rollups.test.ts @@ -188,6 +188,8 @@ describe("recomputeBucketsForMeasurement", () => { sd: 2.0, slope: -0.1, r2: 0.9, + reading_min_value: 80.5, + reading_max_value: 84.5, }, ]); getGlobalBossMock.mockReturnValue({ send: bossSend }); @@ -210,14 +212,14 @@ describe("recomputeBucketsForMeasurement", () => { // instead of raising (or being dropped). Param layout per row: // [userId, type, granularity, bucketStart, source, count, mean, // min, max, sum, sd, slope, r2, sumX, sumXy, sumXx, sumYy, - // computedAt]. + // readingMin, readingMax, computedAt]. expect(txExecuteRawUnsafe).toHaveBeenCalledTimes(1); const [sql, ...params] = txExecuteRawUnsafe.mock.calls[0]; expect(sql).toContain( "ON CONFLICT (user_id, type, granularity, bucket_start, source)", ); expect(sql).toContain("DO UPDATE SET"); - expect(params).toHaveLength(18); + expect(params).toHaveLength(20); expect(params[0]).toBe("user-1"); expect(params[1]).toBe("WEIGHT"); expect(params[2]).toBe("DAY"); @@ -227,6 +229,10 @@ describe("recomputeBucketsForMeasurement", () => { // v1.4.39 — sumValue flows through. Cumulative read paths // (steps, flights, distance, daylight, active-energy) consume it. expect(params[9]).toBe(247.5); + // The reading extremes, beside the range-widened min / max. + expect(params[7]).toBe(80.0); + expect(params[17]).toBe(80.5); + expect(params[18]).toBe(84.5); // WEEK / MONTH / YEAR — three enqueues against the worker queue. expect(bossSend).toHaveBeenCalledTimes(3); @@ -453,13 +459,13 @@ describe("persistRollupRows — large path (via recomputeUserRollups)", () => { expect(deleteMany.mock.calls[0][0].where.userId).toBe("user-1"); expect(deleteMany.mock.calls[0][0].where.granularity).toBe("DAY"); - // 3 chunks of 18 params per row, every one an ON CONFLICT DO UPDATE + // 3 chunks of 20 params per row, every one an ON CONFLICT DO UPDATE // upsert — tolerant of a mid-transaction hook insert AND of a // concurrent fold on the same partition (v1.28.33, issue #486). expect(txExecuteRawUnsafe).toHaveBeenCalledTimes(3); - expect(txExecuteRawUnsafe.mock.calls[0]).toHaveLength(1 + 500 * 18); - expect(txExecuteRawUnsafe.mock.calls[1]).toHaveLength(1 + 500 * 18); - expect(txExecuteRawUnsafe.mock.calls[2]).toHaveLength(1 + 200 * 18); + expect(txExecuteRawUnsafe.mock.calls[0]).toHaveLength(1 + 500 * 20); + expect(txExecuteRawUnsafe.mock.calls[1]).toHaveLength(1 + 500 * 20); + expect(txExecuteRawUnsafe.mock.calls[2]).toHaveLength(1 + 200 * 20); for (const call of txExecuteRawUnsafe.mock.calls) { expect(call[0]).toContain("ON CONFLICT"); expect(call[0]).toContain("DO UPDATE SET"); @@ -575,7 +581,7 @@ describe("persistRollupRows — large path (via recomputeUserRollups)", () => { expect(transaction).toHaveBeenCalledTimes(1); expect(typeof transaction.mock.calls[0][0]).toBe("function"); expect(txExecuteRawUnsafe).toHaveBeenCalledTimes(1); - expect(txExecuteRawUnsafe.mock.calls[0]).toHaveLength(1 + 2 * 18); + expect(txExecuteRawUnsafe.mock.calls[0]).toHaveLength(1 + 2 * 20); expect(result.rowsUpserted).toBe(2); }); }); diff --git a/src/lib/rollups/measurement-rollups.ts b/src/lib/rollups/measurement-rollups.ts index bfaf64b92..0a9eae21b 100644 --- a/src/lib/rollups/measurement-rollups.ts +++ b/src/lib/rollups/measurement-rollups.ts @@ -184,6 +184,10 @@ interface RollupRow { sum_xy: number | null; sum_xx: number | null; sum_yy: number | null; + // The extremes of the readings' own `value`, without the per-row range + // `min_value` / `max_value` widen to. See `MeasurementRollup.readingMinValue`. + reading_min_value: number | null; + reading_max_value: number | null; } /** @@ -703,7 +707,9 @@ async function runRollupAggregate( SUM(${REBASED_X})::double precision AS sum_x, SUM(${REBASED_X} * m."value")::double precision AS sum_xy, SUM(${REBASED_X} * ${REBASED_X})::double precision AS sum_xx, - SUM(m."value" * m."value")::double precision AS sum_yy + SUM(m."value" * m."value")::double precision AS sum_yy, + MIN(m."value")::double precision AS reading_min_value, + MAX(m."value")::double precision AS reading_max_value FROM measurements m WHERE m."user_id" = $1 AND m."type" IN (${typeList}) @@ -753,7 +759,9 @@ async function runRollupAggregate( SUM(${REBASED_X})::double precision AS sum_x, SUM(${REBASED_X} * m."value")::double precision AS sum_xy, SUM(${REBASED_X} * ${REBASED_X})::double precision AS sum_xx, - SUM(m."value" * m."value")::double precision AS sum_yy + SUM(m."value" * m."value")::double precision AS sum_yy, + MIN(m."value")::double precision AS reading_min_value, + MAX(m."value")::double precision AS reading_max_value FROM measurements m WHERE m."user_id" = $1 AND m."measured_at" >= $2 @@ -1089,20 +1097,24 @@ function buildRollupUpsert( row.sum_xy ?? null, row.sum_xx ?? null, row.sum_yy ?? null, + row.reading_min_value ?? null, + row.reading_max_value ?? null, computedAt, ); tuples.push( `($${base + 1}, $${base + 2}::"measurement_type", $${base + 3}::"measurement_rollup_granularity", ` + `$${base + 4}::timestamptz, $${base + 5}::"measurement_source", $${base + 6}::int, ` + `$${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, ` + - `$${base + 13}, $${base + 14}, $${base + 15}, $${base + 16}, $${base + 17}, $${base + 18}::timestamptz)`, + `$${base + 13}, $${base + 14}, $${base + 15}, $${base + 16}, $${base + 17}, $${base + 18}, ` + + `$${base + 19}, $${base + 20}::timestamptz)`, ); } const sql = ` INSERT INTO measurement_rollups (user_id, type, granularity, bucket_start, source, count, mean, min_value, max_value, sum_value, sd, slope, r2, - sum_x, sum_xy, sum_xx, sum_yy, computed_at) + sum_x, sum_xy, sum_xx, sum_yy, reading_min_value, reading_max_value, + computed_at) VALUES ${tuples.join(", ")} ON CONFLICT (user_id, type, granularity, bucket_start, source) DO UPDATE SET @@ -1118,6 +1130,8 @@ function buildRollupUpsert( sum_xy = EXCLUDED.sum_xy, sum_xx = EXCLUDED.sum_xx, sum_yy = EXCLUDED.sum_yy, + reading_min_value = EXCLUDED.reading_min_value, + reading_max_value = EXCLUDED.reading_max_value, computed_at = EXCLUDED.computed_at `; return { sql, params }; @@ -1369,7 +1383,9 @@ export async function enqueueBootTimeRollupBackfill( // catches up — mirrors the v1.4.39 sum_value rollout, which the // v1.4.41 note confirms converged via this same boot queue. Bounded // to the fold window so days outside it (un-foldable by design) - // never re-flag a user on every boot. + // never re-flag a user on every boot. The same arm refills the + // reading extremes of migration 0417 (`reading_min_value IS NULL`); + // the extremes reader treats a NULL as uncovered until it lands. const users = await prisma.$queryRaw>` SELECT DISTINCT id FROM ( SELECT DISTINCT mt."user_id" AS id @@ -1395,7 +1411,7 @@ export async function enqueueBootTimeRollupBackfill( FROM measurement_rollups r WHERE r."granularity" = 'DAY' AND r."bucket_start" >= ${foldWindowStart} - AND r."sum_xy" IS NULL + AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL) ) discovered `; diff --git a/tests/integration/all-time-extremes-tier-parity.test.ts b/tests/integration/all-time-extremes-tier-parity.test.ts new file mode 100644 index 000000000..c73ad99b0 --- /dev/null +++ b/tests/integration/all-time-extremes-tier-parity.test.ts @@ -0,0 +1,344 @@ +/** + * The full-history extremes read the DAY rollup tier and give the figures the + * raw-row fold gave. + * + * `readAllTimeExtremes` used to fold every raw reading of the four types on + * each call. It now folds the DAY tier for the whole days inside the fold + * window and the raw rows for the rest. The reference below is that earlier + * query, verbatim, so the comparison is against what the function used to + * answer, not against a second derivation of the same idea. + * + * The seed has what the two paths could disagree on: two sources on the same + * days (weight by scale and by hand, pulse from a watch's hourly statistics + * with their own range and a strap's dense samples), readings older than the + * fold window, readings either side of a UTC midnight, soft-deleted rows, and + * a reading deleted after the tier was built (through the write path's own + * rollup recompute). + * + * Means are compared to nine significant digits: the tier sums the same day + * values in a different order. Extremes are compared exactly. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { Prisma, type MeasurementType } from "@/generated/prisma/client"; +import { readAllTimeExtremes } from "@/lib/insights/feature-blocks"; +import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; +import { + dayWeightedRowsSql, + foldMeanSql, + isHourlyMeanTypeSql, +} from "@/lib/measurements/day-mean"; +import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; +import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; + +const prisma = getPrismaClient(); +const USER = "all-time-tier-parity"; +const TYPES: MeasurementType[] = [ + "WEIGHT", + "BLOOD_PRESSURE_SYS", + "BLOOD_PRESSURE_DIA", + "PULSE", +]; +const HOUR = 3_600_000; +const DAY = 24 * HOUR; +const today = Date.UTC( + new Date().getUTCFullYear(), + new Date().getUTCMonth(), + new Date().getUTCDate(), +); + +/** The query `readAllTimeExtremes` ran before it read the DAY tier. */ +async function referenceExtremes( + userId: string, + types: readonly MeasurementType[], + floor: Date | null, +) { + const out = new Map(); + const since = + floor === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${floor}`; + const priorityJson = await loadUserSourcePriority(userId); + const rank = Prisma.raw( + buildSourceRankCase(priorityJson, 'p."type"', 'p."source"'), + ); + const typeList = Prisma.join( + types.map((t) => Prisma.sql`${t}::measurement_type`), + ); + const mean = Prisma.raw( + foldMeanSql({ + typeColumn: 'c."type"', + total: "c.total", + count: "c.cnt", + weightedSum: "c.wsum", + weightSum: "c.wdays", + }), + ); + const rows = await prisma.$queryRaw< + Array<{ type: string; mean: number; min: number; max: number }> + >` + WITH hourly_src AS ( + SELECT m.* + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList}) + ${since} + AND ${Prisma.raw(isHourlyMeanTypeSql('m."type"'))} + ), + weighted AS ( + SELECT + w."type", + w."source", + date_trunc('day', w."measured_at") AS day, + SUM(w."value" * w.day_weight)::double precision AS wsum, + SUM(w.day_weight)::double precision AS wdays + FROM ${dayWeightedRowsSql("hourly_src", null, { bySource: true })} w + GROUP BY w."type", w."source", 3 + ), + per_source AS ( + SELECT + m."type", + m."source", + date_trunc('day', m."measured_at") AS day, + COUNT(*)::int AS cnt, + SUM(m."value")::double precision AS total, + MIN(m."value")::double precision AS min_value, + MAX(m."value")::double precision AS max_value + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList}) + ${since} + GROUP BY m."type", m."source", 3 + ), + canon AS ( + SELECT DISTINCT ON (p."type", p.day) + p."type", p.cnt, p.total, p.min_value, p.max_value, w.wsum, w.wdays + FROM per_source p + LEFT JOIN weighted w + ON w."type" = p."type" AND w."source" = p."source" AND w.day = p.day + ORDER BY p."type", p.day, (${rank}), p."source"::text + ) + SELECT + c."type"::text AS type, + ${mean}::double precision AS mean, + MIN(c.min_value)::double precision AS min, + MAX(c.max_value)::double precision AS max + FROM canon c + GROUP BY c."type" + `; + for (const r of rows) { + out.set(r.type, { mean: r.mean, min: r.min, max: r.max }); + } + return out; +} + +/** A small deterministic generator, so a failure reproduces. */ +function lcg(seed: number): () => number { + let s = seed; + return () => { + s = (s * 1_103_515_245 + 12_345) % 2_147_483_648; + return s / 2_147_483_648; + }; +} + +function seed(): Prisma.MeasurementCreateManyInput[] { + const rnd = lcg(7); + const rows: Prisma.MeasurementCreateManyInput[] = []; + let n = 0; + const add = ( + type: MeasurementType, + unit: string, + source: Prisma.MeasurementCreateManyInput["source"], + at: number, + value: number, + extra: Partial = {}, + ) => { + n += 1; + rows.push({ + id: `${USER}-${n}`, + userId: USER, + type, + unit, + source, + value, + measuredAt: new Date(at), + ...extra, + }); + }; + // Weight and blood pressure across six years: older than the fold window + // too. Every fifth day a hand entry beside the scale's, every 23rd day a + // reading a minute either side of UTC midnight, and a few tombstones. + for (let d = 6 * 365; d >= 0; d -= 1) { + const day = today - d * DAY; + add("WEIGHT", "kg", "WITHINGS", day + 6 * HOUR, 70 + 10 * rnd()); + if (d % 5 === 0) + add("WEIGHT", "kg", "MANUAL", day + 20 * HOUR, 69 + 12 * rnd()); + for (const [type, base] of [ + ["BLOOD_PRESSURE_SYS", 110], + ["BLOOD_PRESSURE_DIA", 65], + ] as const) { + add(type, "mmHg", "WITHINGS", day + 7 * HOUR, base + 40 * rnd()); + if (d % 23 === 0) { + add(type, "mmHg", "WITHINGS", day - 60_000, base + 45 * rnd()); + add(type, "mmHg", "WITHINGS", day + 60_000, base - 5 + 50 * rnd()); + } + if (d % 31 === 0) { + add(type, "mmHg", "WITHINGS", day + 9 * HOUR, base + 80, { + deletedAt: new Date(), + }); + } + } + } + // Pulse over the last 400 days: a watch's hourly statistics carrying the + // hour's own min / max, and a strap's dense samples on every other day. + for (let d = 400; d >= 0; d -= 1) { + const day = today - d * DAY; + for (let h = 0; h < 24; h += 3) { + const avg = 55 + 30 * rnd(); + add("PULSE", "bpm", "APPLE_HEALTH", day + h * HOUR, avg, { + valueMin: avg - 15, + valueMax: avg + 40, + externalId: `stats:HKQuantityTypeIdentifierHeartRate:${day + h * HOUR}`, + }); + } + if (d % 2 === 0) { + for (let i = 0; i < 30; i += 1) { + add( + "PULSE", + "bpm", + "WHOOP", + day + 10 * HOUR + i * 120_000, + 50 + 70 * rnd(), + ); + } + } + } + return rows; +} + +async function expectParity(floor: Date | null) { + const [actual, reference] = await Promise.all([ + readAllTimeExtremes(USER, TYPES, floor), + referenceExtremes(USER, TYPES, floor), + ]); + expect([...actual.keys()].sort()).toEqual([...reference.keys()].sort()); + expect(reference.size).toBe(4); + for (const [type, ref] of reference) { + const got = actual.get(type as MeasurementType)!; + expect(got.min, type).toBe(ref.min); + expect(got.max, type).toBe(ref.max); + expect(got.mean! / ref.mean, type).toBeCloseTo(1, 9); + } +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: "Pacific/Auckland" }, + }); + const rows = seed(); + for (let i = 0; i < rows.length; i += 5000) { + await prisma.measurement.createMany({ data: rows.slice(i, i + 5000) }); + } + await recomputeUserRollups(USER, { granularities: ["DAY"] }); +}, 300_000); + +describe("readAllTimeExtremes over the DAY tier", () => { + it("reads the tier for every type", async () => { + const covered = await prisma.measurementRollup.groupBy({ + by: ["type"], + where: { + userId: USER, + granularity: "DAY", + readingMinValue: { not: null }, + }, + }); + expect(covered.map((c) => c.type).sort()).toEqual([...TYPES].sort()); + }); + + it("gives the raw fold's figures over the whole history", async () => { + await expectParity(null); + }); + + it("gives the raw fold's figures from a floor inside a day", async () => { + await expectParity(new Date(today - 200 * DAY + 13 * HOUR + 17 * 60_000)); + }); + + it("follows a deletion through the write path's recompute", async () => { + const hottest = await prisma.measurement.findFirst({ + where: { userId: USER, type: "PULSE", source: "WHOOP", deletedAt: null }, + orderBy: { value: "desc" }, + }); + await prisma.measurement.update({ + where: { id: hottest!.id }, + data: { deletedAt: new Date() }, + }); + await afterMeasurementMutation( + USER, + [{ type: "PULSE", measuredAt: hottest!.measuredAt }], + { source: "manual" }, + ); + await expectParity(null); + }); + + it("is the tier that answers, not the raw rows", async () => { + // Write it, then break it: a tier row raised past every reading must + // move the answer, or the parity above proved nothing about the tier. + const day = new Date(today - 3 * DAY); + await prisma.measurementRollup.updateMany({ + where: { + userId: USER, + type: "WEIGHT", + granularity: "DAY", + bucketStart: day, + }, + data: { readingMaxValue: 999 }, + }); + const out = await readAllTimeExtremes(USER, ["WEIGHT"]); + expect(out.get("WEIGHT")?.max).toBe(999); + }); + + it("reads the raw rows for a type with a tier row lacking the extremes", async () => { + const day = new Date(today - 3 * DAY); + await prisma.measurementRollup.updateMany({ + where: { + userId: USER, + type: "WEIGHT", + granularity: "DAY", + bucketStart: day, + }, + data: { readingMinValue: null }, + }); + const [actual, reference] = await Promise.all([ + readAllTimeExtremes(USER, ["WEIGHT"]), + referenceExtremes(USER, ["WEIGHT"], null), + ]); + expect(actual.get("WEIGHT")?.max).toBe(reference.get("WEIGHT")?.max); + expect(actual.get("WEIGHT")?.min).toBe(reference.get("WEIGHT")?.min); + }); + + it("reads the raw rows for a type whose tier outlived a hard delete", async () => { + // Rebuild a clean tier, then delete the hottest pulse reading with no + // rollup recompute: its DAY row now counts a reading that is gone. + await recomputeUserRollups(USER, { granularities: ["DAY"] }); + const hottest = await prisma.measurement.findFirst({ + where: { userId: USER, type: "PULSE", deletedAt: null }, + orderBy: { value: "desc" }, + }); + await prisma.measurement.delete({ where: { id: hottest!.id } }); + const stale = await prisma.measurementRollup.findMany({ + where: { + userId: USER, + type: "PULSE", + granularity: "DAY", + readingMaxValue: hottest!.value, + }, + }); + expect(stale.length).toBeGreaterThan(0); + await expectParity(null); + const out = await readAllTimeExtremes(USER, ["PULSE"]); + expect(out.get("PULSE")?.max).toBeLessThan(hottest!.value); + }); +}); diff --git a/tests/integration/geo-backfill-attempt.test.ts b/tests/integration/geo-backfill-attempt.test.ts new file mode 100644 index 000000000..8c71bf735 --- /dev/null +++ b/tests/integration/geo-backfill-attempt.test.ts @@ -0,0 +1,107 @@ +/** + * The geo backfill tries a row once per retry wait, and finds its candidates + * through the partial index that holds only them. + * + * Before migration 0418 an address no database could place was looked up on + * every hourly pass for thirty days, and every pass walked thirty days of + * audit rows to find it. Run against the real table: the second pass must not + * pick the row again, a pass past the wait must, and the candidate read must + * go through `audit_logs_geo_backfill_idx` (read off the index's own scan + * counter, so the statement under test is the one Prisma actually sends). + */ +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/geo", () => ({ + lookupIpGeo: vi.fn(async (ip: string) => + ip.startsWith("10.") + ? { location: null, asn: null, carrier: null } + : { location: "Bochum, DE", asn: 3320, carrier: "Deutsche Telekom AG" }, + ), +})); + +const { runGeoBackfill, GEO_BACKFILL_RETRY_DAYS } = + await import("@/lib/jobs/geo-backfill"); +const { lookupIpGeo } = await import("@/lib/geo"); + +const prisma = getPrismaClient(); +const USER = "geo-backfill-attempt"; +const DAY = 86_400_000; +const NOW = new Date(); + +async function indexScans(): Promise { + const [row] = await prisma.$queryRaw>` + SELECT idx_scan AS n FROM pg_stat_user_indexes + WHERE indexrelname = 'audit_logs_geo_backfill_idx'`; + return Number(row?.n ?? 0); +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ data: { id: USER, username: USER } }); + // Enough resolved history that a sequential walk is the costlier plan. + await prisma.$executeRaw` + INSERT INTO audit_logs (id, user_id, action, ip_address, location, asn, carrier, created_at) + SELECT 'resolved-' || g, ${USER}, 'measurement.create', '84.131.0.1', + 'Bochum, DE', 3320, 'Deutsche Telekom AG', + ${NOW}::timestamp - (g * interval '5 seconds') + FROM generate_series(1, 20000) g`; + await prisma.auditLog.createMany({ + data: [ + // Never placeable: a private address. + { + id: "private", + userId: USER, + action: "auth.login", + ipAddress: "10.1.2.3", + createdAt: new Date(NOW.getTime() - 2 * DAY), + }, + // Placeable, missing only its carrier. + { + id: "carrierless", + userId: USER, + action: "auth.login", + ipAddress: "84.131.0.9", + location: "Bochum, DE", + createdAt: new Date(NOW.getTime() - 3 * DAY), + }, + ], + }); + await prisma.$executeRawUnsafe("ANALYZE audit_logs"); +}, 120_000); + +describe("runGeoBackfill", () => { + it("reads its candidates through the partial index", async () => { + const before = await indexScans(); + const summary = await runGeoBackfill(prisma, NOW); + expect(summary.scanned).toBe(2); + expect(summary.carrierResolved).toBe(1); + expect(summary.stillUnresolved).toBe(1); + // Statistics reach the view asynchronously; give them a moment. + let after = await indexScans(); + for (let i = 0; i < 40 && after === before; i += 1) { + await new Promise((r) => setTimeout(r, 250)); + after = await indexScans(); + } + expect(after).toBeGreaterThan(before); + }); + + it("does not try the unplaceable row again inside the wait", async () => { + vi.mocked(lookupIpGeo).mockClear(); + const summary = await runGeoBackfill( + prisma, + new Date(NOW.getTime() + 60 * 60_000), + ); + expect(summary.scanned).toBe(0); + expect(lookupIpGeo).not.toHaveBeenCalled(); + }); + + it("tries it again once the wait has passed", async () => { + const later = new Date(NOW.getTime() + (GEO_BACKFILL_RETRY_DAYS + 1) * DAY); + const summary = await runGeoBackfill(prisma, later); + expect(summary.scanned).toBe(1); + const row = await prisma.auditLog.findUnique({ where: { id: "private" } }); + expect(row?.geoAttemptedAt?.getTime()).toBe(later.getTime()); + }); +});