From a0e725556ba0a44ae5a19fadd2962e98f38ba8b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 19:45:11 +0200 Subject: [PATCH 01/23] chore(deps): raise the hono floor to 4.13.11 --- pnpm-lock.yaml | 22 +++++++++++----------- pnpm-workspace.yaml | 5 +++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6a9f052b4..4a51aab68 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,7 +6,7 @@ settings: overrides: js-yaml@<4.3.2: ^4.3.2 - hono@<4.13.5: ^4.13.5 + hono@<4.13.11: ^4.13.11 baseline-browser-mapping@<2.11.0: ^2.11.0 sharp@<0.35.4: ^0.35.4 dompurify@<3.4.16: ^3.4.16 @@ -669,7 +669,7 @@ packages: resolution: {integrity: sha512-dSneS5qhiauZWGDCeK4o695Xd9nUNjviSZCMQrj10eetr8Uln1ucn6bbphOM6UynAMMtNIzZNSpL9vnASJwrPQ==} engines: {node: '>=18.14.1'} peerDependencies: - hono: ^4.13.5 + hono: ^4.13.11 '@humanfs/core@0.19.2': resolution: {integrity: sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==} @@ -4436,8 +4436,8 @@ packages: hermes-parser@0.25.1: resolution: {integrity: sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==} - hono@4.13.7: - resolution: {integrity: sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==} + hono@4.13.13: + resolution: {integrity: sha512-CQ46U0ZkAGmbT/4UxdzzGJpacP2IeKgY4a5/tOI9AABbpOMfK739wfDXmv1usCk+3RkKj1hQy4/fjhiwa2xlrA==} engines: {node: '>=16.9.0'} html-escaper@2.0.2: @@ -7382,9 +7382,9 @@ snapshots: '@hexagon/base64@1.1.28': {} - '@hono/node-server@1.19.17(hono@4.13.7)': + '@hono/node-server@1.19.17(hono@4.13.13)': dependencies: - hono: 4.13.7 + hono: 4.13.13 '@humanfs/core@0.19.2': dependencies: @@ -7581,7 +7581,7 @@ snapshots: '@modelcontextprotocol/sdk@1.32.1(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@3.25.76)': dependencies: - '@hono/node-server': 1.19.17(hono@4.13.7) + '@hono/node-server': 1.19.17(hono@4.13.13) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -7591,7 +7591,7 @@ snapshots: eventsource-parser: 3.1.1 express: 5.2.1(supports-color@7.2.0) express-rate-limit: 8.6.2(express@5.2.1(supports-color@7.2.0))(supports-color@7.2.0) - hono: 4.13.7 + hono: 4.13.13 jose: 6.2.12 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -7605,7 +7605,7 @@ snapshots: '@modelcontextprotocol/sdk@1.32.1(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@4.6.7)': dependencies: - '@hono/node-server': 1.19.17(hono@4.13.7) + '@hono/node-server': 1.19.17(hono@4.13.13) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -7615,7 +7615,7 @@ snapshots: eventsource-parser: 3.1.1 express: 5.2.1(supports-color@7.2.0) express-rate-limit: 8.6.2(express@5.2.1(supports-color@7.2.0))(supports-color@7.2.0) - hono: 4.13.7 + hono: 4.13.13 jose: 6.2.12 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -11247,7 +11247,7 @@ snapshots: dependencies: hermes-estree: 0.25.1 - hono@4.13.7: {} + hono@4.13.13: {} html-escaper@2.0.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e98b2596f..a699c5cd7 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,11 +1,12 @@ overrides: # CVE-2026-84375 (js-yaml < 4.3.2, HIGH), CVE-2026-84363/84364/84365 (hono - # < 4.13.5, via @hono/node-server under the MCP SDK), CVE-2026-45819 + # < 4.13.5, via @hono/node-server under the MCP SDK; GHSA-5r4p-p66f-jhc7 raised the + # floor to 4.13.11), CVE-2026-45819 # (baseline-browser-mapping < 2.11.0), GHSA-rgj7-g3m4-5g8c (sharp < 0.35.4, # pulled by next). None is a direct dependency; the bounds move the copies # the scanner flags and nothing else. "js-yaml@<4.3.2": "^4.3.2" - "hono@<4.13.5": "^4.13.5" + "hono@<4.13.11": "^4.13.11" "baseline-browser-mapping@<2.11.0": "^2.11.0" "sharp@<0.35.4": "^0.35.4" # GHSA-55q2-fjhq-7xh7: a crafted document gets past the sanitiser and its From 628f37bc621c5811aaa2aea58ee475b5c32965b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 19:47:07 +0200 Subject: [PATCH 02/23] fix(apple-health): show a written-back type's newest reading on its row A reading entered here goes to Apple Health as well, and the quiet verdict already counted it, but the row still showed the newest reading that arrived from the phone, so a type entered two minutes ago read one day old. The server now resolves lastSeenAt to the newer of the two for the types the app writes back, and the row and the verdict agree. Refs #1247 --- docs/api/openapi.yaml | 4 +++- .../healthkit/__tests__/route.test.ts | 3 +++ src/lib/integrations/sync-verdict.ts | 18 ++++++++++++------ src/lib/openapi/routes/integrations.ts | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index ae4fc16ec..c17830d99 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -77130,7 +77130,9 @@ components: glucose while their entry is `bidirectional` or `writeOnly` and enabled): when the newest reading of the type was entered here rather than arriving from Apple Health, or null when none was. Such readings go to the phone and never come back as an arrival, so `stale` already counts them: the type is quiet only when both - this and `lastSeenAt` are. Show it as the server sends it; do not recompute `stale`." + are. Since v1.43.9 `lastSeenAt` on such a type is the newer of this and the newest reading from Apple + Health, because a reading entered here is in Apple Health too; show both as the server sends them and do not + recompute `stale`." anyOf: - type: string format: date-time diff --git a/src/app/api/integrations/healthkit/__tests__/route.test.ts b/src/app/api/integrations/healthkit/__tests__/route.test.ts index d8f44da69..52b77eb46 100644 --- a/src/app/api/integrations/healthkit/__tests__/route.test.ts +++ b/src/app/api/integrations/healthkit/__tests__/route.test.ts @@ -406,6 +406,9 @@ describe("GET /api/integrations/healthkit — sync health", () => { expect(entries.WEIGHT.lastEnteredHereAt).toBe( enteredHereAt.toISOString(), ); + // The reading entered here is in Apple Health too, so it is the type's + // newest reading: the row and the verdict read the same instant. + expect(entries.WEIGHT.lastSeenAt).toBe(enteredHereAt.toISOString()); // A read-only type keeps its own verdict and carries no such field. expect(entries.PULSE.stale).toBe(true); expect(entries.PULSE).not.toHaveProperty("lastEnteredHereAt"); diff --git a/src/lib/integrations/sync-verdict.ts b/src/lib/integrations/sync-verdict.ts index 5b911fc40..8a0863c19 100644 --- a/src/lib/integrations/sync-verdict.ts +++ b/src/lib/integrations/sync-verdict.ts @@ -342,21 +342,27 @@ export function classifyMetricFreshness( return samples.map((sample) => { // A type the app writes back to the phone (#1247): what is entered here // goes to Apple Health and never arrives back, so a person who keeps - // their readings here would otherwise watch the type go quiet. The newer - // of the two is what says whether the type is still in use. + // their readings here would otherwise watch the type go quiet. That + // reading is in Apple Health too, so the newer of the two is the type's + // newest reading: it is what the row shows and what the verdict reads. const written = enteredHere?.has(sample.type) ? { lastEnteredHereAt: enteredHere.get(sample.type) ?? null } : {}; - if (verdict !== "fresh" || SPARSE_METRIC_TYPES.has(sample.type)) { - return { ...sample, ...written, stale: false }; - } const seen = newest( toMillis(sample.lastSeenAt), toMillis(written.lastEnteredHereAt), ); - return { + const resolved = { ...sample, ...written, + lastSeenAt: + seen != null ? new Date(seen).toISOString() : sample.lastSeenAt, + }; + if (verdict !== "fresh" || SPARSE_METRIC_TYPES.has(sample.type)) { + return { ...resolved, stale: false }; + } + return { + ...resolved, stale: seen != null && nowMs - seen > METRIC_QUIET_AFTER_MS, }; }); diff --git a/src/lib/openapi/routes/integrations.ts b/src/lib/openapi/routes/integrations.ts index 9fc6ec356..9e36dfcb6 100644 --- a/src/lib/openapi/routes/integrations.ts +++ b/src/lib/openapi/routes/integrations.ts @@ -293,7 +293,7 @@ const appleHealthMetricFreshnessEntry = metricFreshnessEntry .nullable() .optional() .describe( - "v1.43.7 — present only on a type the app writes back to Apple Health (weight, blood pressure, blood glucose while their entry is `bidirectional` or `writeOnly` and enabled): when the newest reading of the type was entered here rather than arriving from Apple Health, or null when none was. Such readings go to the phone and never come back as an arrival, so `stale` already counts them: the type is quiet only when both this and `lastSeenAt` are. Show it as the server sends it; do not recompute `stale`.", + "v1.43.7 — present only on a type the app writes back to Apple Health (weight, blood pressure, blood glucose while their entry is `bidirectional` or `writeOnly` and enabled): when the newest reading of the type was entered here rather than arriving from Apple Health, or null when none was. Such readings go to the phone and never come back as an arrival, so `stale` already counts them: the type is quiet only when both are. Since v1.43.9 `lastSeenAt` on such a type is the newer of this and the newest reading from Apple Health, because a reading entered here is in Apple Health too; show both as the server sends them and do not recompute `stale`.", ), }) .meta({ From 07d70cd27556a0547cb91ee9bbc4461328380080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:04:21 +0200 Subject: [PATCH 03/23] fix(ai): park a Codex sign-in the issuer refused until the next sign-in A dead refresh token was asked again on every hop of every run: the proactive refresh failed and was swallowed, the request 401'd, the on-401 refresh failed again, and the nightly warm repeated that for the briefing and every card. A refusal that proves the grant dead now flips the connection to the existing expired state (compare-and-set on the refused token, one signal per transition), so the chain, the presence probe and the AI capabilities stop counting it. A new device-code sign-in clears the refused credential's ledger rows so the fresh one resumes in place. --- .../admin/central-codex/device-poll/route.ts | 4 + src/app/api/auth/codex/device-poll/route.ts | 4 + src/lib/ai/provider-health-ledger.ts | 27 ++ src/lib/ai/provider.ts | 147 +++++++++- src/lib/jobs/insight-pregenerate.ts | 3 +- .../integration/codex-credential-park.test.ts | 263 ++++++++++++++++++ 6 files changed, 437 insertions(+), 11 deletions(-) create mode 100644 tests/integration/codex-credential-park.test.ts diff --git a/src/app/api/admin/central-codex/device-poll/route.ts b/src/app/api/admin/central-codex/device-poll/route.ts index b5e77028f..86aab4eed 100644 --- a/src/app/api/admin/central-codex/device-poll/route.ts +++ b/src/app/api/admin/central-codex/device-poll/route.ts @@ -8,6 +8,7 @@ import { prisma } from "@/lib/db"; import { auditLog } from "@/lib/auth/audit"; import { getClientIp, apiSuccess, apiError } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; +import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger"; /** * Operator-shared central Codex — device-code poll (admin only). @@ -91,6 +92,9 @@ export const POST = apiHandler(async (request: NextRequest) => { }); cookieStore.delete("central_codex_device"); + // A new central sign-in: every account's ledger rows describe the refused + // one, and the central connection is one credential shared by all of them. + await forgetProviderHealth({ providerType: "admin-codex" }); await auditLog("admin.central-codex.connected", { userId: user.id, diff --git a/src/app/api/auth/codex/device-poll/route.ts b/src/app/api/auth/codex/device-poll/route.ts index 1853de300..47f9987b0 100644 --- a/src/app/api/auth/codex/device-poll/route.ts +++ b/src/app/api/auth/codex/device-poll/route.ts @@ -9,6 +9,7 @@ import { auditLog } from "@/lib/auth/audit"; import { getClientIp, apiSuccess, apiError } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; import { INSIGHTS_CACHE_CLEARED } from "@/lib/insights/cached-text-cipher"; +import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger"; /** * v1.4.7.1: Device-code flow polling endpoint. @@ -88,6 +89,9 @@ export const POST = apiHandler(async (request: NextRequest) => { }); cookieStore.delete("codex_device"); + // A new sign-in: whatever the ledger held against the old one no longer + // describes this credential. + await forgetProviderHealth({ userId: user.id, providerType: "codex" }); await auditLog("codex.oauth.connected", { userId: user.id, diff --git a/src/lib/ai/provider-health-ledger.ts b/src/lib/ai/provider-health-ledger.ts index 1ffbb461c..b6e25a4b6 100644 --- a/src/lib/ai/provider-health-ledger.ts +++ b/src/lib/ai/provider-health-ledger.ts @@ -266,6 +266,33 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = { }, }; +/** + * Drop the recorded outcomes of a credential that has just been replaced. + * + * The ledger describes a credential, not a provider slot: once the person + * (or, for `admin-codex`, the operator) signs in again, the `auth_failed` + * row of the refused credential would otherwise keep the fresh one at the + * tail of the chain and reported as expired for the rest of its cooldown. + * Without `userId` the rows of every account are dropped — the operator's + * central connection is one credential shared by all of them. Best effort, + * like every ledger write. + */ +export async function forgetProviderHealth(target: { + providerType: ProviderChainType; + userId?: string; +}): Promise { + try { + await prisma.providerHealth.deleteMany({ + where: { + providerType: target.providerType, + ...(target.userId ? { userId: target.userId } : {}), + }, + }); + } catch { + // A stale row only deprioritises; it never blocks. + } +} + /** * Providers whose credential is currently known-bad (auth-class failure * inside the cooldown window) for a user. Drives the proactive diff --git a/src/lib/ai/provider.ts b/src/lib/ai/provider.ts index 51f99b5c6..e05758bed 100644 --- a/src/lib/ai/provider.ts +++ b/src/lib/ai/provider.ts @@ -30,6 +30,7 @@ import { type ProviderWorkAuthority, } from "@/lib/sharing/provider-work-authority"; import { memoizePerRequest } from "@/lib/request-cache"; +import { emitSignal } from "@/lib/logging/signal"; import type { AiModality } from "@/lib/ai/capabilities/types"; import type { ProviderEntryPresence, @@ -42,6 +43,93 @@ import { const TOKEN_REFRESH_BUFFER_MS = 5 * 60 * 1000; // 5 minutes +/** + * Whether a failed Codex refresh proves the sign-in dead: the issuer refused + * the grant itself (`CodexRefreshError.credentialExpired`), as opposed to a + * 429, a 5xx or a network error, which say nothing about the credential. + * Read by shape rather than `instanceof` so a mocked `codex-oauth` module + * classifies the same way. + */ +function isDeadCodexSignIn(error: unknown): boolean { + return ( + (error as { credentialExpired?: unknown } | null)?.credentialExpired === + true + ); +} + +function oauthErrorOf(error: unknown): string | null { + const code = (error as { oauthError?: unknown } | null)?.oauthError; + return typeof code === "string" ? code : null; +} + +/** + * Park a Codex connection whose sign-in the issuer has refused for good. + * + * Retrying a dead refresh token cannot succeed; until this existed every + * generation of every card of every run asked the issuer again, failed with + * the same 401, and logged it. Parking flips the connection to the existing + * `expired` state — the one Settings already renders as "reconnect ChatGPT" — + * so the chain, the presence probe and with it every AI capability stop + * counting it, and nothing is sent until the person signs in again (the + * device-code flow writes `connected`). + * + * Compare-and-set on the refresh token that was refused: a reconnect that + * landed in between wrote a different one and is left alone, and of several + * workers failing at once exactly one flips the row, so the transition is + * signalled once and never again per run. + */ +async function parkUserCodex( + userId: string, + refusedRefreshEncrypted: string, + error: unknown, +): Promise { + try { + const parked = await prisma.user.updateMany({ + where: { + id: userId, + codexConnectionStatus: "connected", + codexRefreshTokenEncrypted: refusedRefreshEncrypted, + }, + data: { codexConnectionStatus: "expired" }, + }); + if (parked.count === 1) { + emitSignal({ + action: "ai.credential.expired", + level: "warn", + meta: { provider: "codex", oauth_error: oauthErrorOf(error) }, + }); + } + } catch { + // Best effort: an unparked connection is retried exactly as before. + } +} + +/** The operator's central Codex, parked the same way. */ +async function parkCentralCodex( + refusedRefreshEncrypted: string, + error: unknown, +): Promise { + try { + const parked = await prisma.appSettings.updateMany({ + where: { + id: "singleton", + adminCodexConnectionStatus: "connected", + adminCodexRefreshTokenEncrypted: refusedRefreshEncrypted, + }, + data: { adminCodexConnectionStatus: "expired" }, + }); + if (parked.count === 1) { + emitSignal({ + action: "ai.credential.expired", + level: "warn", + meta: { provider: "admin-codex", oauth_error: oauthErrorOf(error) }, + }); + } + } catch { + // Best effort, as above. + } +} + class NoProvider implements AIProvider { readonly type = "none" as const; @@ -308,9 +396,15 @@ async function resolveCodexProvider( codexTokenExpiresAt: fresh.expiresAt, }, }); - } catch { - // Fall through — CodexClient will trigger an on-401 refresh - // and persist there. + } catch (error) { + // A refused grant parks the connection and drops it from this chain: + // calling Codex with the expired access token could only 401 into a + // second refusal. Anything else falls through — CodexClient will + // trigger an on-401 refresh and persist there. + if (isDeadCodexSignIn(error)) { + await parkUserCodex(userId, user.codexRefreshTokenEncrypted, error); + return null; + } } } @@ -336,9 +430,27 @@ async function resolveCodexProvider( refreshEncrypted: freshUser.codexRefreshTokenEncrypted, }); if (!decoded) { - throw codexReconnectRequired("token storage corrupt"); + const error = codexReconnectRequired("token storage corrupt"); + await parkUserCodex( + userId, + freshUser.codexRefreshTokenEncrypted, + error, + ); + throw error; + } + let fresh; + try { + fresh = await refreshDeviceTokens(decoded.refreshToken); + } catch (error) { + if (isDeadCodexSignIn(error)) { + await parkUserCodex( + userId, + freshUser.codexRefreshTokenEncrypted, + error, + ); + } + throw error; } - const fresh = await refreshDeviceTokens(decoded.refreshToken); const enc = encryptCodexCreds(fresh); await prisma.user.update({ where: { id: userId }, @@ -421,9 +533,14 @@ async function resolveAdminCodexProvider(): Promise { adminCodexTokenExpiresAt: enc.expiresAt, }, }); - } catch { - // Fall through — CodexClient triggers an on-401 refresh and persists via - // the callback below. + } catch (error) { + // A refused grant parks the central connection (see `parkUserCodex`); + // anything else falls through — CodexClient triggers an on-401 refresh + // and persists via the callback below. + if (isDeadCodexSignIn(error)) { + await parkCentralCodex(settings.adminCodexRefreshTokenEncrypted, error); + return null; + } } } @@ -454,9 +571,19 @@ async function resolveAdminCodexProvider(): Promise { expiresAt: fresh.adminCodexTokenExpiresAt, }); if (!decoded) { - throw codexReconnectRequired("central token storage corrupt"); + const error = codexReconnectRequired("central token storage corrupt"); + await parkCentralCodex(fresh.adminCodexRefreshTokenEncrypted, error); + throw error; + } + let refreshed; + try { + refreshed = await refreshDeviceTokens(decoded.refreshToken); + } catch (error) { + if (isDeadCodexSignIn(error)) { + await parkCentralCodex(fresh.adminCodexRefreshTokenEncrypted, error); + } + throw error; } - const refreshed = await refreshDeviceTokens(decoded.refreshToken); const enc = encryptAdminCodexCreds(refreshed); await prisma.appSettings.update({ where: { id: "singleton" }, diff --git a/src/lib/jobs/insight-pregenerate.ts b/src/lib/jobs/insight-pregenerate.ts index c67b1c929..509551b51 100644 --- a/src/lib/jobs/insight-pregenerate.ts +++ b/src/lib/jobs/insight-pregenerate.ts @@ -888,7 +888,8 @@ export async function runInsightPregenerate( } else if (generated.credentialExpired) { // v1.42 — a dead sign-in (a Codex refresh token the issuer // revoked) does not heal in 45 minutes either; the person has - // to reconnect, and the ledger already tells them so. + // to reconnect. The refusal parked the connection as expired, + // which Settings shows, and the next run no longer counts it. settle("auth_failed", generated.reason); } else { settle("failed", generated.reason); diff --git a/tests/integration/codex-credential-park.test.ts b/tests/integration/codex-credential-park.test.ts new file mode 100644 index 000000000..95ea57391 --- /dev/null +++ b/tests/integration/codex-credential-park.test.ts @@ -0,0 +1,263 @@ +/** + * A Codex sign-in the issuer refused is parked until the person signs in + * again, against the real rows. + * + * Before the park, a dead refresh token was asked again on every hop of + * every run: the chain resolved Codex, the proactive refresh failed and was + * swallowed, the request 401'd, the on-401 refresh failed again, and the + * nightly warm repeated that for the briefing and every card. Parking flips + * the connection to the `expired` state Settings already renders, so the + * chain and the presence probe stop counting it and nothing leaves until a + * new sign-in writes `connected`. + */ +import { NextRequest } from "next/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/logging/signal", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, emitSignal: vi.fn(actual.emitSignal) }; +}); + +// Only the two calls that would leave the machine are replaced; the +// credential codec stays real so the rows hold genuine ciphertext. +vi.mock("@/lib/ai/codex-oauth", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + refreshDeviceTokens: vi.fn(), + pollDeviceCode: vi.fn(), + }; +}); + +import { + CodexRefreshError, + encryptAdminCodexCreds, + encryptCodexCreds, + pollDeviceCode, + refreshDeviceTokens, +} from "@/lib/ai/codex-oauth"; +import { encrypt } from "@/lib/crypto"; +import { emitSignal } from "@/lib/logging/signal"; +import { probeProviderChain, resolveProviderChain } from "@/lib/ai/provider"; +import { postgresProviderHealthLedger } from "@/lib/ai/provider-health-ledger"; + +const prisma = getPrismaClient(); + +const HOUR_MS = 60 * 60 * 1000; + +function creds(tag: string, expiresInMs: number) { + return { + accessToken: `access-${tag}`, + refreshToken: `refresh-${tag}`, + accountId: `account-${tag}`, + expiresAt: new Date(Date.now() + expiresInMs), + }; +} + +function parkSignals(provider: string) { + return vi + .mocked(emitSignal) + .mock.calls.filter( + ([input]) => + input.action === "ai.credential.expired" && + input.meta?.provider === provider, + ); +} + +async function seedCodexUser() { + // An access token that expired long ago: the state a dead sign-in sits in. + const enc = encryptCodexCreds(creds("old", -HOUR_MS)); + const user = await prisma.user.create({ + data: { + username: "codex-park", + email: "codex-park@example.test", + aiProviderChain: [{ providerType: "codex", priority: 1, enabled: true }], + codexAccessTokenEncrypted: enc.accessEncrypted, + codexRefreshTokenEncrypted: enc.refreshEncrypted, + codexConnectionStatus: "connected", + codexConnectedAt: new Date(), + }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return user; +} + +beforeEach(async () => { + await truncateAllTables(prisma); + cookieJar.clear(); + headerJar.clear(); + vi.mocked(emitSignal).mockClear(); + vi.mocked(refreshDeviceTokens).mockReset(); + vi.mocked(pollDeviceCode).mockReset(); +}); + +afterEach(() => { + vi.mocked(refreshDeviceTokens).mockReset(); +}); + +describe("a refused Codex sign-in", () => { + it("is parked once, stops being asked, and resumes after a new sign-in", async () => { + const user = await seedCodexUser(); + vi.mocked(refreshDeviceTokens).mockRejectedValue( + new CodexRefreshError(400, "invalid_grant"), + ); + + // Two workers meet the refusal at once: one transition, one signal. + const [first, second] = await Promise.all([ + resolveProviderChain(user.id), + resolveProviderChain(user.id), + ]); + expect(first.map((p) => p.providerType)).not.toContain("codex"); + expect(second.map((p) => p.providerType)).not.toContain("codex"); + expect( + ( + await prisma.user.findUniqueOrThrow({ + where: { id: user.id }, + select: { codexConnectionStatus: true }, + }) + ).codexConnectionStatus, + ).toBe("expired"); + expect(parkSignals("codex")).toHaveLength(1); + expect(parkSignals("codex")[0][0].meta?.oauth_error).toBe("invalid_grant"); + + // Parked: later runs neither ask the issuer nor count Codex as present. + const asked = vi.mocked(refreshDeviceTokens).mock.calls.length; + for (let i = 0; i < 3; i += 1) { + const chain = await resolveProviderChain(user.id); + expect(chain.map((p) => p.providerType)).not.toContain("codex"); + } + expect(vi.mocked(refreshDeviceTokens).mock.calls.length).toBe(asked); + expect(parkSignals("codex")).toHaveLength(1); + const presence = await probeProviderChain(user.id); + expect(presence.availableTypes).not.toContain("codex"); + + // The hop that met the refusal left an auth_failed row behind. + await postgresProviderHealthLedger.recordFailure(user.id, "codex", 401); + expect( + (await postgresProviderHealthLedger.getSkipHints(user.id)).get("codex") + ?.reason, + ).toBe("credential_expired"); + + // The person signs in again through the device-code flow. + vi.mocked(pollDeviceCode).mockResolvedValue({ + status: "connected", + creds: creds("new", 10 * HOUR_MS), + } as Awaited>); + cookieJar.set( + "codex_device", + encrypt(JSON.stringify({ deviceAuthId: "device", userCode: "CODE" })), + ); + const { POST } = await import("@/app/api/auth/codex/device-poll/route"); + const res = await POST( + new NextRequest("http://localhost/api/auth/codex/device-poll", { + method: "POST", + }), + ); + expect(res.status).toBe(200); + + // Resumed in place: connected, first in the chain, not benched by the + // refused credential's ledger row, and the fresh token needs no refresh. + const resumed = await resolveProviderChain(user.id); + expect(resumed[0]?.providerType).toBe("codex"); + expect( + (await postgresProviderHealthLedger.getSkipHints(user.id)).has("codex"), + ).toBe(false); + expect(vi.mocked(refreshDeviceTokens).mock.calls.length).toBe(asked); + }); + + it("is not parked by a refusal that says nothing about the credential", async () => { + const user = await seedCodexUser(); + vi.mocked(refreshDeviceTokens).mockRejectedValue( + new CodexRefreshError(503, null), + ); + + const chain = await resolveProviderChain(user.id); + + // A brown-out at the issuer: the connection stays, the client's own + // on-401 path gets its chance, and nothing is signalled. + expect(chain.map((p) => p.providerType)).toContain("codex"); + expect( + ( + await prisma.user.findUniqueOrThrow({ + where: { id: user.id }, + select: { codexConnectionStatus: true }, + }) + ).codexConnectionStatus, + ).toBe("connected"); + expect(parkSignals("codex")).toHaveLength(0); + }); + + it("parks the operator's central connection the same way", async () => { + const enc = encryptAdminCodexCreds(creds("central", -HOUR_MS)); + await prisma.appSettings.create({ + data: { + id: "singleton", + adminCodexAccessTokenEncrypted: enc.accessEncrypted, + adminCodexRefreshTokenEncrypted: enc.refreshEncrypted, + adminCodexAccountIdEncrypted: enc.accountIdEncrypted, + adminCodexTokenExpiresAt: enc.expiresAt, + adminCodexConnectionStatus: "connected", + }, + }); + const user = await prisma.user.create({ + data: { + username: "central-park", + email: "central-park@example.test", + aiProviderChain: [], + useCentralCodex: true, + }, + }); + vi.mocked(refreshDeviceTokens).mockRejectedValue( + new CodexRefreshError(401, null), + ); + + const chain = await resolveProviderChain(user.id); + expect(chain.map((p) => p.providerType)).not.toContain("admin-codex"); + expect( + ( + await prisma.appSettings.findUniqueOrThrow({ + where: { id: "singleton" }, + select: { adminCodexConnectionStatus: true }, + }) + ).adminCodexConnectionStatus, + ).toBe("expired"); + expect(parkSignals("admin-codex")).toHaveLength(1); + + await resolveProviderChain(user.id); + expect(vi.mocked(refreshDeviceTokens)).toHaveBeenCalledTimes(1); + }); +}); From 567a564913020e6bd992930bb929a70376f53146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:07:25 +0200 Subject: [PATCH 04/23] perf(auth): resolve the caller to a narrow user projection The session lookup and the Bearer resolver loaded the whole users row on every authenticated request, including the profile picture bytes, the cached briefing text and every stored integration credential. Both now select SESSION_USER_SELECT, a named allowlist of the small columns the request path reads, and so does the owner load for a switched record. The few routes that need a credential or a large column read it themselves at the moment of use: the factor verifiers load the password hash and TOTP secret, the profile response loads the insurance number, and the snapshot read loads the cached briefing text. A structural guard holds the lookups to the projection and keeps every Bytes and secret-shaped column out of it. --- .../session-user-projection-guard.test.ts | 137 ++++++++++++++++++ .../admin/users/[id]/reset-password/route.ts | 5 +- src/app/api/admin/users/[id]/route.ts | 5 +- .../me/mfa/disable/__tests__/route.test.ts | 9 +- src/app/api/auth/me/mfa/disable/route.ts | 11 +- src/app/api/auth/me/mfa/totp/confirm/route.ts | 9 +- src/app/api/auth/me/route.ts | 12 +- .../passkeys/[id]/__tests__/route.test.ts | 11 +- src/app/api/auth/passkeys/[id]/route.ts | 12 +- .../api/auth/password/__tests__/route.test.ts | 7 +- src/app/api/auth/password/route.ts | 11 +- .../api/record-settings/integrations/route.ts | 64 ++++---- src/lib/api-handler.ts | 16 +- .../enrollment-proof-second-factor.test.ts | 31 +++- src/lib/auth/bearer.ts | 5 +- src/lib/auth/existing-factor-proof.ts | 40 +++-- src/lib/auth/mfa/verify-factor.ts | 22 +++ src/lib/auth/second-factor.ts | 13 +- src/lib/auth/session-user.ts | 126 ++++++++++++++++ src/lib/auth/session.ts | 15 +- src/lib/daily/daily-briefing-push.ts | 15 +- src/lib/daily/load-digest.ts | 4 +- .../snapshot-read-briefing-capability.test.ts | 23 ++- .../snapshot-read-briefing-today.test.ts | 24 ++- .../__tests__/snapshot-read-cache-key.test.ts | 8 +- src/lib/dashboard/snapshot-read.ts | 40 +++-- src/lib/jobs/reaction-line.ts | 6 +- 27 files changed, 543 insertions(+), 138 deletions(-) create mode 100644 src/__tests__/session-user-projection-guard.test.ts create mode 100644 src/lib/auth/session-user.ts diff --git a/src/__tests__/session-user-projection-guard.test.ts b/src/__tests__/session-user-projection-guard.test.ts new file mode 100644 index 000000000..36a64c618 --- /dev/null +++ b/src/__tests__/session-user-projection-guard.test.ts @@ -0,0 +1,137 @@ +/** + * The caller lookups load a projection of `User`, never the whole row. + * + * Every authenticated request resolves its caller through the session lookup + * (`lib/auth/session.ts`), the Bearer resolver (`lib/auth/bearer.ts`) or, for a + * switched record, the owner load in `lib/api-handler.ts`. They used to + * `include: { user: true }` / `findUnique({ where })`, which copied the profile + * picture (megabytes), the cached briefing text and every stored credential + * into memory on each request. They now name `SESSION_USER_SELECT`. + * + * Two halves: + * 1. each of those lookups carries the projection, and no file anywhere loads + * a related user row whole with `include: { user: true }`; + * 2. the projection itself leaves out every `Bytes` column of the model and + * every secret-shaped column, derived from `schema.prisma` so a new + * column of either kind is covered without editing this file. + */ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { walkSourceFiles } from "./helpers/source-files"; +import { SESSION_USER_SELECT } from "@/lib/auth/session-user"; + +const SRC = join(process.cwd(), "src"); + +function read(rel: string): string { + return readFileSync(join(SRC, rel), "utf8"); +} + +function stripComments(src: string): string { + return src + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/(^|[^:])\/\/.*$/gm, "$1"); +} + +/** The `{ ... }` argument of every call matching `callee`, braces balanced. */ +function callObjects(src: string, callee: RegExp): string[] { + const code = stripComments(src); + const out: string[] = []; + const re = new RegExp(callee.source + /\s*\(\s*\{/.source, "g"); + for (let m = re.exec(code); m; m = re.exec(code)) { + const open = m.index + m[0].length - 1; + let depth = 0; + let i = open; + for (; i < code.length; i++) { + if (code[i] === "{") depth++; + else if (code[i] === "}" && --depth === 0) break; + } + out.push(code.slice(open, i + 1)); + } + return out; +} + +const USER_LOOKUP = /prisma\.user\.find(?:Unique|First)(?:OrThrow)?/; +const SESSION_LOOKUP = /prisma\.session\.find(?:Unique|First)(?:OrThrow)?/; + +describe("caller lookups load the session projection", () => { + it("the session lookups include the user through SESSION_USER_SELECT", () => { + const calls = callObjects( + read("lib/auth/session.ts"), + SESSION_LOOKUP, + ).filter((body) => /\binclude\s*:/.test(body)); + expect(calls.length).toBeGreaterThanOrEqual(2); + for (const body of calls) { + expect(body).toMatch( + /include\s*:\s*\{\s*user\s*:\s*\{\s*select\s*:\s*SESSION_USER_SELECT\s*\}\s*\}/, + ); + } + }); + + it("the Bearer resolver and the switched-owner load select SESSION_USER_SELECT", () => { + for (const rel of ["lib/auth/bearer.ts", "lib/api-handler.ts"]) { + const calls = callObjects(read(rel), USER_LOOKUP); + expect(calls.length, rel).toBeGreaterThan(0); + for (const body of calls) { + expect(body, rel).toMatch(/\bselect\s*:/); + } + expect( + calls.some((body) => /select\s*:\s*SESSION_USER_SELECT\b/.test(body)), + rel, + ).toBe(true); + } + }); + + it("no file loads a related user row whole", () => { + const files = walkSourceFiles(SRC, { floor: 3000 }) + .filter((p) => !p.startsWith("generated/")) + .filter((p) => !p.includes("__tests__")) + .filter((p) => !/\.test\.tsx?$/.test(p)); + expect(files.length).toBeGreaterThan(0); + const offenders = files.filter((rel) => + /\buser\s*:\s*true\b/.test( + stripComments(read(rel)).replace(/select\s*:\s*\{[^{}]*\}/g, ""), + ), + ); + expect(offenders).toEqual([]); + }); +}); + +describe("SESSION_USER_SELECT leaves out bytes and secrets", () => { + const schema = readFileSync( + join(process.cwd(), "prisma", "schema.prisma"), + "utf8", + ); + const model = schema.match(/^model User \{([\s\S]*?)^\}/m)?.[1] ?? ""; + const columns = [ + ...model.matchAll(/^\s+(\w+)\s+([A-Z]\w*)(\??)(?:\s|$)/gm), + ].map(([, name, type]) => ({ name, type })); + const selected = Object.keys(SESSION_USER_SELECT); + + it("reads the User model", () => { + expect(columns.length).toBeGreaterThan(50); + expect(selected.length).toBeGreaterThan(0); + }); + + it("names only columns the model has", () => { + const names = new Set(columns.map((c) => c.name)); + expect(selected.filter((key) => !names.has(key))).toEqual([]); + }); + + it("selects no Bytes column", () => { + const bytes = columns.filter((c) => c.type === "Bytes").map((c) => c.name); + expect(bytes).toContain("avatarBytes"); + expect(selected.filter((key) => bytes.includes(key))).toEqual([]); + }); + + it("selects no credential, secret or large cached text", () => { + const secretShaped = + /Encrypted$|^passwordHash$|^totpSecret|^totpLastStep$|^telegramBotToken$|^insightsCachedText$|^environmentAqHistoryJson$/; + const secrets = columns + .map((c) => c.name) + .filter((name) => secretShaped.test(name)); + expect(secrets.length).toBeGreaterThan(10); + expect(selected.filter((key) => secrets.includes(key))).toEqual([]); + }); +}); diff --git a/src/app/api/admin/users/[id]/reset-password/route.ts b/src/app/api/admin/users/[id]/reset-password/route.ts index ae913dddc..257102aeb 100644 --- a/src/app/api/admin/users/[id]/reset-password/route.ts +++ b/src/app/api/admin/users/[id]/reset-password/route.ts @@ -60,7 +60,10 @@ export const POST = apiHandler( return apiError("Password required", 422); } - const target = await prisma.user.findUnique({ where: { id } }); + const target = await prisma.user.findUnique({ + where: { id }, + select: { locale: true, username: true }, + }); if (!target) return apiError("User not found", 404); const locale = await resolveServerLocale({ diff --git a/src/app/api/admin/users/[id]/route.ts b/src/app/api/admin/users/[id]/route.ts index 8a954c7dd..f55cb71fc 100644 --- a/src/app/api/admin/users/[id]/route.ts +++ b/src/app/api/admin/users/[id]/route.ts @@ -60,7 +60,10 @@ export const PUT = apiHandler( return returnAllZodIssues(parsed.error, 422); } - const target = await prisma.user.findUnique({ where: { id } }); + const target = await prisma.user.findUnique({ + where: { id }, + select: { role: true }, + }); if (!target) return apiError("User not found", 404); // Prevent removing the last admin diff --git a/src/app/api/auth/me/mfa/disable/__tests__/route.test.ts b/src/app/api/auth/me/mfa/disable/__tests__/route.test.ts index a819d6a08..96134f00b 100644 --- a/src/app/api/auth/me/mfa/disable/__tests__/route.test.ts +++ b/src/app/api/auth/me/mfa/disable/__tests__/route.test.ts @@ -34,6 +34,7 @@ vi.mock("@/lib/rate-limit", () => ({ vi.mock("@/lib/auth/mfa/verify-factor", () => ({ verifyMfaFactor: vi.fn(), + loadFactorUser: vi.fn(), })); vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); @@ -42,7 +43,7 @@ import { POST } from "../route"; import { requireMfaManagementAuth } from "@/lib/api-handler"; import { prisma } from "@/lib/db"; import { destroyOtherSessions } from "@/lib/auth/session"; -import { verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; +import { loadFactorUser, verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; function req() { return new Request("http://localhost/api/auth/me/mfa/disable", { @@ -54,6 +55,12 @@ function req() { beforeEach(() => { vi.clearAllMocks(); + vi.mocked(loadFactorUser).mockResolvedValue({ + id: "u1", + totpSecretEncrypted: "enc", + totpLastStep: null, + totpConfirmedAt: new Date("2026-01-01"), + }); vi.mocked(requireMfaManagementAuth).mockResolvedValue({ transport: "cookie", user: { id: "u1" }, diff --git a/src/app/api/auth/me/mfa/disable/route.ts b/src/app/api/auth/me/mfa/disable/route.ts index 1493d61e4..201f4c61f 100644 --- a/src/app/api/auth/me/mfa/disable/route.ts +++ b/src/app/api/auth/me/mfa/disable/route.ts @@ -30,7 +30,7 @@ import { auditLog } from "@/lib/auth/audit"; import { prisma } from "@/lib/db"; import { destroyOtherSessions } from "@/lib/auth/session"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; -import { verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; +import { loadFactorUser, verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; import { mfaDisableSchema } from "@/lib/validations/mfa"; export const dynamic = "force-dynamic"; @@ -72,11 +72,10 @@ export const POST = apiHandler(async (req: Request) => { ); } - const factor = await verifyMfaFactor( - user, - parsed.data.method, - parsed.data.code, - ); + const factorUser = await loadFactorUser(user.id); + const factor = factorUser + ? await verifyMfaFactor(factorUser, parsed.data.method, parsed.data.code) + : { ok: false, replay: false }; if (!factor.ok) { await auditLog("auth.mfa.failed", { userId: user.id, diff --git a/src/app/api/auth/me/mfa/totp/confirm/route.ts b/src/app/api/auth/me/mfa/totp/confirm/route.ts index 6f30c6c6d..ec21b4bec 100644 --- a/src/app/api/auth/me/mfa/totp/confirm/route.ts +++ b/src/app/api/auth/me/mfa/totp/confirm/route.ts @@ -37,6 +37,7 @@ import { destroyOtherSessions } from "@/lib/auth/session"; import { decrypt } from "@/lib/crypto"; import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; import { verifyTotp } from "@/lib/auth/mfa/totp"; +import { loadFactorUser } from "@/lib/auth/mfa/verify-factor"; import { generateRecoveryCodes, persistRecoveryCodes, @@ -69,7 +70,9 @@ export const POST = apiHandler(async (req: Request) => { if (user.totpConfirmedAt) { throw new HttpError(409, "A second factor is already active"); } - if (!user.totpSecretEncrypted) { + // The pending secret and its replay floor are not on the per-request user. + const pending = await loadFactorUser(user.id); + if (!pending?.totpSecretEncrypted) { throw new HttpError(409, "Start enrollment before confirming"); } @@ -90,11 +93,11 @@ export const POST = apiHandler(async (req: Request) => { // Decrypt is fail-closed — a missing/rotated key throws rather than // confirming against junk. - const secret = decrypt(user.totpSecretEncrypted); + const secret = decrypt(pending.totpSecretEncrypted); const result = verifyTotp( secret, parsed.data.code, - user.totpLastStep === null ? null : Number(user.totpLastStep), + pending.totpLastStep === null ? null : Number(pending.totpLastStep), ); if (!result.valid) { diff --git a/src/app/api/auth/me/route.ts b/src/app/api/auth/me/route.ts index 41a819538..09cebfdc0 100644 --- a/src/app/api/auth/me/route.ts +++ b/src/app/api/auth/me/route.ts @@ -162,6 +162,7 @@ export const GET = apiHandler(async () => { onboardingRow, ai, coachReasoning, + identity, ] = await Promise.all([ // The actor's own row is already in hand; a switched session needs the // record's `gender`, which is the column the cycle gate derives from. @@ -235,6 +236,13 @@ export const GET = apiHandler(async () => { recordId, authority: aiAuthority, }), + // The encrypted insurance number is not on the per-request user + // (`SESSION_USER_SELECT`); it is read here, for the one response that + // shows it. + prisma.user.findUnique({ + where: { id: user.id }, + select: { insuranceNumberEncrypted: true }, + }), ]); // The masking step, and the one place the record scoping is narrowed rather @@ -286,9 +294,9 @@ export const GET = apiHandler(async () => { // one row never 500s the whole profile fetch (the field just reads // null and the user re-enters it). let insuranceNumber: string | null = null; - if (user.insuranceNumberEncrypted) { + if (identity?.insuranceNumberEncrypted) { try { - insuranceNumber = decrypt(user.insuranceNumberEncrypted); + insuranceNumber = decrypt(identity.insuranceNumberEncrypted); } catch { insuranceNumber = null; } diff --git a/src/app/api/auth/passkeys/[id]/__tests__/route.test.ts b/src/app/api/auth/passkeys/[id]/__tests__/route.test.ts index c4e4adde5..2d8f7039c 100644 --- a/src/app/api/auth/passkeys/[id]/__tests__/route.test.ts +++ b/src/app/api/auth/passkeys/[id]/__tests__/route.test.ts @@ -21,6 +21,7 @@ vi.mock("@/lib/api-handler", () => ({ vi.mock("@/lib/db", () => ({ prisma: { + user: { findUnique: vi.fn() }, passkey: { findUnique: vi.fn(), update: vi.fn(), @@ -40,7 +41,9 @@ import { DELETE, PATCH } from "../route"; import { requireAuth, requireMfaManagementAuth } from "@/lib/api-handler"; import { prisma } from "@/lib/db"; -const USER = { id: "user-1", username: "u", passwordHash: "argon2id$…" }; +const USER = { id: "user-1", username: "u" }; +/** The stored credential row; the session user carries no hash. */ +const STORED = { passwordHash: "argon2id$…" }; const ROW = { id: "pk-1", @@ -75,6 +78,7 @@ const commitElevation = vi.fn().mockResolvedValue(undefined); beforeEach(() => { vi.resetAllMocks(); + vi.mocked(prisma.user.findUnique).mockResolvedValue(STORED as never); vi.mocked(requireAuth).mockResolvedValue({ user: USER } as never); commitElevation.mockClear().mockResolvedValue(undefined); vi.mocked(requireMfaManagementAuth).mockResolvedValue({ @@ -256,9 +260,12 @@ describe("DELETE /api/auth/passkeys/{id} — the removal's gate", () => { name: "Only one", } as never); vi.mocked(prisma.passkey.count).mockResolvedValue(1 as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + passwordHash: null, + } as never); vi.mocked(requireMfaManagementAuth).mockResolvedValue({ transport: "cookie", - user: { ...USER, passwordHash: null }, + user: USER, session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, commitElevation, } as never); diff --git a/src/app/api/auth/passkeys/[id]/route.ts b/src/app/api/auth/passkeys/[id]/route.ts index 87cd24b77..38fb8ed38 100644 --- a/src/app/api/auth/passkeys/[id]/route.ts +++ b/src/app/api/auth/passkeys/[id]/route.ts @@ -128,10 +128,14 @@ export const DELETE = apiHandler( } // Check: at least 1 auth method must remain - const passkeyCount = await prisma.passkey.count({ - where: { userId: user.id }, - }); - const hasPassword = !!user.passwordHash; + const [passkeyCount, credential] = await Promise.all([ + prisma.passkey.count({ where: { userId: user.id } }), + prisma.user.findUnique({ + where: { id: user.id }, + select: { passwordHash: true }, + }), + ]); + const hasPassword = !!credential?.passwordHash; if (passkeyCount <= 1 && !hasPassword) { return apiError( diff --git a/src/app/api/auth/password/__tests__/route.test.ts b/src/app/api/auth/password/__tests__/route.test.ts index 1ebb1bf31..5cee4d7cd 100644 --- a/src/app/api/auth/password/__tests__/route.test.ts +++ b/src/app/api/auth/password/__tests__/route.test.ts @@ -20,7 +20,7 @@ vi.mock("@/lib/api-handler", async () => { vi.mock("@/lib/db", () => ({ prisma: { - user: { update: vi.fn() }, + user: { update: vi.fn(), findUnique: vi.fn() }, }, })); @@ -92,7 +92,6 @@ const SESSION_OK = { username: "tester", email: "t@example.com", role: "USER" as const, - passwordHash: "h", onboardingCompletedAt: new Date(), locale: "en", }, @@ -109,6 +108,10 @@ function postReq(body: unknown): NextRequest { beforeEach(() => { vi.resetAllMocks(); vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + // The hash is read off the stored row, not the session user. + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + passwordHash: "h", + } as never); vi.mocked(checkRateLimit).mockResolvedValue({ allowed: true } as never); // Default: account clears the step-up (no MFA enrolled, or a fresh factor). vi.mocked(requireFreshMfaIfEnrolled).mockResolvedValue(SESSION_OK as never); diff --git a/src/app/api/auth/password/route.ts b/src/app/api/auth/password/route.ts index 629473990..db421e863 100644 --- a/src/app/api/auth/password/route.ts +++ b/src/app/api/auth/password/route.ts @@ -58,11 +58,18 @@ export const POST = apiHandler(async (request: NextRequest) => { } const { currentPassword, newPassword } = parsed.data; - if (!user.passwordHash) { + const credential = await prisma.user.findUnique({ + where: { id: user.id }, + select: { passwordHash: true }, + }); + if (!credential?.passwordHash) { return apiError("No password set for this account", 400); } - const currentValid = await verifyPassword(user.passwordHash, currentPassword); + const currentValid = await verifyPassword( + credential.passwordHash, + currentPassword, + ); if (!currentValid) { await recordReproofFailure( user.id, diff --git a/src/app/api/record-settings/integrations/route.ts b/src/app/api/record-settings/integrations/route.ts index ced499e0b..027e26b17 100644 --- a/src/app/api/record-settings/integrations/route.ts +++ b/src/app/api/record-settings/integrations/route.ts @@ -38,38 +38,50 @@ export const GET = apiHandler(async () => { const context = await requireGuardianAuth(); const access = resolveGuardianRecordSettingsAccess(context); if (!access) throw new SharingAccessDeniedError(); - const [statuses, withings, whoop, fitbit, googleHealth] = await Promise.all([ - Promise.all( - MANAGED_INTEGRATIONS.map((integration) => - getIntegrationStatus(access.recordId, integration), + const [statuses, withings, whoop, fitbit, googleHealth, credentials] = + await Promise.all([ + Promise.all( + MANAGED_INTEGRATIONS.map((integration) => + getIntegrationStatus(access.recordId, integration), + ), ), - ), - prisma.withingsConnection.findUnique({ - where: { userId: access.recordId }, - select: { id: true }, - }), - prisma.whoopConnection.findUnique({ - where: { userId: access.recordId }, - select: { whoopUserId: true }, - }), - prisma.fitbitConnection.findUnique({ - where: { userId: access.recordId }, - select: { id: true }, - }), - prisma.googleHealthConnection.findUnique({ - where: { userId: access.recordId }, - select: { id: true }, - }), - ]); + prisma.withingsConnection.findUnique({ + where: { userId: access.recordId }, + select: { id: true }, + }), + prisma.whoopConnection.findUnique({ + where: { userId: access.recordId }, + select: { whoopUserId: true }, + }), + prisma.fitbitConnection.findUnique({ + where: { userId: access.recordId }, + select: { id: true }, + }), + prisma.googleHealthConnection.findUnique({ + where: { userId: access.recordId }, + select: { id: true }, + }), + // Presence only; the credentials themselves are not on the per-request + // user and are not read past this null check. + prisma.user.findUnique({ + where: { id: context.user.id }, + select: { + nightscoutUrlEncrypted: true, + polarAccessTokenEncrypted: true, + ouraAccessTokenEncrypted: true, + stravaAccessTokenEncrypted: true, + }, + }), + ]); const connected: Record = { withings: withings !== null, whoop: whoop?.whoopUserId != null, fitbit: fitbit !== null, "google-health": googleHealth !== null, - nightscout: context.user.nightscoutUrlEncrypted !== null, - polar: context.user.polarAccessTokenEncrypted !== null, - oura: context.user.ouraAccessTokenEncrypted !== null, - strava: context.user.stravaAccessTokenEncrypted !== null, + nightscout: (credentials?.nightscoutUrlEncrypted ?? null) !== null, + polar: (credentials?.polarAccessTokenEncrypted ?? null) !== null, + oura: (credentials?.ouraAccessTokenEncrypted ?? null) !== null, + strava: (credentials?.stravaAccessTokenEncrypted ?? null) !== null, // Not offered on a managed profile (no managed integrations before // v1.44); listed so the map stays total. dawarich: false, diff --git a/src/lib/api-handler.ts b/src/lib/api-handler.ts index 5f0f9c9fa..35f907691 100644 --- a/src/lib/api-handler.ts +++ b/src/lib/api-handler.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import { headers } from "next/headers"; -import type { AccountGrant, User } from "@/generated/prisma/client"; +import type { AccountGrant } from "@/generated/prisma/client"; +import { SESSION_USER_SELECT, type SessionUser } from "./auth/session-user"; import { applyBypassRouteHeaders, bypassRouteRefusal, @@ -472,7 +473,7 @@ export type AuthContext = { */ readonly recordEpoch?: number; }; - user: User; + user: SessionUser; /** * Transport the caller authenticated with, derived from the SAME branch that * sets the wide event's `auth_method`: the cookie-session path resolves @@ -886,7 +887,7 @@ export async function requireActorAuth(): Promise { */ export interface RecordAuthContext extends AuthContext { /** The authenticated caller. Equals `user` when the caller acts as themselves. */ - readonly actor: User; + readonly actor: SessionUser; /** The grant being exercised, or null when the caller acts as themselves. */ readonly grantId: string | null; } @@ -1125,7 +1126,7 @@ export async function requireGuardianAuth(): Promise { async function resolveSwitchedRecord( auth: AuthContext, carrier: Extract, - admit: (grant: AccountGrant, owner: User) => string | null, + admit: (grant: AccountGrant, owner: SessionUser) => string | null, ): Promise { /** Every refusal below is this one error, with the reason kept off the wire. */ const denied = (reason: string): SharingAccessDeniedError => { @@ -1155,6 +1156,7 @@ async function resolveSwitchedRecord( const owner = await prisma.user.findUnique({ where: { id: grant.grantorId }, + select: SESSION_USER_SELECT, }); // Unreachable while the FK cascade holds — a grant naming a deleted account // is deleted with it. Fail closed anyway: an authorization we cannot resolve @@ -1584,7 +1586,7 @@ export async function requireRecentProof(options: { * caller themselves (the admin routes, which must stay on `requireAdmin`). */ export async function assertRecentCookieProof( - user: User, + user: SessionUser, sessionId: string, ): Promise { const enrolled = await hasSecondFactorEnrolled(user); @@ -1713,7 +1715,7 @@ export const STEP_UP_ELEVATION_HEADER = "x-step-up"; * own device login. A comment would not have caught that; a type does. */ export type MfaManagementContext = { - user: User; + user: SessionUser; /** * Spend the elevation. Call it immediately BEFORE the mutation, once every * cheap validation has passed — a 429, a 422, or a wrong factor code must not @@ -1837,7 +1839,7 @@ export async function requireMfaManagementAuth( * the wire. */ async function resolveBearerElevation( - auth: { user: User; apiTokenId: string }, + auth: { user: SessionUser; apiTokenId: string }, options: { freshFactor: boolean; proofSource: FreshFactorProofSource }, ): Promise<() => Promise> { const { freshFactor, proofSource } = options; diff --git a/src/lib/auth/__tests__/enrollment-proof-second-factor.test.ts b/src/lib/auth/__tests__/enrollment-proof-second-factor.test.ts index 0caecb261..a6dac826c 100644 --- a/src/lib/auth/__tests__/enrollment-proof-second-factor.test.ts +++ b/src/lib/auth/__tests__/enrollment-proof-second-factor.test.ts @@ -14,6 +14,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; vi.mock("@/lib/db", () => ({ prisma: { session: { findUnique: vi.fn(), update: vi.fn() }, + user: { findUnique: vi.fn() }, webauthnMfaCredential: { count: vi.fn() }, passkey: { count: vi.fn() }, authChallenge: { findUnique: vi.fn() }, @@ -38,7 +39,10 @@ vi.mock("@/lib/auth/mfa/webauthn", () => ({ verifyMfaAuthentication: vi.fn(), createMfaAuthenticationOptions: vi.fn(), })); -vi.mock("@/lib/auth/mfa/verify-factor", () => ({ verifyMfaFactor: vi.fn() })); +vi.mock("@/lib/auth/mfa/verify-factor", () => ({ + verifyMfaFactor: vi.fn(), + loadFactorUser: vi.fn(), +})); vi.mock("@/lib/api-handler", () => ({ apiHandler: (fn: unknown) => fn, requireCookieAuth: vi.fn(), @@ -47,7 +51,7 @@ vi.mock("@/lib/api-handler", () => ({ import { prisma } from "@/lib/db"; import { checkRateLimit, refundRateLimit } from "@/lib/rate-limit"; import { verifyPassword } from "@/lib/auth/password"; -import { verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; +import { loadFactorUser, verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; import { createRegistrationOptions } from "@/lib/auth/passkey"; import { requireCookieAuth } from "@/lib/api-handler"; import { @@ -79,6 +83,21 @@ const ENROLLED: { }; const PLAIN = { ...ENROLLED, totpSecretEncrypted: null, totpConfirmedAt: null }; +/** + * The account as the request sees it: its secrets live in the stored row, + * which the verifiers read for themselves, and not on the session user. + */ +function stored(user: typeof ENROLLED | typeof PLAIN) { + vi.mocked(prisma.user.findUnique).mockResolvedValue(user as never); + vi.mocked(loadFactorUser).mockResolvedValue(user as never); + return { + id: user.id, + username: user.username, + email: user.email, + totpConfirmedAt: user.totpConfirmedAt, + }; +} + function req(body?: unknown): Request { return new Request("http://localhost/api/auth/me/mfa/totp/setup", { method: "POST", @@ -101,7 +120,7 @@ function session(stamps: { function gate(user: typeof ENROLLED | typeof PLAIN, request: Request) { return checkCookieEnrollmentProof({ - user: user as never, + user: stored(user), sessionId: "sess-1", request, ipAddress: "203.0.113.9", @@ -198,7 +217,7 @@ describe("POST /api/auth/passkey/register-options", () => { it("refuses a password on an account with a second factor, before verifying it", async () => { vi.mocked(requireCookieAuth).mockResolvedValue({ - user: ENROLLED, + user: stored(ENROLLED), session: { id: "sess-1" }, } as never); const res = await PASSKEY_REGISTER_OPTIONS( @@ -214,7 +233,7 @@ describe("POST /api/auth/passkey/register-options", () => { it("still takes a password on an account without one", async () => { vi.mocked(requireCookieAuth).mockResolvedValue({ - user: PLAIN, + user: stored(PLAIN), session: { id: "sess-1" }, } as never); vi.mocked(verifyPassword).mockResolvedValue(true); @@ -236,7 +255,7 @@ describe("changing the email address", () => { opts: { cookie?: boolean; password?: string | null } = {}, ) { return authorizeSensitiveChange({ - user: user as never, + user: stored(user), cookieSessionId: opts.cookie === false ? null : "sess-1", currentPassword: opts.password ?? null, ipAddress: "203.0.113.9", diff --git a/src/lib/auth/bearer.ts b/src/lib/auth/bearer.ts index 8abb7997f..3af3317fe 100644 --- a/src/lib/auth/bearer.ts +++ b/src/lib/auth/bearer.ts @@ -14,9 +14,9 @@ * HTTP path keeps its existing `auth.bearer.*` audit trail), so this layer adds * no I/O beyond the two reads the validation needs. */ -import type { User } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import { hashToken } from "@/lib/auth/hmac"; +import { SESSION_USER_SELECT, type SessionUser } from "@/lib/auth/session-user"; /** Why a Bearer token was rejected. Stable handles for audit details. */ export type BearerAuthReason = @@ -67,7 +67,7 @@ export class BearerAuthError extends Error { /** The validated identity behind a Bearer token. */ export interface BearerResolution { - user: User; + user: SessionUser; /** The `ApiToken` row id (the token-id half of the `:` binding). */ tokenId: string; /** The token's granted scopes (`["*"]` = wildcard). */ @@ -147,6 +147,7 @@ export async function resolveBearerToken( const user = await prisma.user.findUnique({ where: { id: apiToken.userId }, + select: SESSION_USER_SELECT, }); if (!user) { diff --git a/src/lib/auth/existing-factor-proof.ts b/src/lib/auth/existing-factor-proof.ts index d4d46ad08..a2e5d5fe5 100644 --- a/src/lib/auth/existing-factor-proof.ts +++ b/src/lib/auth/existing-factor-proof.ts @@ -16,7 +16,6 @@ * (same key, so its counter carries straight over). A shared bucket means a * guesser cannot multiply the budget by rotating between routes. */ -import type { User } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import { apiError } from "@/lib/api-response"; import { REPROOF_REQUIRED_CODE } from "@/lib/api-errors"; @@ -34,7 +33,7 @@ import { import { verifyPassword } from "@/lib/auth/password"; import { verifyAuthentication } from "@/lib/auth/passkey"; import { verifyMfaAuthentication } from "@/lib/auth/mfa/webauthn"; -import { verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; +import { loadFactorUser, verifyMfaFactor } from "@/lib/auth/mfa/verify-factor"; import { stepUpMintSchema, type StepUpMintRequest, @@ -93,6 +92,9 @@ export async function refundReproof(userId: string): Promise { export type ProofOutcome = { ok: true } | { ok: false; reason: string }; +/** The caller a proof is checked for: who they are and whether TOTP is live. */ +type ProofUser = { id: string; totpConfirmedAt: Date | null }; + /** * Verify a proof against the account. Never throws for a bad proof: a * malformed or expired WebAuthn challenge is a failed attempt, not a 500. @@ -102,24 +104,25 @@ export type ProofOutcome = { ok: true } | { ok: false; reason: string }; * A TOTP code is accepted only from a confirmed secret (`verifyMfaFactor`). */ export async function verifyExistingFactorProof( - user: Pick< - User, - | "id" - | "passwordHash" - | "totpSecretEncrypted" - | "totpLastStep" - | "totpConfirmedAt" - >, + user: ProofUser, proof: ExistingFactorProof, ): Promise { + // The secrets are read here, for this one check: the caller's row is the + // per-request projection, which carries none of them. if (proof.method === "password") { - if (!user.passwordHash) return { ok: false, reason: "no_password" }; - const ok = await verifyPassword(user.passwordHash, proof.password); + const row = await prisma.user.findUnique({ + where: { id: user.id }, + select: { passwordHash: true }, + }); + if (!row?.passwordHash) return { ok: false, reason: "no_password" }; + const ok = await verifyPassword(row.passwordHash, proof.password); return ok ? { ok } : { ok: false, reason: "bad_password" }; } if (proof.method === "totp") { - const result = await verifyMfaFactor(user, "totp", proof.code); + const factorUser = await loadFactorUser(user.id); + if (!factorUser) return { ok: false, reason: "bad_totp" }; + const result = await verifyMfaFactor(factorUser, "totp", proof.code); if (result.ok) return { ok: true }; return { ok: false, reason: result.replay ? "totp_replay" : "bad_totp" }; } @@ -230,7 +233,7 @@ export const RECENT_PROOF_MAX_AGE_MS = 5 * 60 * 1000; * throttled and audited through the shared re-proof bucket. */ export async function checkCookieEnrollmentProof(args: { - user: User; + user: ProofUser; sessionId: string; request: Request; ipAddress: string | null; @@ -373,14 +376,7 @@ export type SensitiveChangeProof = * of such an account; the shipped app does not change it. */ export async function authorizeSensitiveChange(args: { - user: Pick< - User, - | "id" - | "passwordHash" - | "totpSecretEncrypted" - | "totpLastStep" - | "totpConfirmedAt" - >; + user: ProofUser; /** The session row id on a cookie request; null on a Bearer request. */ cookieSessionId: string | null; currentPassword: string | null; diff --git a/src/lib/auth/mfa/verify-factor.ts b/src/lib/auth/mfa/verify-factor.ts index 36536af2c..eea2f834e 100644 --- a/src/lib/auth/mfa/verify-factor.ts +++ b/src/lib/auth/mfa/verify-factor.ts @@ -36,6 +36,28 @@ export interface FactorUser { totpConfirmedAt: Date | null; } +/** + * Load the factor columns for one check, at the moment of the check. + * + * The per-request user (`SESSION_USER_SELECT`) carries no secret material, so + * a route that verifies a factor for an already-authenticated caller reads the + * secret and its replay floor here. Reading them fresh is also the right + * answer for the floor: it is the value the guarded update below compares. + */ +export async function loadFactorUser( + userId: string, +): Promise { + return prisma.user.findUnique({ + where: { id: userId }, + select: { + id: true, + totpSecretEncrypted: true, + totpLastStep: true, + totpConfirmedAt: true, + }, + }); +} + export interface FactorResult { ok: boolean; /** TOTP only: the code matched the secret but its step was already used. */ diff --git a/src/lib/auth/second-factor.ts b/src/lib/auth/second-factor.ts index 78d5ea8e6..b28935755 100644 --- a/src/lib/auth/second-factor.ts +++ b/src/lib/auth/second-factor.ts @@ -1,4 +1,3 @@ -import type { User } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; /** @@ -53,19 +52,25 @@ export type RecentProofMethod = "password" | "totp" | "webauthn" | "passkey"; * endpoint cannot disagree. */ export async function recentProofMethods( - user: Pick, + user: { id: string; totpConfirmedAt: Date | null }, enrolled?: boolean, ): Promise { const hasSecond = enrolled ?? (await hasSecondFactorEnrolled(user)); - const [keys, passkeys] = await Promise.all([ + // Whether a password exists is read here rather than off the caller's row: + // the hash is not part of the per-request user (`SESSION_USER_SELECT`). + const [keys, passkeys, credential] = await Promise.all([ prisma.webauthnMfaCredential.count({ where: { userId: user.id } }), prisma.passkey.count({ where: { userId: user.id } }), + prisma.user.findUnique({ + where: { id: user.id }, + select: { passwordHash: true }, + }), ]); const methods: RecentProofMethod[] = []; if (hasSecond) { if (user.totpConfirmedAt) methods.push("totp"); if (keys > 0) methods.push("webauthn"); - } else if (user.passwordHash) { + } else if (credential?.passwordHash) { methods.push("password"); } if (passkeys > 0) methods.push("passkey"); diff --git a/src/lib/auth/session-user.ts b/src/lib/auth/session-user.ts new file mode 100644 index 000000000..d54cfeccd --- /dev/null +++ b/src/lib/auth/session-user.ts @@ -0,0 +1,126 @@ +import type { Prisma } from "@/generated/prisma/client"; + +/** + * The columns of `User` that ride on every authenticated request. + * + * Every cookie session and every Bearer token resolves its caller through one + * of the lookups in `session.ts` / `bearer.ts`, so whatever this projection + * names is read once per request, for every request the app serves. The row it + * replaced was the whole `users` record, and that record carries a profile + * picture of up to a few megabytes, the cached briefing text, the encrypted + * home location and air-quality history, and every stored integration + * credential. None of that is needed to answer "who is calling", and all of it + * was copied into a fresh buffer on each request. + * + * So the projection is an allowlist: small scalars that the request path reads + * (identity, role, locale and display preferences, the onboarding and MFA + * flags, the small preference documents). Anything else is read by the one + * route that needs it, with its own `select`, at the moment it needs it: + * + * - bytes and large text: `avatarBytes`, `insightsCachedText`, + * `insightsCachedTextEncrypted`, `homeLocationEncrypted`, + * `environmentAqHistoryJson`, `dawarichPrefsEncrypted`; + * - credentials and secrets: `passwordHash`, `totpSecretEncrypted`, + * `totpLastStep`, `telegramBotToken`, every `*Encrypted` integration and + * provider key, `insuranceNumberEncrypted`. + * + * `src/__tests__/session-user-projection-guard.test.ts` holds both halves: no + * caller lookup may load the full row, and none of the columns above may join + * this list. + */ +export const SESSION_USER_SELECT = { + id: true, + username: true, + email: true, + role: true, + oidcIssuer: true, + oidcSub: true, + createdAt: true, + updatedAt: true, + managedProfileAt: true, + heightCm: true, + dateOfBirth: true, + gender: true, + timezone: true, + homeTimezone: true, + homeSince: true, + environmentAirQualityEnabled: true, + codexTokenExpiresAt: true, + codexConnectedAt: true, + codexConnectionStatus: true, + useCentralCodex: true, + insightsPrivacyMode: true, + insightsCachedAt: true, + insightsCachedLocale: true, + insightsSnapshotHash: true, + insightsWarmFailedAt: true, + insightsBriefingRerollDate: true, + morningDigestRefreshedOn: true, + telegramChatId: true, + telegramEnabled: true, + nightscoutAllowPrivateHost: true, + dawarichAllowPrivateHost: true, + ouraScopeDenied: true, + stravaAthleteId: true, + stravaLastActivityAt: true, + stravaBackfillCompletedAt: true, + stravaBackfillResumeBefore: true, + locale: true, + thresholdsJson: true, + dashboardWidgetsJson: true, + insightsLayoutJson: true, + medicationListLayoutJson: true, + documentsLayoutJson: true, + moodTagLayoutJson: true, + glucoseUnit: true, + hasDiabetes: true, + labsLocalOcrEnabled: true, + unitPreference: true, + timeFormat: true, + dateFormat: true, + aiProvider: true, + aiModel: true, + aiBaseUrl: true, + aiCompatBaseUrl: true, + aiCompatModel: true, + aiProviderChain: true, + aiResponseTimeoutSeconds: true, + documentsAutoAiRead: true, + onboardingCompletedAt: true, + onboardingTourCompleted: true, + onboardingTourProgressJson: true, + disclaimerAcknowledgedAt: true, + disclaimerAcknowledgedVersion: true, + healthKitConfigJson: true, + healthKitLastSyncedAt: true, + healthKitLastSyncTrigger: true, + healthKitLastBackgroundSyncAt: true, + displayName: true, + lastReportPracticeName: true, + coachPrefsJson: true, + sourcePriorityJson: true, + lastSyncedAt: true, + syncResetAt: true, + moodReminderEnabled: true, + reportSelectionJson: true, + notificationPrefs: true, + disableCoach: true, + aiUsageLevel: true, + modulePreferencesJson: true, + healthScoreConfigJson: true, + documentQuotaBytes: true, + coachLastSeenAt: true, + avatarContentType: true, + avatarUpdatedAt: true, + totpConfirmedAt: true, + mfaEnforced: true, + passkeyUpgradeNudgeDismissed: true, + fullName: true, + insurerName: true, + insurerIkNumber: true, +} as const satisfies Prisma.UserSelect; + +/** The caller as every authenticated request sees them. */ +export type SessionUser = Prisma.UserGetPayload<{ + select: typeof SESSION_USER_SELECT; +}>; diff --git a/src/lib/auth/session.ts b/src/lib/auth/session.ts index 72dc0e0af..06ec6381b 100644 --- a/src/lib/auth/session.ts +++ b/src/lib/auth/session.ts @@ -1,8 +1,8 @@ import { randomBytes } from "node:crypto"; import { prisma } from "@/lib/db"; import { cookies } from "next/headers"; -import type { User } from "@/generated/prisma/client"; import { hashToken } from "@/lib/auth/hmac"; +import { SESSION_USER_SELECT, type SessionUser } from "@/lib/auth/session-user"; import { ensureDbCompatibility } from "@/lib/db-compat"; import { getEvent } from "@/lib/logging/context"; import { shouldEmitSecureCookie } from "@/lib/auth/secure-cookie"; @@ -99,13 +99,13 @@ async function findSessionByCookie(cookieValue: string) { if (cookieValue.startsWith(SESSION_TOKEN_PREFIX)) { return await prisma.session.findUnique({ where: { tokenHash: hashToken(cookieValue) }, - include: { user: true }, + include: { user: { select: SESSION_USER_SELECT } }, }); } const legacy = await prisma.session.findUnique({ where: { id: cookieValue }, - include: { user: true }, + include: { user: { select: SESSION_USER_SELECT } }, }); // A row that already holds a secret has retired its id as a credential. return legacy && legacy.tokenHash === null ? legacy : null; @@ -148,7 +148,10 @@ async function findSessionByCookie(cookieValue: string) { */ export async function validateSessionFromCookieValue( cookieValue: string | undefined | null, -): Promise<{ session: { id: string; expiresAt: Date }; user: User } | null> { +): Promise<{ + session: { id: string; expiresAt: Date }; + user: SessionUser; +} | null> { if (!cookieValue) return null; const session = await findSessionByCookie(cookieValue); if (!session || session.expiresAt < new Date()) return null; @@ -179,7 +182,7 @@ export async function validateSessionWithCreatedAt( cookieValue: string | undefined | null, ): Promise<{ session: { id: string; createdAt: Date; expiresAt: Date }; - user: User; + user: SessionUser; } | null> { if (!cookieValue) return null; const session = await findSessionByCookie(cookieValue); @@ -296,7 +299,7 @@ export async function getSession(): Promise<{ actingAsUserId: string | null; recordEpoch: number; }; - user: User; + user: SessionUser; } | null> { try { await ensureDbCompatibility(); diff --git a/src/lib/daily/daily-briefing-push.ts b/src/lib/daily/daily-briefing-push.ts index de858255f..1fa56dab9 100644 --- a/src/lib/daily/daily-briefing-push.ts +++ b/src/lib/daily/daily-briefing-push.ts @@ -26,7 +26,7 @@ * on `MEASUREMENT_ANOMALY`, never here. It fires ONLY inside a local morning * window, which is the quiet-hours guarantee for this nudge. */ -import type { User } from "@/generated/prisma/client"; +import { SESSION_USER_SELECT, type SessionUser } from "@/lib/auth/session-user"; import type { PrismaClient } from "@/generated/prisma/client"; import { annotate, getEvent } from "@/lib/logging/context"; @@ -82,11 +82,15 @@ export type DailyBriefingDispatchResult = export interface DailyBriefingDispatchDeps { dispatch?: typeof dispatchNotification; - loadDigest?: (user: User, now: Date, locale: Locale) => Promise; + loadDigest?: ( + user: SessionUser, + now: Date, + locale: Locale, + ) => Promise; } function loadDailyBriefingDigest( - user: User, + user: SessionUser, now: Date, locale: Locale, ): Promise { @@ -181,7 +185,10 @@ export async function maybeDispatchDailyBriefing( const loadDigest = deps.loadDigest ?? loadDailyBriefingDigest; try { - const user = await prisma.user.findUnique({ where: { id: userId } }); + const user = await prisma.user.findUnique({ + where: { id: userId }, + select: SESSION_USER_SELECT, + }); if (!user) { annotate({ action: { name: "daily.briefing_push.missing_user" } }); return "missing-user"; diff --git a/src/lib/daily/load-digest.ts b/src/lib/daily/load-digest.ts index d08043635..a79699426 100644 --- a/src/lib/daily/load-digest.ts +++ b/src/lib/daily/load-digest.ts @@ -12,7 +12,7 @@ * module-gated at the build layer) plus the resolved module map the item * builders consult. */ -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; @@ -547,7 +547,7 @@ export interface DailyDigestLoadOptions { } export async function loadDailyDigest( - user: User, + user: SessionUser, now: Date = new Date(), options: DailyDigestLoadOptions = {}, ): Promise { diff --git a/src/lib/dashboard/__tests__/snapshot-read-briefing-capability.test.ts b/src/lib/dashboard/__tests__/snapshot-read-briefing-capability.test.ts index 5c16fa762..ebe57a1d8 100644 --- a/src/lib/dashboard/__tests__/snapshot-read-briefing-capability.test.ts +++ b/src/lib/dashboard/__tests__/snapshot-read-briefing-capability.test.ts @@ -11,7 +11,7 @@ */ import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; import { AI_AVAILABLE, aiUnavailable, @@ -29,7 +29,19 @@ const briefingBody = { const buildDashboardSnapshot = vi.fn(async () => briefingBody); const aiCapabilityToServe = vi.fn(); -vi.mock("@/lib/db", () => ({ prisma: {} })); +// Generated a moment ago, so the today rule keeps it and these tests see +// only the capability decision. Read off its own row, not the session user. +vi.mock("@/lib/db", () => ({ + prisma: { + user: { + findUnique: async () => ({ + insightsCachedText: JSON.stringify({ + briefingGeneratedAt: new Date().toISOString(), + }), + }), + }, + }, +})); vi.mock("@/lib/dashboard/snapshot", async (importOriginal) => ({ ...(await importOriginal()), buildDashboardSnapshot: () => buildDashboardSnapshot(), @@ -44,16 +56,11 @@ vi.mock("@/lib/ai/capabilities/gate", () => ({ const { readDashboardSnapshotCached } = await import("../snapshot-read"); const { __resetAllCachesForTests } = await import("@/lib/cache/server-cache"); -// Generated a moment ago, so the today rule keeps it and these tests see -// only the capability decision. const USER = { id: "user-briefing-gate", locale: "en", timezone: "UTC", - insightsCachedText: JSON.stringify({ - briefingGeneratedAt: new Date().toISOString(), - }), -} as unknown as User; +} as unknown as SessionUser; beforeEach(() => { __resetAllCachesForTests(); diff --git a/src/lib/dashboard/__tests__/snapshot-read-briefing-today.test.ts b/src/lib/dashboard/__tests__/snapshot-read-briefing-today.test.ts index 9be475763..7cf124e4a 100644 --- a/src/lib/dashboard/__tests__/snapshot-read-briefing-today.test.ts +++ b/src/lib/dashboard/__tests__/snapshot-read-briefing-today.test.ts @@ -5,7 +5,7 @@ */ import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; import { AI_AVAILABLE, aiUnavailable, @@ -47,7 +47,13 @@ const body = (pulseLastSeen: string) => ({ let current = body("2026-10-04T06:50:00.000Z"); const buildDashboardSnapshot = vi.fn(async () => current); -vi.mock("@/lib/db", () => ({ prisma: {} })); +// The cached briefing text is read off its own row, not the session user. +const cacheRow = vi.hoisted(() => ({ + current: null as { insightsCachedText: string | null } | null, +})); +vi.mock("@/lib/db", () => ({ + prisma: { user: { findUnique: async () => cacheRow.current } }, +})); vi.mock("@/lib/dashboard/snapshot", async (importOriginal) => ({ ...(await importOriginal()), buildDashboardSnapshot: () => buildDashboardSnapshot(), @@ -72,16 +78,18 @@ const { __resetAllCachesForTests } = await import("@/lib/cache/server-cache"); /** 2026-10-05 21:00 in Berlin. */ const NOW = new Date("2026-10-05T19:00:00.000Z"); -function user(generatedAt: string | null): User { - return { - id: "user-briefing-today", - locale: "en", - timezone: "Europe/Berlin", +function user(generatedAt: string | null): SessionUser { + cacheRow.current = { insightsCachedText: JSON.stringify({ dailyBriefing: BRIEFING, ...(generatedAt ? { briefingGeneratedAt: generatedAt } : {}), }), - } as unknown as User; + }; + return { + id: "user-briefing-today", + locale: "en", + timezone: "Europe/Berlin", + } as unknown as SessionUser; } beforeEach(() => { diff --git a/src/lib/dashboard/__tests__/snapshot-read-cache-key.test.ts b/src/lib/dashboard/__tests__/snapshot-read-cache-key.test.ts index 317455e16..5abc61ed3 100644 --- a/src/lib/dashboard/__tests__/snapshot-read-cache-key.test.ts +++ b/src/lib/dashboard/__tests__/snapshot-read-cache-key.test.ts @@ -14,12 +14,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; const buildDashboardSnapshot = vi.fn(async () => ({ tiles: {} })); const resolveServerLocale = vi.fn(async () => "de"); -vi.mock("@/lib/db", () => ({ prisma: {} })); +vi.mock("@/lib/db", () => ({ + prisma: { user: { findUnique: async () => null } }, +})); vi.mock("@/lib/dashboard/snapshot", () => ({ buildDashboardSnapshot: (...args: unknown[]) => buildDashboardSnapshot(...(args as [])), @@ -46,7 +48,7 @@ const { invalidateUserMedications, } = await import("@/lib/cache/invalidate"); -const USER = { id: "user-key-guard", locale: "de" } as unknown as User; +const USER = { id: "user-key-guard", locale: "de" } as unknown as SessionUser; beforeEach(() => { __resetAllCachesForTests(); diff --git a/src/lib/dashboard/snapshot-read.ts b/src/lib/dashboard/snapshot-read.ts index 23857a904..cd98809e0 100644 --- a/src/lib/dashboard/snapshot-read.ts +++ b/src/lib/dashboard/snapshot-read.ts @@ -21,7 +21,7 @@ * that moves the registry back off the global breaks this paragraph, not * just a performance assumption. */ -import type { User } from "@/generated/prisma/client"; +import type { SessionUser } from "@/lib/auth/session-user"; import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; @@ -41,7 +41,10 @@ import { briefingForToday } from "@/lib/daily/briefing-today"; import { requestTodayBriefingWarm } from "@/lib/daily/briefing-today-warm"; import { readBriefingGeneratedAt } from "@/lib/insights/briefing-generated-at"; import { userDayKey } from "@/lib/tz/format"; -import { openInsightsCachedText } from "@/lib/insights/cached-text-cipher"; +import { + INSIGHTS_CACHE_SELECT, + openInsightsCachedText, +} from "@/lib/insights/cached-text-cipher"; /** * Per-key TTL for the snapshot cache entry. Strictly greater than the @@ -118,12 +121,29 @@ export interface SnapshotReadOptions { * timing surface; the RSC prefetch passes none. */ export async function readDashboardSnapshotCached( - user: User, + user: SessionUser, time?: (label: string, builder: () => Promise) => Promise, options: SnapshotReadOptions = {}, ): Promise { - // `User` row is already in hand at both call sites — no extra round-trip. - const cachedText = openInsightsCachedText(user); + // The cached briefing text is not on the per-request user + // (`SESSION_USER_SELECT`), so it is read here, beside the locale: every + // read needs it for the generated-at stamp below, not only a cache miss. + const [cacheRow, locale] = await Promise.all([ + prisma.user.findUnique({ + where: { id: user.id }, + select: INSIGHTS_CACHE_SELECT, + }), + // v1.21.2 — the briefing recall + forward-look is locale-specific + // prose, so the snapshot cache key carries the resolved locale: an EN and + // a DE session never share a snapshot cell (and never see each other's + // memory wording). The `${user.id}|` prefix still covers the key under + // the measurement-write stale-sweep. + resolveServerLocale({ + userLocale: user.locale, + override: options.locale ?? null, + }), + ]); + const cachedText = openInsightsCachedText(cacheRow); const snapshotUser: SnapshotUserInput = { id: user.id, username: user.username, @@ -143,16 +163,6 @@ export async function readDashboardSnapshotCached( healthScoreConfigJson: user.healthScoreConfigJson, }; - // v1.21.2 — the briefing recall + forward-look is locale-specific - // prose, so the snapshot cache key carries the resolved locale: an EN and - // a DE session never share a snapshot cell (and never see each other's - // memory wording). The `${user.id}|` prefix still covers the key under - // the measurement-write stale-sweep. - const locale = await resolveServerLocale({ - userLocale: user.locale, - override: options.locale ?? null, - }); - // Stale-while-revalidate: a measurement write marks the analytics // bucket stale rather than hard-evicting the snapshot, so a busy iOS // sync serves the prior snapshot immediately (within the bucket's diff --git a/src/lib/jobs/reaction-line.ts b/src/lib/jobs/reaction-line.ts index 1051aae19..4ec108073 100644 --- a/src/lib/jobs/reaction-line.ts +++ b/src/lib/jobs/reaction-line.ts @@ -58,6 +58,7 @@ import { import { extractAssessmentSummary } from "@/lib/insights/status-shared"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { fenceUserText } from "@/lib/ai/coach/data-fence"; +import { SESSION_USER_SELECT } from "@/lib/auth/session-user"; import { singleUserTurn, type CompletionResult } from "@/lib/ai/types"; import type { Locale } from "@/lib/i18n/config"; import { resolveJobLocale } from "@/lib/i18n/job-locale"; @@ -500,7 +501,10 @@ export async function runReactionLine( }; } - const user = await prisma.user.findUnique({ where: { id: job.userId } }); + const user = await prisma.user.findUnique({ + where: { id: job.userId }, + select: SESSION_USER_SELECT, + }); if (!user) return { status: "skipped", reason: "no_user" }; const locale = await resolveJobLocale(user.locale); From 8cdb53cb0e144411084c43d0f9abda755c75b9f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:13:13 +0200 Subject: [PATCH 05/23] perf(withings): write a measure sync in sets and refold only what changed A manual full sync re-reads the whole history, and nearly all of it is already stored. Each reading cost a findFirst plus an update, and every re-read day was marked dirty, so the run refolded the DAY tier and queued three coarse recomputes for every day of the account: about thirty seconds of database work behind 1.4 s of Withings calls. The measures leg now reads the stored rows of the window once, skips the unchanged ones, updates the changed or tombstoned ones and inserts the new ones in batches. Only inserted or changed readings count as touched. recomputeBucketsForKeys refolds exactly the touched DAY buckets in one aggregate and one transaction, and queues one coarse job per run of adjacent buckets instead of three per day. --- src/lib/rollups/measurement-rollups.ts | 170 ++++++++++++++- .../__tests__/sync-measure-watermark.test.ts | 49 +++-- src/lib/withings/__tests__/sync-plan.test.ts | 31 +-- .../__tests__/sync-plausibility.test.ts | 42 ++-- src/lib/withings/sync.ts | 171 +++++++++------ tests/integration/measurement-rollups.test.ts | 164 ++++++++++++++ .../integration/withings-sync-volume.test.ts | 204 ++++++++++++++++++ 7 files changed, 725 insertions(+), 106 deletions(-) create mode 100644 tests/integration/withings-sync-volume.test.ts diff --git a/src/lib/rollups/measurement-rollups.ts b/src/lib/rollups/measurement-rollups.ts index cf33a2c6d..bfaf64b92 100644 --- a/src/lib/rollups/measurement-rollups.ts +++ b/src/lib/rollups/measurement-rollups.ts @@ -358,6 +358,143 @@ export async function recomputeBucketsForMeasurement( ); } +/** Keys one set-based DAY refold carries; bounds the statement's arrays. */ +const KEYED_REFOLD_CHUNK = 2_000; + +/** + * The set-based sibling of {@link recomputeBucketsForMeasurement}, for a write + * that touched many `(type, day)` buckets at once (a sync, an import, a batch). + * + * The per-key hook costs an aggregate, a transaction and three queue sends per + * bucket, so a sync that re-read two years of history paid that seven thousand + * times over. Here the DAY tier is refolded for exactly the touched buckets in + * one aggregate and one transaction per chunk, and the coarser tiers are + * enqueued once per contiguous run of buckets rather than once per day. The + * result is the one the per-key loop produced: every touched day re-derived + * from its live rows, a day left empty dropped, untouched days left alone. + */ +export async function recomputeBucketsForKeys( + userId: string, + rows: Array<{ type: MeasurementType; measuredAt: Date }>, +): Promise { + const keys = collapseToTypeDayKeys(rows); + for (let i = 0; i < keys.length; i += KEYED_REFOLD_CHUNK) { + await refoldDayBuckets(userId, keys.slice(i, i + KEYED_REFOLD_CHUNK)); + } + for (const granularity of ASYNC_GRANULARITIES) { + for (const run of bucketRuns(keys, granularity)) { + await enqueueRollupRecompute({ userId, granularity, ...run }); + } + } +} + +async function refoldDayBuckets( + userId: string, + keys: Array<{ type: MeasurementType; measuredAt: Date }>, +): Promise { + if (keys.length === 0) return; + const types = [...new Set(keys.map((k) => k.type))]; + const days = [...new Set(keys.map((k) => k.measuredAt.getTime()))].sort( + (a, b) => a - b, + ); + const wanted = new Set( + keys.map((k) => `${k.type}|${k.measuredAt.getTime()}`), + ); + const aggregate = await runRollupAggregate({ + userId, + types, + granularity: "DAY", + from: new Date(days[0]!), + to: new Date(days[days.length - 1]! + 24 * 60 * 60 * 1000), + days: days.map((d) => new Date(d)), + }); + // The day filter is per day, not per (type, day): another touched type's + // day comes back too. Its aggregate would be right, but it was not touched, + // so it is not rewritten. + const ordered = aggregate + .filter((r) => + wanted.has(`${r.type}|${new Date(r.bucket_start).getTime()}`), + ) + .sort((a, b) => + a.type !== b.type + ? a.type < b.type + ? -1 + : 1 + : a.source !== b.source + ? a.source < b.source + ? -1 + : 1 + : a.bucket_start.getTime() - b.bucket_start.getTime(), + ); + // Delete exactly the touched buckets (every source, so a source that left a + // day does not strand its row, and a day that emptied out is dropped), then + // upsert the fresh aggregate, in one transaction like `persistRollupRows`. + await prisma.$transaction( + async (tx) => { + await tx.$executeRawUnsafe( + `DELETE FROM measurement_rollups + WHERE user_id = $1 + AND granularity = 'DAY'::"measurement_rollup_granularity" + AND (type, bucket_start) IN ( + SELECT k.t::"measurement_type", k.b + FROM unnest($2::text[], $3::timestamptz[]) AS k(t, b) + )`, + userId, + keys.map((k) => k.type), + keys.map((k) => k.measuredAt.toISOString()), + ); + for (let i = 0; i < ordered.length; i += 500) { + const { sql, params } = buildRollupUpsert( + userId, + "DAY", + ordered.slice(i, i + 500), + ); + await tx.$executeRawUnsafe(sql, ...params); + } + }, + { maxWait: 10_000, timeout: 120_000 }, + ); +} + +/** + * The coarse buckets the touched days fall in, merged per type into runs of + * adjacent buckets: one recompute job per run, each covering whole buckets. + */ +function bucketRuns( + keys: Array<{ type: MeasurementType; measuredAt: Date }>, + granularity: RollupGranularity, +): Array<{ type: MeasurementType; from: Date; to: Date }> { + const spans = new Map< + string, + { type: MeasurementType; from: Date; to: Date } + >(); + for (const k of keys) { + const span = bucketSpan(k.measuredAt, granularity); + spans.set(`${k.type}|${span.from.getTime()}`, { type: k.type, ...span }); + } + const sorted = [...spans.values()].sort((a, b) => + a.type !== b.type + ? a.type < b.type + ? -1 + : 1 + : a.from.getTime() - b.from.getTime(), + ); + const runs: Array<{ type: MeasurementType; from: Date; to: Date }> = []; + for (const span of sorted) { + const last = runs[runs.length - 1]; + if ( + last && + last.type === span.type && + last.to.getTime() === span.from.getTime() + ) { + last.to = span.to; + } else { + runs.push({ ...span }); + } + } + return runs; +} + /** * Best-effort pg-boss enqueue. Silent no-op when no boss is attached * (route is running in a context without the worker — typically the @@ -399,7 +536,16 @@ export async function enqueueRollupRecompute(input: { // is still QUEUED — safe here precisely because the handler re-reads the // bucket's live rows when it starts, so the queued job still sees writes // that land after it was enqueued. - singletonKey: `${input.userId}|${input.type}|${input.granularity}|${input.from.toISOString()}`, + // + // A job covering a run of several buckets (`recomputeBucketsForKeys`) + // carries its end in the key too: under the bare start it would collapse + // into a queued single-bucket job and the rest of the run would be lost. + singletonKey: + `${input.userId}|${input.type}|${input.granularity}|${input.from.toISOString()}` + + (bucketSpan(input.from, input.granularity).to.getTime() === + input.to.getTime() + ? "" + : `|${input.to.toISOString()}`), }); } @@ -480,10 +626,21 @@ async function runRollupAggregate( granularity: RollupGranularity; from: Date; to: Date; + /** + * Only the readings of these UTC days, inside `[from, to)`. A keyed + * refold (`recomputeBucketsForKeys`) passes the touched days so a handful + * spread across years does not aggregate everything in between. + */ + days?: Date[]; }, db: Pick = prisma, ): Promise { if (input.types && input.types.length === 0) return []; + // Bound as the fourth parameter, never spliced. + const dayFilter = input.days + ? `AND date_trunc('day', m."measured_at") = ANY($4::timestamptz[])` + : ""; + const dayParams = input.days ? [input.days.map((d) => d.toISOString())] : []; // Postgres rejects `date_trunc($1::text, m."measured_at")` in a // GROUP BY because the planner sees the parameterised expression @@ -553,6 +710,7 @@ async function runRollupAggregate( AND m."measured_at" >= $2 AND m."measured_at" < $3 AND m."deleted_at" IS NULL + ${dayFilter} GROUP BY m."type", m."source", ${dateTrunc} `; return applyHourlyMeanDay( @@ -563,6 +721,7 @@ async function runRollupAggregate( input.userId, input.from, input.to, + ...dayParams, ), ); } @@ -600,6 +759,7 @@ async function runRollupAggregate( AND m."measured_at" >= $2 AND m."measured_at" < $3 AND m."deleted_at" IS NULL + ${dayFilter} GROUP BY m."type", m."source", ${dateTrunc} `; return applyHourlyMeanDay( @@ -610,6 +770,7 @@ async function runRollupAggregate( input.userId, input.from, input.to, + ...dayParams, ), ); } @@ -635,6 +796,7 @@ async function applyHourlyMeanDay( granularity: RollupGranularity; from: Date; to: Date; + days?: Date[]; }, rows: RollupRow[], ): Promise { @@ -685,6 +847,11 @@ async function applyHourlyMeanDay( AND m."measured_at" >= $2 AND m."measured_at" < $3 AND m."deleted_at" IS NULL + ${ + input.days + ? `AND date_trunc('day', m."measured_at") = ANY($4::timestamptz[])` + : "" + } GROUP BY m."type", m."source", date_trunc('hour', m."measured_at") ) h GROUP BY 1, 2, 3 @@ -694,6 +861,7 @@ async function applyHourlyMeanDay( input.userId, input.from, input.to, + ...(input.days ? [input.days.map((d) => d.toISOString())] : []), ); const byKey = new Map( means.map((m) => [ diff --git a/src/lib/withings/__tests__/sync-measure-watermark.test.ts b/src/lib/withings/__tests__/sync-measure-watermark.test.ts index 3dcb7ae9b..a242e89c0 100644 --- a/src/lib/withings/__tests__/sync-measure-watermark.test.ts +++ b/src/lib/withings/__tests__/sync-measure-watermark.test.ts @@ -16,8 +16,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("@/lib/db", () => ({ prisma: { measurement: { - findFirst: vi.fn(), - create: vi.fn(), + findMany: vi.fn(), + createManyAndReturn: vi.fn(), update: vi.fn(), }, withingsConnection: { @@ -66,7 +66,7 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { >("@/lib/rollups/measurement-rollups"); return { ...actual, - recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForKeys: vi.fn().mockResolvedValue(undefined), }; }); @@ -96,10 +96,6 @@ import { import { getValidToken, syncUserMeasurements } from "../sync"; -const P2002 = Object.assign(new Error("Unique constraint failed"), { - code: "P2002", -}); - beforeEach(() => { vi.clearAllMocks(); vi.mocked(prisma.$transaction).mockImplementation(async (run) => @@ -116,8 +112,15 @@ beforeEach(() => { lastSyncedAt: new Date(Date.now() - 60 * 60 * 1000), } as never); vi.mocked(prisma.withingsConnection.update).mockResolvedValue({} as never); - vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null); - vi.mocked(prisma.measurement.create).mockResolvedValue({} as never); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([]); + vi.mocked(prisma.measurement.createManyAndReturn).mockImplementation( + (async (args: { data: Array<{ type: string; measuredAt: Date }> }) => + args.data.map((row, i) => ({ + id: `m-${i}`, + type: row.type, + measuredAt: row.measuredAt, + }))) as never, + ); vi.mocked(prisma.measurement.update).mockResolvedValue({} as never); }); @@ -168,10 +171,20 @@ describe("syncUserMeasurements — watermark hold on hard row failure (F4)", () measuredAt: new Date("2026-05-16T08:05:00Z"), }, ] as never); - // The first write hard-fails (a transient DB error), the second succeeds. - vi.mocked(prisma.measurement.create) - .mockRejectedValueOnce(new Error("connection reset by peer")) - .mockResolvedValue({} as never); + // The weight is stored with an older value; its update hard-fails (a + // transient DB error). The pulse is new and inserts. + vi.mocked(prisma.measurement.findMany).mockResolvedValue([ + { + id: "m-weight", + type: "WEIGHT", + measuredAt: new Date("2026-05-16T08:00:00Z"), + value: 79, + deletedAt: null, + }, + ] as never); + vi.mocked(prisma.measurement.update).mockRejectedValueOnce( + new Error("connection reset by peer"), + ); const result = await syncUserMeasurements("user-1"); @@ -187,7 +200,7 @@ describe("syncUserMeasurements — watermark hold on hard row failure (F4)", () ); }); - it("does NOT hold the watermark for a benign P2002 idempotent-write collision", async () => { + it("does NOT hold the watermark for a duplicate a concurrent run inserted first", async () => { vi.mocked(fetchMeasurements).mockResolvedValue([ { type: "WEIGHT", @@ -195,11 +208,15 @@ describe("syncUserMeasurements — watermark hold on hard row failure (F4)", () measuredAt: new Date("2026-05-16T08:00:00Z"), }, ] as never); - vi.mocked(prisma.measurement.create).mockRejectedValueOnce(P2002); + // A concurrent run inserted the same reading first: the batched insert + // skips it as a duplicate. + vi.mocked(prisma.measurement.createManyAndReturn).mockResolvedValueOnce( + [] as never, + ); const { imported } = await syncUserMeasurements("user-1"); - // P2002 means the row is already present — nothing lost — so the cycle is + // A skipped duplicate is already present — nothing lost — so the cycle is // still healthy: stamp the watermark + success, record no failure. expect(imported).toBe(0); expect(prisma.withingsConnection.update).toHaveBeenCalledTimes(1); diff --git a/src/lib/withings/__tests__/sync-plan.test.ts b/src/lib/withings/__tests__/sync-plan.test.ts index 89834898e..1cf80a2a5 100644 --- a/src/lib/withings/__tests__/sync-plan.test.ts +++ b/src/lib/withings/__tests__/sync-plan.test.ts @@ -10,8 +10,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("@/lib/db", () => ({ prisma: { measurement: { - findFirst: vi.fn(), - create: vi.fn(), + findMany: vi.fn(), + createManyAndReturn: vi.fn(), update: vi.fn(), }, withingsConnection: { @@ -61,7 +61,7 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { >("@/lib/rollups/measurement-rollups"); return { ...actual, - recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForKeys: vi.fn().mockResolvedValue(undefined), }; }); @@ -179,8 +179,10 @@ function persistedLegs(): Array<{ leg: unknown; value: unknown }> { function createdTypes(): string[] { return vi - .mocked(prisma.measurement.create) - .mock.calls.map((c) => (c[0] as { data: { type: string } }).data.type); + .mocked(prisma.measurement.createManyAndReturn) + .mock.calls.flatMap((c) => + (c[0] as { data: Array<{ type: string }> }).data.map((d) => d.type), + ); } beforeEach(() => { @@ -193,14 +195,15 @@ beforeEach(() => { connection() as never, ); vi.mocked(prisma.withingsConnection.update).mockResolvedValue({} as never); - vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null); - vi.mocked(prisma.measurement.create).mockImplementation((async (args: { - data: { type: string; measuredAt: Date }; - }) => ({ - id: `m-${args.data.type}`, - type: args.data.type, - measuredAt: args.data.measuredAt, - })) as never); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([]); + vi.mocked(prisma.measurement.createManyAndReturn).mockImplementation( + (async (args: { data: Array<{ type: string; measuredAt: Date }> }) => + args.data.map((d) => ({ + id: `m-${d.type}`, + type: d.type, + measuredAt: d.measuredAt, + }))) as never, + ); vi.mocked(prisma.$executeRaw).mockResolvedValue(1 as never); }); @@ -338,7 +341,7 @@ describe("syncUserMeasurements inside the Withings API plan", () => { for (let run = 0; run < 4; run++) { vi.mocked(prisma.$executeRaw).mockClear(); - vi.mocked(prisma.measurement.create).mockClear(); + vi.mocked(prisma.measurement.createManyAndReturn).mockClear(); requestedStartdates.length = 0; await syncUserMeasurements("user-1"); const [persisted] = persistedLegs(); diff --git a/src/lib/withings/__tests__/sync-plausibility.test.ts b/src/lib/withings/__tests__/sync-plausibility.test.ts index c980998a5..b80a6715c 100644 --- a/src/lib/withings/__tests__/sync-plausibility.test.ts +++ b/src/lib/withings/__tests__/sync-plausibility.test.ts @@ -14,10 +14,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("@/lib/db", () => ({ prisma: { measurement: { - findFirst: vi.fn(), findMany: vi.fn(), - create: vi.fn(), createMany: vi.fn(), + createManyAndReturn: vi.fn(), update: vi.fn(), }, withingsConnection: { @@ -67,6 +66,7 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { return { ...actual, recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForKeys: vi.fn().mockResolvedValue(undefined), }; }); @@ -125,13 +125,15 @@ beforeEach(() => { lastSyncedAt: new Date(Date.now() - 60 * 60 * 1000), } as never); vi.mocked(prisma.withingsConnection.update).mockResolvedValue({} as never); - vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null); vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); - vi.mocked(prisma.measurement.create).mockResolvedValue({ - id: "m-1", - type: "PULSE", - measuredAt: new Date(), - } as never); + vi.mocked(prisma.measurement.createManyAndReturn).mockImplementation( + (async (args: { data: Array<{ type: string; measuredAt: Date }> }) => + args.data.map((d, i) => ({ + id: `m-${i}`, + type: d.type, + measuredAt: d.measuredAt, + }))) as never, + ); vi.mocked(prisma.measurement.createMany).mockResolvedValue({ count: 1, } as never); @@ -157,7 +159,11 @@ describe("syncUserMeasurements — plausibility gate", () => { ); expect(result).toEqual({ imported: 1, failed: false }); - expect(prisma.measurement.create).toHaveBeenCalledTimes(1); + expect(prisma.measurement.createManyAndReturn).toHaveBeenCalledTimes(1); + expect( + vi.mocked(prisma.measurement.createManyAndReturn).mock.calls[0]?.[0] + ?.data, + ).toHaveLength(1); expect(tally()).toBeUndefined(); }); @@ -182,7 +188,11 @@ describe("syncUserMeasurements — plausibility gate", () => { // Only the real reading landed, and the run is still a clean success: one // impossible sample must not strand the rest behind a held watermark. expect(result).toEqual({ imported: 1, failed: false }); - expect(prisma.measurement.create).toHaveBeenCalledTimes(1); + expect(prisma.measurement.createManyAndReturn).toHaveBeenCalledTimes(1); + expect( + vi.mocked(prisma.measurement.createManyAndReturn).mock.calls[0]?.[0] + ?.data, + ).toHaveLength(1); expect(tally()).toEqual({ total: 1, buckets: [ @@ -196,9 +206,15 @@ describe("syncUserMeasurements — plausibility gate", () => { // The slot already holds a real pulse; the provider re-sends the same // instant carrying a decode slip. Ungated, the update branch rewrote the // good value with the bad one. - vi.mocked(prisma.measurement.findFirst).mockResolvedValue({ - id: "existing-row", - } as never); + vi.mocked(prisma.measurement.findMany).mockResolvedValueOnce([ + { + id: "existing-row", + type: "PULSE", + measuredAt: new Date("2026-05-16T08:00:00Z"), + value: 61, + deletedAt: null, + }, + ] as never); vi.mocked(fetchMeasurements).mockResolvedValue([ { type: "PULSE", diff --git a/src/lib/withings/sync.ts b/src/lib/withings/sync.ts index 1c84a09b0..6bf90b248 100644 --- a/src/lib/withings/sync.ts +++ b/src/lib/withings/sync.ts @@ -29,7 +29,6 @@ import { } from "./api-plan"; import { getUserWithingsCredentials } from "./credentials"; import { annotate, getEvent } from "@/lib/logging/context"; -import { isP2002 } from "@/lib/prisma-errors"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { @@ -45,7 +44,7 @@ import { } from "./response-classifier"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate"; import { @@ -349,7 +348,8 @@ export async function syncUserMeasurements( // through Withings. We collect first, fold once at the end so a 100- // row monthly catch-up costs at most ~N (type, day) recomputes rather // than N per-row hooks. Best-effort: a populator hiccup never fails - // the user's sync. + // the user's sync. Only readings this run inserted or changed count as + // touched: a re-read reading that is already stored dirties nothing. const touched: Array<{ type: MeasurementType; measuredAt: Date }> = []; const insertedArrivals: Array<{ id: string; @@ -367,69 +367,118 @@ export async function syncUserMeasurements( let anyRowFailed = false; let firstRowError: unknown = null; + // The run reads a window (the whole history on a full sync) and most of it + // is usually stored already. One read of the stored rows in that window + // splits it three ways: unchanged (nothing to write, nothing to refold), + // changed or tombstoned (an update), and new (one batched insert). The + // per-row `findFirst` + `update` this replaced cost two round trips per + // reading and marked every re-read day dirty, which is where a full sync's + // forty seconds went. + const latest = new Map(); for (const m of measures) { - const measType = m.type as MeasurementType; - try { - // v1.4.25 — Migration 0055 added `sleepStage` to the - // composite unique with `NULLS NOT DISTINCT`, so a non-sleep - // upsert keyed on (userId, type, measuredAt, source) with - // sleepStage IS NULL is still safe. But Prisma's typed compound - // input requires a non-null `sleepStage`, so we model the - // idempotent write as a `findFirst` + `create`/`update`. The - // unique index still serializes concurrent inserts, so the worst - // case is a Prisma P2002 we catch and ignore. - const existing = await prisma.measurement.findFirst({ - where: { - userId, - type: measType, - measuredAt: m.measuredAt, - source: "WITHINGS", - sleepStage: null, + latest.set(`${m.type}|${m.measuredAt.getTime()}`, m); + } + const incoming = [...latest.values()]; + const existingByKey = new Map< + string, + { id: string; value: number; deletedAt: Date | null } + >(); + if (incoming.length > 0) { + let first = incoming[0]!.measuredAt; + let last = first; + for (const m of incoming) { + if (m.measuredAt < first) first = m.measuredAt; + if (m.measuredAt > last) last = m.measuredAt; + } + const stored = await prisma.measurement.findMany({ + where: { + userId, + source: "WITHINGS", + sleepStage: null, + type: { + in: [...new Set(incoming.map((m) => m.type as MeasurementType))], }, - select: { id: true }, + measuredAt: { gte: first, lte: last }, + }, + select: { + id: true, + type: true, + measuredAt: true, + value: true, + deletedAt: true, + }, + }); + for (const row of stored) { + existingByKey.set(`${row.type}|${row.measuredAt.getTime()}`, row); + } + } + + const toCreate: typeof incoming = []; + for (const m of incoming) { + const measType = m.type as MeasurementType; + const existing = existingByKey.get(`${m.type}|${m.measuredAt.getTime()}`); + if (!existing) { + toCreate.push(m); + continue; + } + if (existing.value === m.value && existing.deletedAt === null) { + imported++; + continue; + } + try { + await prisma.measurement.update({ + where: { id: existing.id }, + // `deletedAt: null` is a no-op on a live row, a deliberate + // RESURRECTION on a tombstoned one — Withings is the source of + // truth for its own rows, so a re-fetched measure brings the row + // back (mirrors Google / Fitbit). + data: { value: m.value, deletedAt: null }, }); - if (existing) { - await prisma.measurement.update({ - where: { id: existing.id }, - // `deletedAt: null` is a no-op on a live row, a deliberate - // RESURRECTION on a tombstoned one — Withings is the source of - // truth for its own rows, so a re-fetched measure brings the row - // back (mirrors Google / Fitbit). - data: { value: m.value, deletedAt: null }, - }); - } else { - const created = await prisma.measurement.create({ - data: { - userId, - type: measType, - value: m.value, - unit: getUnitForType(m.type), - measuredAt: m.measuredAt, - source: "WITHINGS", - }, - select: { id: true, type: true, measuredAt: true }, - }); - insertedArrivals.push(created); - } touched.push({ type: measType, measuredAt: m.measuredAt }); imported++; } catch (err) { - if (isP2002(err)) { - // Benign: the composite unique index serialised a concurrent insert of - // this exact reading — the row is present, nothing is lost. Warn and - // continue WITHOUT holding the watermark (this is the expected - // idempotent-write race the findFirst+create models around). - getEvent()?.addWarning( - `Withings measure upsert hit an idempotent P2002 (row already present): ${err}`, - ); - } else { - // A genuine write failure strands this reading. Flag the cycle so the - // watermark is held and the next scheduled tick refetches the overlap - // window and retries. - anyRowFailed = true; - if (firstRowError === null) firstRowError = err; - getEvent()?.addWarning(`Failed to upsert measure: ${err}`); + // A genuine write failure strands this reading. Flag the cycle so the + // watermark is held and the next scheduled tick refetches the overlap + // window and retries. + anyRowFailed = true; + if (firstRowError === null) firstRowError = err; + getEvent()?.addWarning(`Failed to update measure: ${err}`); + } + } + + // New readings in one statement per chunk. `skipDuplicates` absorbs the + // idempotent-write race the unique index serialises (a concurrent webhook + // run inserting the same reading): that row is present and nothing is lost, + // exactly the P2002 the per-row path caught and ignored. + for (let i = 0; i < toCreate.length; i += 1_000) { + const chunk = toCreate.slice(i, i + 1_000); + try { + const created = await prisma.measurement.createManyAndReturn({ + data: chunk.map((m) => ({ + userId, + type: m.type as MeasurementType, + value: m.value, + unit: getUnitForType(m.type), + measuredAt: m.measuredAt, + source: "WITHINGS" as const, + })), + skipDuplicates: true, + select: { id: true, type: true, measuredAt: true }, + }); + for (const row of created) { + insertedArrivals.push(row); + touched.push({ type: row.type, measuredAt: row.measuredAt }); } + imported += created.length; + if (created.length < chunk.length) { + getEvent()?.addWarning( + `Withings measure insert skipped ${chunk.length - created.length} rows already present (idempotent race)`, + ); + } + } catch (err) { + anyRowFailed = true; + if (firstRowError === null) firstRowError = err; + getEvent()?.addWarning(`Failed to insert measures: ${err}`); } } @@ -446,9 +495,7 @@ export async function syncUserMeasurements( // chart at the 30-day range under-counted recent days. try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); // v1.8.0 — a Withings sync that lands new weight / BP / pulse / // body-comp rows dirties the matching per-metric assessment caches. diff --git a/tests/integration/measurement-rollups.test.ts b/tests/integration/measurement-rollups.test.ts index 2d91a56e1..2e382c9a5 100644 --- a/tests/integration/measurement-rollups.test.ts +++ b/tests/integration/measurement-rollups.test.ts @@ -29,6 +29,7 @@ import { getPrismaClient, truncateAllTables } from "./setup"; import { ROLLUP_FULL_BACKFILL_QUEUE, enqueueBootTimeRollupBackfill, + recomputeBucketsForKeys, recomputeBucketsForMeasurement, recomputeUserRollups, ALL_GRANULARITIES, @@ -954,3 +955,166 @@ describe("measurement rollups — integration", () => { expect(aggregate.summaries.BLOOD_PRESSURE_SYS).toBeDefined(); }); }); + +describe("keyed DAY refold", () => { + /** The DAY tier as comparable rows, `computedAt` aside. */ + async function dayTier(userId: string) { + const rows = await getPrismaClient().measurementRollup.findMany({ + where: { userId, granularity: "DAY" }, + orderBy: [{ type: "asc" }, { bucketStart: "asc" }, { source: "asc" }], + }); + // Floating sums depend on the order the scan feeds them; the last bits + // of a double are not what the comparison is about. + return rows.map(({ computedAt: _computedAt, ...rest }) => + Object.fromEntries( + Object.entries(rest).map(([k, v]) => [ + k, + typeof v === "number" ? Number(v.toPrecision(12)) : v, + ]), + ), + ); + } + + it("leaves the tier the per-key hook leaves, touching nothing else", async () => { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username: "keyed-refold", email: "keyed-refold@example.test" }, + }); + const at = (iso: string) => new Date(iso); + const reading = ( + type: "PULSE" | "WEIGHT", + source: "MANUAL" | "WITHINGS", + iso: string, + value: number, + ) => ({ + userId: user.id, + type, + source, + value, + unit: type === "PULSE" ? "bpm" : "kg", + measuredAt: at(iso), + }); + await prisma.measurement.createMany({ + data: [ + // Two sources on one day, pulse over several hours (its day is a + // mean of hourly means), a weight two years earlier, and a day whose + // only reading is about to be deleted. + reading("PULSE", "WITHINGS", "2026-05-10T07:10:00.000Z", 58), + reading("PULSE", "WITHINGS", "2026-05-10T07:40:00.000Z", 66), + reading("PULSE", "MANUAL", "2026-05-10T19:00:00.000Z", 71), + reading("WEIGHT", "WITHINGS", "2024-03-02T06:00:00.000Z", 82.4), + reading("WEIGHT", "WITHINGS", "2026-05-11T06:00:00.000Z", 81.0), + reading("WEIGHT", "MANUAL", "2025-01-15T06:00:00.000Z", 83.3), + ], + }); + const keys = [ + { type: "PULSE" as const, measuredAt: at("2026-05-10T07:10:00.000Z") }, + { type: "WEIGHT" as const, measuredAt: at("2024-03-02T06:00:00.000Z") }, + { type: "WEIGHT" as const, measuredAt: at("2026-05-11T06:00:00.000Z") }, + ]; + // An untouched bucket, folded once, must keep its row as it is. + await recomputeBucketsForMeasurement( + user.id, + "WEIGHT", + at("2025-01-15T06:00:00.000Z"), + ); + for (const k of keys) { + await recomputeBucketsForMeasurement(user.id, k.type, k.measuredAt); + } + await prisma.measurement.updateMany({ + where: { userId: user.id, measuredAt: at("2026-05-11T06:00:00.000Z") }, + data: { deletedAt: new Date() }, + }); + await prisma.measurement.create({ + data: reading("PULSE", "WITHINGS", "2026-05-10T12:00:00.000Z", 90), + }); + const untouched = await prisma.measurementRollup.findFirstOrThrow({ + where: { + userId: user.id, + type: "WEIGHT", + granularity: "DAY", + bucketStart: at("2025-01-15T00:00:00.000Z"), + }, + }); + + // The per-key hook's answer, then the keyed refold's, from the same rows. + for (const k of keys) { + await recomputeBucketsForMeasurement(user.id, k.type, k.measuredAt); + } + const perKey = await dayTier(user.id); + await prisma.measurementRollup.deleteMany({ + where: { userId: user.id, bucketStart: { not: untouched.bucketStart } }, + }); + + const sends: Array<{ singletonKey: string }> = []; + vi.mocked(getGlobalBoss).mockReturnValue({ + send: vi.fn(async (_queue: string, _payload: unknown, opts: never) => { + sends.push(opts); + return "job"; + }), + } as never); + await recomputeBucketsForKeys(user.id, keys); + + expect(await dayTier(user.id)).toEqual(perKey); + expect( + perKey.some( + (r) => + (r.bucketStart as Date).getTime() === + at("2026-05-11T00:00:00Z").getTime(), + ), + ).toBe(false); + const after = await prisma.measurementRollup.findFirstOrThrow({ + where: { + userId: user.id, + type: "WEIGHT", + granularity: "DAY", + bucketStart: untouched.bucketStart, + }, + }); + expect(after.computedAt).toEqual(untouched.computedAt); + + // One coarse job per bucket here: no two touched buckets of one type are + // adjacent, so no run merges. + expect(sends.map((o) => o.singletonKey).sort()).toEqual( + [ + `${user.id}|PULSE|WEEK|2026-05-04T00:00:00.000Z`, + `${user.id}|PULSE|MONTH|2026-05-01T00:00:00.000Z`, + `${user.id}|PULSE|YEAR|2026-01-01T00:00:00.000Z`, + `${user.id}|WEIGHT|WEEK|2024-02-26T00:00:00.000Z`, + `${user.id}|WEIGHT|WEEK|2026-05-11T00:00:00.000Z`, + `${user.id}|WEIGHT|MONTH|2024-03-01T00:00:00.000Z`, + `${user.id}|WEIGHT|MONTH|2026-05-01T00:00:00.000Z`, + `${user.id}|WEIGHT|YEAR|2024-01-01T00:00:00.000Z`, + `${user.id}|WEIGHT|YEAR|2026-01-01T00:00:00.000Z`, + ].sort(), + ); + }); + + it("merges adjacent buckets into one ranged job", async () => { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username: "keyed-runs", email: "keyed-runs@example.test" }, + }); + const sends: Array<{ singletonKey: string }> = []; + vi.mocked(getGlobalBoss).mockReturnValue({ + send: vi.fn(async (_queue: string, _payload: unknown, opts: never) => { + sends.push(opts); + return "job"; + }), + } as never); + // Three consecutive weeks of a daily reading. + const keys = Array.from({ length: 21 }, (_, i) => ({ + type: "WEIGHT" as const, + measuredAt: new Date(Date.UTC(2026, 4, 4 + i, 6)), + })); + + await recomputeBucketsForKeys(user.id, keys); + + const weekKeys = sends + .map((o) => o.singletonKey) + .filter((k) => k.includes("|WEEK|")); + expect(weekKeys).toEqual([ + `${user.id}|WEIGHT|WEEK|2026-05-04T00:00:00.000Z|2026-05-25T00:00:00.000Z`, + ]); + }); +}); diff --git a/tests/integration/withings-sync-volume.test.ts b/tests/integration/withings-sync-volume.test.ts new file mode 100644 index 000000000..c2b97d9aa --- /dev/null +++ b/tests/integration/withings-sync-volume.test.ts @@ -0,0 +1,204 @@ +/** + * A Withings measure sync at the volume a full re-read of a real account + * carries: two years of daily weigh-ins (seven body-composition values each) + * and a daily blood-pressure reading, about 7 300 measures over 7 300 + * (type, day) buckets. + * + * The manual full sync re-reads the whole history, and nearly all of it is + * already stored. The run must therefore cost what CHANGED, not what was + * READ: one set-based read and write for the measures, and a rollup refold + * only for the days that gained or changed a reading. The timings are logged + * for the record; the assertions are on behaviour, so a slow CI host cannot + * fail them. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { encrypt } from "@/lib/crypto"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/withings/client", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, fetchMeasurements: vi.fn() }; +}); + +import { fetchMeasurements } from "@/lib/withings/client"; +import { syncUserMeasurements } from "@/lib/withings/sync"; + +const USER = "user-withings-volume"; +const DAY_MS = 86_400_000; +const DAYS = 730; + +const prisma = getPrismaClient(); + +const WEIGH_IN: Array<[string, number]> = [ + ["WEIGHT", 81.4], + ["BODY_FAT", 21.5], + ["FAT_MASS", 17.5], + ["FAT_FREE_MASS", 63.9], + ["MUSCLE_MASS", 60.7], + ["BONE_MASS", 3.2], + ["TOTAL_BODY_WATER", 46.1], +]; +const BP: Array<[string, number]> = [ + ["BLOOD_PRESSURE_SYS", 124], + ["BLOOD_PRESSURE_DIA", 79], + ["PULSE", 64], +]; + +/** Two years of history ending yesterday, at fixed UTC minutes. */ +function history(anchor: number) { + const out: Array<{ type: string; value: number; measuredAt: Date }> = []; + for (let d = DAYS; d >= 1; d -= 1) { + const day = anchor - d * DAY_MS; + for (const [type, base] of WEIGH_IN) { + out.push({ + type, + value: base + (d % 7) * 0.1, + measuredAt: new Date(day + 6 * 3_600_000 + 12 * 60_000), + }); + } + for (const [type, base] of BP) { + out.push({ + type, + value: base + (d % 5), + measuredAt: new Date(day + 20 * 3_600_000 + 41 * 60_000), + }); + } + } + return out; +} + +beforeEach(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { + id: USER, + username: "withings-volume", + email: "withings-volume@example.test", + }, + }); + await prisma.withingsConnection.create({ + data: { + userId: USER, + withingsUserId: "wu-volume", + accessToken: encrypt("access-token"), + refreshToken: encrypt("refresh-token"), + tokenExpiresAt: new Date(Date.now() + 60 * 60 * 1000), + scope: "user.metrics,user.activity", + }, + }); +}); + +afterEach(() => { + vi.mocked(fetchMeasurements).mockReset(); +}); + +describe("Withings full sync at account volume", () => { + it("writes the history once, then re-reads it at the cost of what changed", async () => { + const anchor = Date.UTC(2026, 9, 20); + const all = history(anchor); + vi.mocked(fetchMeasurements).mockResolvedValue(all); + + let started = performance.now(); + const first = await syncUserMeasurements(USER, { fullSync: true }); + const firstMs = performance.now() - started; + expect(first).toEqual({ imported: all.length, failed: false }); + expect( + await prisma.measurement.count({ + where: { userId: USER, source: "WITHINGS" }, + }), + ).toBe(all.length); + expect( + await prisma.measurementRollup.count({ + where: { userId: USER, granularity: "DAY" }, + }), + ).toBe(DAYS * (WEIGH_IN.length + BP.length)); + + // The re-read: the same history, one corrected weigh-in, one reading + // that was deleted here and comes back, and a new reading today. + const corrected = all.find((m) => m.type === "WEIGHT")!; + const resurrected = all.find((m) => m.type === "PULSE")!; + await prisma.measurement.updateMany({ + where: { + userId: USER, + type: "PULSE", + measuredAt: resurrected.measuredAt, + }, + data: { deletedAt: new Date() }, + }); + const today = { + type: "WEIGHT", + value: 80.9, + measuredAt: new Date(anchor + 7 * 3_600_000), + }; + const reread = [ + ...all.map((m) => (m === corrected ? { ...m, value: 79.0 } : m)), + today, + ]; + vi.mocked(fetchMeasurements).mockResolvedValue(reread); + + const before = new Date(); + started = performance.now(); + const second = await syncUserMeasurements(USER, { fullSync: true }); + const secondMs = performance.now() - started; + console.log( + `withings full sync: ${all.length} measures first run ${firstMs.toFixed(0)} ms, re-read ${secondMs.toFixed(0)} ms`, + ); + + expect(second).toEqual({ imported: reread.length, failed: false }); + const stored = await prisma.measurement.findMany({ + where: { + userId: USER, + source: "WITHINGS", + OR: [ + { type: "WEIGHT", measuredAt: corrected.measuredAt }, + { type: "PULSE", measuredAt: resurrected.measuredAt }, + { type: "WEIGHT", measuredAt: today.measuredAt }, + ], + }, + select: { type: true, value: true, measuredAt: true, deletedAt: true }, + }); + expect(stored).toHaveLength(3); + expect(stored.every((m) => m.deletedAt === null)).toBe(true); + expect( + stored.find( + (m) => m.measuredAt.getTime() === corrected.measuredAt.getTime(), + )?.value, + ).toBe(79.0); + + // Only the three days that changed were refolded; the other buckets keep + // the aggregate the first run wrote. + const refolded = await prisma.measurementRollup.findMany({ + where: { + userId: USER, + granularity: "DAY", + computedAt: { gte: before }, + }, + select: { type: true, bucketStart: true, mean: true }, + }); + const dayOf = (d: Date) => + new Date(Math.floor(d.getTime() / DAY_MS) * DAY_MS); + expect( + refolded.map((r) => [r.type, r.bucketStart.toISOString()]).sort(), + ).toEqual([ + ["PULSE", dayOf(resurrected.measuredAt).toISOString()], + ["WEIGHT", dayOf(corrected.measuredAt).toISOString()], + ["WEIGHT", dayOf(today.measuredAt).toISOString()], + ]); + expect( + refolded.find( + (r) => + r.type === "WEIGHT" && + r.bucketStart.getTime() === dayOf(corrected.measuredAt).getTime(), + )?.mean, + ).toBe(79.0); + }); +}); From f01f53a1cc491a5d8662e3591b994a5c9036659c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:20:00 +0200 Subject: [PATCH 06/23] perf(rollups): refold the buckets of every provider sync in one pass Every provider sync and the shared mutation tail refolded its touched (type, day) buckets one at a time: an aggregate, a transaction and three queue sends per day. They now hand the whole set to recomputeBucketsForKeys, which leaves the DAY tier exactly as the per-key hook did (pinned by a parity test against it) at the cost of one aggregate per chunk of keys and one coarse job per run of buckets. --- src/__tests__/measurement-write-tail-guard.test.ts | 4 +++- src/app/api/import/__tests__/route.test.ts | 12 +++++++++++- src/app/api/import/csv/__tests__/route.test.ts | 12 +++++++++++- .../measurements/bulk-delete/__tests__/route.test.ts | 12 +++++++++++- .../api/measurements/restore/__tests__/route.test.ts | 12 +++++++++++- src/lib/fitbit/sync-core.ts | 6 ++---- src/lib/google-health/sync-core.ts | 6 ++---- src/lib/nightscout/__tests__/sync.test.ts | 7 ++++++- src/lib/nightscout/sync.ts | 6 ++---- src/lib/oura/__tests__/sync.test.ts | 7 ++++++- src/lib/oura/sync.ts | 6 ++---- src/lib/polar/sync.ts | 6 ++---- .../__tests__/after-measurement-mutation.test.ts | 9 +++++++-- src/lib/rollups/after-measurement-mutation.ts | 8 +++----- src/lib/whoop/__tests__/sync.test.ts | 9 +++++++-- src/lib/whoop/sync-core.ts | 8 +++----- src/lib/withings/__tests__/sync-activity.test.ts | 12 +++++++++++- src/lib/withings/__tests__/sync-ecg.test.ts | 12 +++++++++++- src/lib/withings/__tests__/sync-sleep.test.ts | 12 +++++++++++- src/lib/withings/sync-activity.ts | 6 ++---- src/lib/withings/sync-ecg.ts | 6 ++---- src/lib/withings/sync-sleep.ts | 6 ++---- 22 files changed, 128 insertions(+), 56 deletions(-) diff --git a/src/__tests__/measurement-write-tail-guard.test.ts b/src/__tests__/measurement-write-tail-guard.test.ts index 4deacda2d..da045a6ed 100644 --- a/src/__tests__/measurement-write-tail-guard.test.ts +++ b/src/__tests__/measurement-write-tail-guard.test.ts @@ -176,7 +176,9 @@ describe("measurement write tail", () => { const missing = handRolled.filter((file) => { const src = readFileSync(join(SRC, file), "utf8"); return !( - /recomputeBucketsForMeasurement|afterMeasurementMutation/.test(src) && + /recomputeBucketsFor(Measurement|Keys)|afterMeasurementMutation/.test( + src, + ) && /invalidateStatusInsightsForTypes|afterMeasurementMutation/.test(src) ); }); diff --git a/src/app/api/import/__tests__/route.test.ts b/src/app/api/import/__tests__/route.test.ts index 096b72f8c..ef9e668fe 100644 --- a/src/app/api/import/__tests__/route.test.ts +++ b/src/app/api/import/__tests__/route.test.ts @@ -54,9 +54,19 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { const actual = await vi.importActual< typeof import("@/lib/rollups/measurement-rollups") >("@/lib/rollups/measurement-rollups"); + const perKey = vi.fn().mockResolvedValue(undefined); return { ...actual, - recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForMeasurement: perKey, + // The keyed refold, observed as the per-key recomputes it stands for. + recomputeBucketsForKeys: vi.fn( + async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await perKey(userId, k.type, k.measuredAt); + }, + ), }; }); diff --git a/src/app/api/import/csv/__tests__/route.test.ts b/src/app/api/import/csv/__tests__/route.test.ts index 3ca886bea..a6d8dcaa9 100644 --- a/src/app/api/import/csv/__tests__/route.test.ts +++ b/src/app/api/import/csv/__tests__/route.test.ts @@ -58,9 +58,19 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { const actual = await vi.importActual< typeof import("@/lib/rollups/measurement-rollups") >("@/lib/rollups/measurement-rollups"); + const perKey = vi.fn().mockResolvedValue(undefined); return { ...actual, - recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForMeasurement: perKey, + // The keyed refold, observed as the per-key recomputes it stands for. + recomputeBucketsForKeys: vi.fn( + async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await perKey(userId, k.type, k.measuredAt); + }, + ), }; }); diff --git a/src/app/api/measurements/bulk-delete/__tests__/route.test.ts b/src/app/api/measurements/bulk-delete/__tests__/route.test.ts index 3a3438a12..b41d53ccb 100644 --- a/src/app/api/measurements/bulk-delete/__tests__/route.test.ts +++ b/src/app/api/measurements/bulk-delete/__tests__/route.test.ts @@ -47,10 +47,20 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { const actual = await vi.importActual< typeof import("@/lib/rollups/measurement-rollups") >("@/lib/rollups/measurement-rollups"); + const perKey = vi.fn().mockResolvedValue(undefined); return { // keep the real collapse so the test exercises the actual fan-out logic collapseToTypeDayKeys: actual.collapseToTypeDayKeys, - recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForMeasurement: perKey, + // The keyed refold, observed as the per-key recomputes it stands for. + recomputeBucketsForKeys: vi.fn( + async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await perKey(userId, k.type, k.measuredAt); + }, + ), }; }); vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); diff --git a/src/app/api/measurements/restore/__tests__/route.test.ts b/src/app/api/measurements/restore/__tests__/route.test.ts index 8ef10c8c3..0fa7e8427 100644 --- a/src/app/api/measurements/restore/__tests__/route.test.ts +++ b/src/app/api/measurements/restore/__tests__/route.test.ts @@ -47,10 +47,20 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { const actual = await vi.importActual< typeof import("@/lib/rollups/measurement-rollups") >("@/lib/rollups/measurement-rollups"); + const perKey = vi.fn().mockResolvedValue(undefined); return { // keep the real collapse so the test exercises the actual fan-out logic collapseToTypeDayKeys: actual.collapseToTypeDayKeys, - recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForMeasurement: perKey, + // The keyed refold, observed as the per-key recomputes it stands for. + recomputeBucketsForKeys: vi.fn( + async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await perKey(userId, k.type, k.measuredAt); + }, + ), }; }); vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); diff --git a/src/lib/fitbit/sync-core.ts b/src/lib/fitbit/sync-core.ts index c636b3071..e52c656c9 100644 --- a/src/lib/fitbit/sync-core.ts +++ b/src/lib/fitbit/sync-core.ts @@ -18,7 +18,7 @@ import { getEvent } from "@/lib/logging/context"; import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { refreshAccessToken } from "./client"; import { getUserFitbitCredentials } from "./credentials"; @@ -894,9 +894,7 @@ export async function upsertFitbitMeasurements( try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); invalidateStatusInsightsForTypes( userId, keys.map((k) => k.type), diff --git a/src/lib/google-health/sync-core.ts b/src/lib/google-health/sync-core.ts index 542755bc8..4741df368 100644 --- a/src/lib/google-health/sync-core.ts +++ b/src/lib/google-health/sync-core.ts @@ -41,7 +41,7 @@ import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit import { recordSyncFailure, type FailureKind } from "@/lib/integrations/status"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate"; import { @@ -1587,9 +1587,7 @@ export async function upsertGoogleHealthMeasurements( try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); invalidateStatusInsightsForTypes( userId, keys.map((k) => k.type), diff --git a/src/lib/nightscout/__tests__/sync.test.ts b/src/lib/nightscout/__tests__/sync.test.ts index e43560469..18f3c8a9b 100644 --- a/src/lib/nightscout/__tests__/sync.test.ts +++ b/src/lib/nightscout/__tests__/sync.test.ts @@ -43,7 +43,12 @@ vi.mock("@/lib/integrations/status", async (importOriginal) => ({ vi.mock("@/lib/rollups/measurement-rollups", () => ({ collapseToTypeDayKeys: (rows: Array<{ type: string; measuredAt: Date }>) => rows.map((r) => ({ type: r.type, measuredAt: r.measuredAt })), - recomputeBucketsForMeasurement: recomputeMock, + recomputeBucketsForKeys: async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await recomputeMock(userId, k.type, k.measuredAt); + }, })); vi.mock("@/lib/insights/comprehensive-generate", () => ({ diff --git a/src/lib/nightscout/sync.ts b/src/lib/nightscout/sync.ts index c1c0edc92..fa3998a0b 100644 --- a/src/lib/nightscout/sync.ts +++ b/src/lib/nightscout/sync.ts @@ -34,7 +34,7 @@ import type { IntegrationClassification } from "@/lib/integrations/http-status-c import type { SyncWriteResult } from "@/lib/outcome/written-outcome"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { emitInsertedMeasurementArrivals, @@ -327,9 +327,7 @@ export async function upsertNightscoutEntries( try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); invalidateStatusInsightsForTypes( userId, keys.map((k) => k.type), diff --git a/src/lib/oura/__tests__/sync.test.ts b/src/lib/oura/__tests__/sync.test.ts index 299f6e892..1f2d22728 100644 --- a/src/lib/oura/__tests__/sync.test.ts +++ b/src/lib/oura/__tests__/sync.test.ts @@ -98,7 +98,12 @@ vi.mock("@/lib/integrations/status", async (importOriginal) => ({ vi.mock("@/lib/rollups/measurement-rollups", () => ({ collapseToTypeDayKeys: (rows: Array<{ type: string; measuredAt: Date }>) => rows.map((r) => ({ type: r.type, measuredAt: r.measuredAt })), - recomputeBucketsForMeasurement: recomputeMock, + recomputeBucketsForKeys: async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await recomputeMock(userId, k.type, k.measuredAt); + }, })); vi.mock("@/lib/insights/comprehensive-generate", () => ({ diff --git a/src/lib/oura/sync.ts b/src/lib/oura/sync.ts index d401fa2b6..0a5c9b384 100644 --- a/src/lib/oura/sync.ts +++ b/src/lib/oura/sync.ts @@ -45,7 +45,7 @@ import { } from "@/lib/integrations/status"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { emitInsertedMeasurementArrivals, @@ -805,9 +805,7 @@ export async function upsertOuraMeasurements( ); try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); invalidateStatusInsightsForTypes( userId, keys.map((k) => k.type), diff --git a/src/lib/polar/sync.ts b/src/lib/polar/sync.ts index 29147bf24..fede4d187 100644 --- a/src/lib/polar/sync.ts +++ b/src/lib/polar/sync.ts @@ -48,7 +48,7 @@ import { } from "@/lib/integrations/status"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { invalidateUserMeasurements } from "@/lib/cache/invalidate"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate"; @@ -398,9 +398,7 @@ export async function upsertPolarMeasurements( ); try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); invalidateStatusInsightsForTypes( userId, keys.map((k) => k.type), diff --git a/src/lib/rollups/__tests__/after-measurement-mutation.test.ts b/src/lib/rollups/__tests__/after-measurement-mutation.test.ts index 681356457..cf80d3cb6 100644 --- a/src/lib/rollups/__tests__/after-measurement-mutation.test.ts +++ b/src/lib/rollups/__tests__/after-measurement-mutation.test.ts @@ -4,7 +4,7 @@ * invalidation). * * Watched red: with the recompute loop removed from the helper the - * first test fails naming the missing `recomputeBucketsForMeasurement` + * first test fails naming the missing `recomputeBucketsForKeys` * call; with the `.catch` isolation removed the failure-isolation test * throws. Both verified red against a deliberately broken helper before * the real implementation landed. @@ -27,7 +27,12 @@ vi.mock("@/lib/rollups/measurement-rollups", async (importOriginal) => { await importOriginal(); return { ...actual, - recomputeBucketsForMeasurement: mocks.recompute, + recomputeBucketsForKeys: async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await mocks.recompute(userId, k.type, k.measuredAt); + }, }; }); diff --git a/src/lib/rollups/after-measurement-mutation.ts b/src/lib/rollups/after-measurement-mutation.ts index f12c17160..aa501d1d6 100644 --- a/src/lib/rollups/after-measurement-mutation.ts +++ b/src/lib/rollups/after-measurement-mutation.ts @@ -8,7 +8,7 @@ * * 1. the `(type, day)` rollup buckets the mutation dirtied — recomputed * inline (DAY) + enqueued (WEEK/MONTH/YEAR) via - * `recomputeBucketsForMeasurement`. The recompute handles deletions + * `recomputeBucketsForKeys`. The recompute handles deletions * inherently: a day left empty drops its rollup row. * 2. the cached per-metric status assessments — re-warmed via * `invalidateStatusInsightsForTypes` (fire-and-forget). @@ -20,7 +20,7 @@ */ import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { invalidateStatusInsightsForTypes, @@ -50,9 +50,7 @@ export async function afterMeasurementMutation( // 500-row batch fires at most one DAY recompute per touched bucket. const keys = collapseToTypeDayKeys(rows); try { - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); } catch (err) { console.warn(`[${label}] rollup recompute failed`, err); } diff --git a/src/lib/whoop/__tests__/sync.test.ts b/src/lib/whoop/__tests__/sync.test.ts index 82e4b9e84..969bc0ee3 100644 --- a/src/lib/whoop/__tests__/sync.test.ts +++ b/src/lib/whoop/__tests__/sync.test.ts @@ -90,8 +90,13 @@ vi.mock("@/lib/integrations/status", () => ({ vi.mock("@/lib/rollups/measurement-rollups", () => ({ collapseToTypeDayKeys: (rows: Array<{ type: string; measuredAt: Date }>) => rows.map((r) => ({ type: r.type, measuredAt: r.measuredAt })), - recomputeBucketsForMeasurement: (...a: unknown[]) => - recomputeBucketsMock(...a), + recomputeBucketsForKeys: async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) + await recomputeBucketsMock(userId, k.type, k.measuredAt); + }, })); vi.mock("@/lib/insights/comprehensive-generate", () => ({ diff --git a/src/lib/whoop/sync-core.ts b/src/lib/whoop/sync-core.ts index bb6da4b42..4ba22baeb 100644 --- a/src/lib/whoop/sync-core.ts +++ b/src/lib/whoop/sync-core.ts @@ -14,7 +14,7 @@ * `(userId, type, source = WHOOP, externalId)` so a re-post (WHOOP * re-scores recovery/sleep after the fact) overwrites in place rather than * minting a duplicate. After the upserts the rollup tier is re-folded - * (`recomputeBucketsForMeasurement`) and the status-insight caches are + * (`recomputeBucketsForKeys`) and the status-insight caches are * invalidated, identical to the Withings tail. * * The incremental window starts from `lastSyncedAt - overlap`. WHOOP re-scores @@ -37,7 +37,7 @@ import { } from "@/lib/integrations/status"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate"; import { @@ -503,9 +503,7 @@ export async function upsertWhoopMeasurements( ); try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); } catch (err) { getEvent()?.addWarning( `whoop: rollup recompute failed for ${userId}: ${err}`, diff --git a/src/lib/withings/__tests__/sync-activity.test.ts b/src/lib/withings/__tests__/sync-activity.test.ts index c9dff410a..7155da2f8 100644 --- a/src/lib/withings/__tests__/sync-activity.test.ts +++ b/src/lib/withings/__tests__/sync-activity.test.ts @@ -69,9 +69,19 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { const actual = await vi.importActual< typeof import("@/lib/rollups/measurement-rollups") >("@/lib/rollups/measurement-rollups"); + const perKey = vi.fn().mockResolvedValue(undefined); return { ...actual, - recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForMeasurement: perKey, + // The keyed refold, observed as the per-key recomputes it stands for. + recomputeBucketsForKeys: vi.fn( + async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await perKey(userId, k.type, k.measuredAt); + }, + ), }; }); diff --git a/src/lib/withings/__tests__/sync-ecg.test.ts b/src/lib/withings/__tests__/sync-ecg.test.ts index dc8cae0b7..916677346 100644 --- a/src/lib/withings/__tests__/sync-ecg.test.ts +++ b/src/lib/withings/__tests__/sync-ecg.test.ts @@ -68,9 +68,19 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { const actual = await vi.importActual< typeof import("@/lib/rollups/measurement-rollups") >("@/lib/rollups/measurement-rollups"); + const perKey = vi.fn().mockResolvedValue(undefined); return { ...actual, - recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForMeasurement: perKey, + // The keyed refold, observed as the per-key recomputes it stands for. + recomputeBucketsForKeys: vi.fn( + async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await perKey(userId, k.type, k.measuredAt); + }, + ), }; }); diff --git a/src/lib/withings/__tests__/sync-sleep.test.ts b/src/lib/withings/__tests__/sync-sleep.test.ts index d5bb12404..ea945c843 100644 --- a/src/lib/withings/__tests__/sync-sleep.test.ts +++ b/src/lib/withings/__tests__/sync-sleep.test.ts @@ -67,9 +67,19 @@ vi.mock("@/lib/rollups/measurement-rollups", async () => { const actual = await vi.importActual< typeof import("@/lib/rollups/measurement-rollups") >("@/lib/rollups/measurement-rollups"); + const perKey = vi.fn().mockResolvedValue(undefined); return { ...actual, - recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + recomputeBucketsForMeasurement: perKey, + // The keyed refold, observed as the per-key recomputes it stands for. + recomputeBucketsForKeys: vi.fn( + async ( + userId: string, + keys: Array<{ type: string; measuredAt: Date }>, + ) => { + for (const k of keys) await perKey(userId, k.type, k.measuredAt); + }, + ), }; }); diff --git a/src/lib/withings/sync-activity.ts b/src/lib/withings/sync-activity.ts index 4bcdb7d50..1a075fec6 100644 --- a/src/lib/withings/sync-activity.ts +++ b/src/lib/withings/sync-activity.ts @@ -45,7 +45,7 @@ import { getUnitForType } from "@/lib/validations/measurement"; import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate"; @@ -540,9 +540,7 @@ export async function syncUserActivity( // catch-up costs at most ~N (type, day) recomputes. try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); // v1.8.0 — drop the per-metric assessment caches the synced types // dirty (steps feed the general overview). Fire-and-forget. diff --git a/src/lib/withings/sync-ecg.ts b/src/lib/withings/sync-ecg.ts index b4fe85fe7..865532dc5 100644 --- a/src/lib/withings/sync-ecg.ts +++ b/src/lib/withings/sync-ecg.ts @@ -56,7 +56,7 @@ import { safeFetch } from "@/lib/safe-fetch"; import { persistEcgRecording } from "@/lib/ecg/persist-recording"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate"; @@ -519,9 +519,7 @@ export async function syncUserEcg( // then drop the per-metric assessment caches the new events dirty. try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); invalidateStatusInsightsForTypes( userId, keys.map((k) => k.type), diff --git a/src/lib/withings/sync-sleep.ts b/src/lib/withings/sync-sleep.ts index 0611127ec..41dd7f5b1 100644 --- a/src/lib/withings/sync-sleep.ts +++ b/src/lib/withings/sync-sleep.ts @@ -45,7 +45,7 @@ import { safeFetch } from "@/lib/safe-fetch"; import { getUnitForType } from "@/lib/validations/measurement"; import { collapseToTypeDayKeys, - recomputeBucketsForMeasurement, + recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate"; import { maybeEnqueueMorningRefresh } from "@/lib/daily/morning-refresh-trigger"; @@ -628,9 +628,7 @@ export async function syncUserSleep( // ~10 stage rows per night land in one DAY recompute. try { const keys = collapseToTypeDayKeys(touched); - for (const k of keys) { - await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); - } + await recomputeBucketsForKeys(userId, keys); // v1.8.0 — drop the per-metric assessment caches the synced types // dirty (sleep + resting-HR feed the pulse / general cards). From 1cfae835789e49bb0da94d23b455a07d7e4bb5d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:22:19 +0200 Subject: [PATCH 07/23] perf(insights): fold the all-time extremes from the DAY rollup tier readAllTimeExtremes read every raw reading of weight, blood pressure and pulse twice on each call, a sequential pass over the measurements table that made it the largest read on the database. The DAY tier already holds the same per-source day fold except for the extremes of the readings themselves: min_value and max_value widen to a row's own intra-hour range. Migration 0417 adds reading_min_value and reading_max_value to measurement_rollups. The writer fills them from the same aggregate, and the boot re-fold refills existing rows through the arm that already watches sum_xy. The reader folds the tier for whole UTC days inside the fold window and the raw rows for the rest: older days, the partial day a lookback floor falls in, and any type whose tier rows lack the new columns. The raw arm reads narrow columns once instead of twice. An integration test compares the result with the previous query on a seeded history with two sources per day, hourly pulse statistics with their own range, readings older than the window, midnight edges and deletions, and shows the tier is the path that answers. --- .../migration.sql | 24 ++ prisma/schema.prisma | 46 ++- src/lib/insights/feature-blocks.ts | 177 +++++++--- .../__tests__/measurement-rollups.test.ts | 20 +- src/lib/rollups/measurement-rollups.ts | 28 +- .../all-time-extremes-tier-parity.test.ts | 321 ++++++++++++++++++ 6 files changed, 542 insertions(+), 74 deletions(-) create mode 100644 prisma/migrations/0417_measurement_rollup_reading_extremes/migration.sql create mode 100644 tests/integration/all-time-extremes-tier-parity.test.ts diff --git a/prisma/migrations/0417_measurement_rollup_reading_extremes/migration.sql b/prisma/migrations/0417_measurement_rollup_reading_extremes/migration.sql new file mode 100644 index 000000000..a42916f48 --- /dev/null +++ b/prisma/migrations/0417_measurement_rollup_reading_extremes/migration.sql @@ -0,0 +1,24 @@ +-- The lowest and highest reading value per rollup bucket. +-- +-- The full-history extremes the insight features carry (`allTimeMin`, +-- `allTimeMax`, `allTimeAvg` for weight, blood pressure and pulse) were read +-- from the raw `measurements` rows on every call: two passes over every +-- reading of those types the account ever recorded, the largest read on the +-- database. The DAY rollups already hold everything that fold needs except +-- the extremes of the readings themselves: `min_value` / `max_value` widen to +-- a row's own `value_min` / `value_max`, the intra-hour range an hourly +-- heart-rate statistic carries, so they are not the same figure. +-- +-- Two nullable columns. Existing rows read NULL until the boot re-fold +-- (`rollup-full-backfill`, discovered by the NULL arm beside `sum_xy`) +-- refills them; new writes fill them from the same aggregate as every other +-- column. A reader treats a type with a NULL in its window as uncovered and +-- reads the raw rows, as before. +-- +-- Reversibility: +-- ALTER TABLE "measurement_rollups" DROP COLUMN IF EXISTS "reading_min_value"; +-- ALTER TABLE "measurement_rollups" DROP COLUMN IF EXISTS "reading_max_value"; + +ALTER TABLE "measurement_rollups" ADD COLUMN IF NOT EXISTS "reading_min_value" DOUBLE PRECISION; + +ALTER TABLE "measurement_rollups" ADD COLUMN IF NOT EXISTS "reading_max_value" DOUBLE PRECISION; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c38d79ad9..f3337df8c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -2253,31 +2253,31 @@ enum RollupGranularity { } model MeasurementRollup { - userId String @map("user_id") - type MeasurementType - granularity RollupGranularity - bucketStart DateTime @map("bucket_start") @db.Timestamptz(3) + userId String @map("user_id") + type MeasurementType + granularity RollupGranularity + bucketStart DateTime @map("bucket_start") @db.Timestamptz(3) /// v1.11.1 — source-aware grain. Rollup rows are minted per source so the /// read path can collapse overlapping sources (e.g. WHOOP + Apple Watch /// resting heart rate) to the ladder-canonical reading, while cumulative /// metrics (steps, energy) sum per source. Always non-null; migration 0115 /// purged the legacy source-blind aggregates and the boot backfill re-mints /// per-source rows. - source MeasurementSource - count Int - mean Float - minValue Float @map("min_value") - maxValue Float @map("max_value") - sd Float? - slope Float? - r2 Float? + source MeasurementSource + count Int + mean Float + minValue Float @map("min_value") + maxValue Float @map("max_value") + sd Float? + slope Float? + r2 Float? /// v1.4.39 W-SUM — cumulative-metric sum for the bucket. Populated /// alongside `mean / count` for every type; cumulative read paths /// (steps, flights, distance, daylight, active-energy) consume this /// directly instead of re-deriving from raw measurements. Nullable /// to keep the migration additive — the boot backfill fills NULLs /// for existing rows; new rows always carry the sum. - sumValue Float? @map("sum_value") + sumValue Float? @map("sum_value") /// v1.20.0 F6 — per-bucket OLS regression accumulators over the same /// canonical-source raw rows the live REGR_* query reads. With /// `n = count` and `Σy = mean·count` these close the additive set so a @@ -2291,11 +2291,21 @@ model MeasurementRollup { /// parity-matches the cross-bucket compose. Nullable to keep migration 0190 /// additive — the boot backfill fills NULLs for pre-migration rows; new /// rows always carry them (they ride the same aggregate as mean / sum). - sumX Float? @map("sum_x") - sumXy Float? @map("sum_xy") - sumXx Float? @map("sum_xx") - sumYy Float? @map("sum_yy") - computedAt DateTime @default(now()) @map("computed_at") @db.Timestamptz(3) + sumX Float? @map("sum_x") + sumXy Float? @map("sum_xy") + sumXx Float? @map("sum_xx") + sumYy Float? @map("sum_yy") + /// The lowest and highest reading `value` of the bucket. `minValue` / + /// `maxValue` widen to a row's own range (`Measurement.valueMin` / + /// `valueMax`, the intra-hour range an hourly heart-rate statistic + /// carries); these do not, so a reader whose figure is over the readings + /// themselves (the full-history extremes in `feature-blocks.ts`) can fold + /// the DAY tier instead of the raw rows. Nullable to keep migration 0417 + /// additive: the boot re-fold fills existing rows, new rows always carry + /// them, and a reader treats a type with a NULL in its range as uncovered. + readingMinValue Float? @map("reading_min_value") + readingMaxValue Float? @map("reading_max_value") + computedAt DateTime @default(now()) @map("computed_at") @db.Timestamptz(3) user User @relation(fields: [userId], references: [id], onDelete: Cascade) diff --git a/src/lib/insights/feature-blocks.ts b/src/lib/insights/feature-blocks.ts index c9b75c195..70024c404 100644 --- a/src/lib/insights/feature-blocks.ts +++ b/src/lib/insights/feature-blocks.ts @@ -17,11 +17,17 @@ import { foldMeanSql, isHourlyMeanTypeSql, } from "@/lib/measurements/day-mean"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { + loadUserSourcePriority, + utcBucketEnd, +} from "@/lib/rollups/measurement-read"; import { classifyAgainstEffectiveRange } from "@/lib/labs/reference-range"; import { calendarDaysUntil } from "@/lib/measurement-reminders/due-day"; import { resolveLabFields } from "@/lib/labs/serialise"; -import { readRollupBuckets } from "@/lib/rollups/measurement-rollups"; +import { + readRollupBuckets, + ROLLUP_FOLD_WINDOW_MS, +} from "@/lib/rollups/measurement-rollups"; import { deriveBucketedTypes } from "@/lib/signals/adapters/correlation"; import type { AggregatedFeatures, @@ -40,11 +46,10 @@ interface AllTimeExtremes { } /** - * v1.18.11 — full-history min / max / mean per measurement type via ONE - * grouped SQL aggregation, with NO row materialisation in JS. Fills the - * `allTime*` feature fields; the windowed `summarize()` covers trends + recent - * windows, and this covers the long-horizon figures the prompt labels - * "allTime". + * v1.18.11 — full-history min / max / mean per measurement type, folded in + * SQL with NO row materialisation in JS. Fills the `allTime*` feature fields; + * the windowed `summarize()` covers trends + recent windows, and this covers + * the long-horizon figures the prompt labels "allTime". * * One source per (type, day): each source is folded per day first, each day * keeps its ladder-canonical source, then the days fold into the figures — @@ -52,6 +57,25 @@ interface AllTimeExtremes { * row let a device that samples densely (a watch's pulse) outweigh the * readings the source ladder prefers on the same days. * + * Where the days come from. A day is the same per-source fold either way — + * count, sum, the extremes of the readings and, for an hourly-mean type, the + * day value — and the DAY rollup tier already holds exactly that per (type, + * source, day), recomputed on every write. Reading the raw rows instead meant + * two passes over every reading of these types the account ever recorded on + * every call, the largest read on the database. So the figures fold: + * + * - the DAY tier, for every whole UTC day from `tierFrom` on, for each type + * whose tier rows in that range all carry the reading extremes + * (migration 0417; the boot re-fold fills older rows); + * - the raw rows for the rest: the days before `tierFrom` (older than the + * fold window, or the partial day the `floor` falls in), and every day of + * a type the tier does not cover yet. + * + * The split is at a UTC midnight, so no day is folded from both, and the + * canonical pick per day is the same on both sides. The tier's pulse day + * value follows the stored day-mean method; the raw fold follows this + * process's mirror of it, which trails a change by at most a minute. + * * Only the four types that expose `allTime*` fields are aggregated (weight, * systolic, diastolic, pulse). Returns a map keyed by `MeasurementType`; a type * with no rows is simply absent. @@ -67,15 +91,88 @@ export async function readAllTimeExtremes( ): Promise> { const out = new Map(); if (types.length === 0) return out; + // The first whole UTC day the tier may answer: after the fold window's + // edge (the fold writes nothing older, and the edge day may be partial) + // and after the floor's own day (only part of it is inside the floor). + const windowEdge = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); + const tierFrom = utcBucketEnd( + floor !== null && floor > windowEdge ? floor : windowEdge, + "DAY", + ); + const typeList = (list: readonly MeasurementType[]) => + Prisma.join(list.map((t) => Prisma.sql`${t}::measurement_type`)); + + const [priorityJson, tierRows] = await Promise.all([ + loadUserSourcePriority(userId), + prisma.$queryRaw>` + SELECT r."type"::text AS type + FROM measurement_rollups r + WHERE r."user_id" = ${userId} + AND r."granularity" = 'DAY' + AND r."type" IN (${typeList(types)}) + AND r."bucket_start" >= ${tierFrom} + GROUP BY r."type" + HAVING bool_and( + r."reading_min_value" IS NOT NULL AND r."reading_max_value" IS NOT NULL + ) + `, + ]); + const tierTypes = new Set(tierRows.map((r) => r.type)); + const fromTier = types.filter((t) => tierTypes.has(t)); + const rawOnly = types.filter((t) => !tierTypes.has(t)); + const since = floor === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${floor}`; - const priorityJson = await loadUserSourcePriority(userId); - const rank = Prisma.raw( - buildSourceRankCase(priorityJson, 'p."type"', 'p."source"'), - ); - const typeList = Prisma.join( - types.map((t) => Prisma.sql`${t}::measurement_type`), - ); + const rawArms: Prisma.Sql[] = []; + if (rawOnly.length > 0) { + rawArms.push(Prisma.sql` + SELECT m."type", m."source", m."measured_at", m."value" + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList(rawOnly)}) + ${since}`); + } + if (fromTier.length > 0) { + rawArms.push(Prisma.sql` + SELECT m."type", m."source", m."measured_at", m."value" + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList(fromTier)}) + ${since} + AND m."measured_at" < ${tierFrom}`); + } + const rank = (alias: string) => + Prisma.raw( + buildSourceRankCase(priorityJson, `${alias}."type"`, `${alias}."source"`), + ); + // The tier's canonical days, in the shape of the raw fold's: the day's + // reading-weighted sum and count, its reading extremes, and its day value + // with a weight of one (a DAY row's `mean` is the day value for an + // hourly-mean type). + const tierCanon = + fromTier.length === 0 + ? Prisma.empty + : Prisma.sql` + UNION ALL + SELECT * FROM ( + SELECT DISTINCT ON (r."type", r."bucket_start") + r."type", + r."count" AS cnt, + COALESCE(r."sum_value", r."count" * r."mean")::double precision + AS total, + r."reading_min_value" AS min_value, + r."reading_max_value" AS max_value, + r."mean" AS wsum, + 1.0::double precision AS wdays + FROM measurement_rollups r + WHERE r."user_id" = ${userId} + AND r."granularity" = 'DAY' + AND r."type" IN (${typeList(fromTier)}) + AND r."bucket_start" >= ${tierFrom} + ORDER BY r."type", r."bucket_start", (${rank("r")}), r."source"::text + ) tier`; // A day of pulse is the mean of its hours' means and the full history the // mean of its days (`day-mean.ts`). `weighted` carries, per source and day, // the day-weighted sum and the day's weight (one) for those types only; @@ -92,14 +189,10 @@ export async function readAllTimeExtremes( const rows = await prisma.$queryRaw< Array<{ type: string; mean: number; min: number; max: number }> >` - WITH hourly_src AS ( - SELECT m.* - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."deleted_at" IS NULL - AND m."type" IN (${typeList}) - ${since} - AND ${Prisma.raw(isHourlyMeanTypeSql('m."type"'))} + WITH live AS (${Prisma.join(rawArms, " UNION ALL ")}), + hourly_src AS ( + SELECT * FROM live l + WHERE ${Prisma.raw(isHourlyMeanTypeSql('l."type"'))} ), weighted AS ( SELECT @@ -113,34 +206,32 @@ export async function readAllTimeExtremes( ), per_source AS ( SELECT - m."type", - m."source", - date_trunc('day', m."measured_at") AS day, + l."type", + l."source", + date_trunc('day', l."measured_at") AS day, COUNT(*)::int AS cnt, - SUM(m."value")::double precision AS total, - MIN(m."value")::double precision AS min_value, - MAX(m."value")::double precision AS max_value - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."deleted_at" IS NULL - AND m."type" IN (${typeList}) - ${since} - GROUP BY m."type", m."source", 3 - ), - canon AS ( - SELECT DISTINCT ON (p."type", p.day) - p."type", p.cnt, p.total, p.min_value, p.max_value, w.wsum, w.wdays - FROM per_source p - LEFT JOIN weighted w - ON w."type" = p."type" AND w."source" = p."source" AND w.day = p.day - ORDER BY p."type", p.day, (${rank}), p."source"::text + SUM(l."value")::double precision AS total, + MIN(l."value")::double precision AS min_value, + MAX(l."value")::double precision AS max_value + FROM live l + GROUP BY l."type", l."source", 3 ) SELECT c."type"::text AS type, ${mean}::double precision AS mean, MIN(c.min_value)::double precision AS min, MAX(c.max_value)::double precision AS max - FROM canon c + FROM ( + SELECT * FROM ( + SELECT DISTINCT ON (p."type", p.day) + p."type", p.cnt, p.total, p.min_value, p.max_value, w.wsum, w.wdays + FROM per_source p + LEFT JOIN weighted w + ON w."type" = p."type" AND w."source" = p."source" AND w.day = p.day + ORDER BY p."type", p.day, (${rank("p")}), p."source"::text + ) raw + ${tierCanon} + ) c GROUP BY c."type" `; for (const r of rows) { diff --git a/src/lib/rollups/__tests__/measurement-rollups.test.ts b/src/lib/rollups/__tests__/measurement-rollups.test.ts index 3426575fa..69161ada6 100644 --- a/src/lib/rollups/__tests__/measurement-rollups.test.ts +++ b/src/lib/rollups/__tests__/measurement-rollups.test.ts @@ -188,6 +188,8 @@ describe("recomputeBucketsForMeasurement", () => { sd: 2.0, slope: -0.1, r2: 0.9, + reading_min_value: 80.5, + reading_max_value: 84.5, }, ]); getGlobalBossMock.mockReturnValue({ send: bossSend }); @@ -210,14 +212,14 @@ describe("recomputeBucketsForMeasurement", () => { // instead of raising (or being dropped). Param layout per row: // [userId, type, granularity, bucketStart, source, count, mean, // min, max, sum, sd, slope, r2, sumX, sumXy, sumXx, sumYy, - // computedAt]. + // readingMin, readingMax, computedAt]. expect(txExecuteRawUnsafe).toHaveBeenCalledTimes(1); const [sql, ...params] = txExecuteRawUnsafe.mock.calls[0]; expect(sql).toContain( "ON CONFLICT (user_id, type, granularity, bucket_start, source)", ); expect(sql).toContain("DO UPDATE SET"); - expect(params).toHaveLength(18); + expect(params).toHaveLength(20); expect(params[0]).toBe("user-1"); expect(params[1]).toBe("WEIGHT"); expect(params[2]).toBe("DAY"); @@ -227,6 +229,10 @@ describe("recomputeBucketsForMeasurement", () => { // v1.4.39 — sumValue flows through. Cumulative read paths // (steps, flights, distance, daylight, active-energy) consume it. expect(params[9]).toBe(247.5); + // The reading extremes, beside the range-widened min / max. + expect(params[7]).toBe(80.0); + expect(params[17]).toBe(80.5); + expect(params[18]).toBe(84.5); // WEEK / MONTH / YEAR — three enqueues against the worker queue. expect(bossSend).toHaveBeenCalledTimes(3); @@ -453,13 +459,13 @@ describe("persistRollupRows — large path (via recomputeUserRollups)", () => { expect(deleteMany.mock.calls[0][0].where.userId).toBe("user-1"); expect(deleteMany.mock.calls[0][0].where.granularity).toBe("DAY"); - // 3 chunks of 18 params per row, every one an ON CONFLICT DO UPDATE + // 3 chunks of 20 params per row, every one an ON CONFLICT DO UPDATE // upsert — tolerant of a mid-transaction hook insert AND of a // concurrent fold on the same partition (v1.28.33, issue #486). expect(txExecuteRawUnsafe).toHaveBeenCalledTimes(3); - expect(txExecuteRawUnsafe.mock.calls[0]).toHaveLength(1 + 500 * 18); - expect(txExecuteRawUnsafe.mock.calls[1]).toHaveLength(1 + 500 * 18); - expect(txExecuteRawUnsafe.mock.calls[2]).toHaveLength(1 + 200 * 18); + expect(txExecuteRawUnsafe.mock.calls[0]).toHaveLength(1 + 500 * 20); + expect(txExecuteRawUnsafe.mock.calls[1]).toHaveLength(1 + 500 * 20); + expect(txExecuteRawUnsafe.mock.calls[2]).toHaveLength(1 + 200 * 20); for (const call of txExecuteRawUnsafe.mock.calls) { expect(call[0]).toContain("ON CONFLICT"); expect(call[0]).toContain("DO UPDATE SET"); @@ -575,7 +581,7 @@ describe("persistRollupRows — large path (via recomputeUserRollups)", () => { expect(transaction).toHaveBeenCalledTimes(1); expect(typeof transaction.mock.calls[0][0]).toBe("function"); expect(txExecuteRawUnsafe).toHaveBeenCalledTimes(1); - expect(txExecuteRawUnsafe.mock.calls[0]).toHaveLength(1 + 2 * 18); + expect(txExecuteRawUnsafe.mock.calls[0]).toHaveLength(1 + 2 * 20); expect(result.rowsUpserted).toBe(2); }); }); diff --git a/src/lib/rollups/measurement-rollups.ts b/src/lib/rollups/measurement-rollups.ts index cf33a2c6d..c3cd5f397 100644 --- a/src/lib/rollups/measurement-rollups.ts +++ b/src/lib/rollups/measurement-rollups.ts @@ -184,6 +184,10 @@ interface RollupRow { sum_xy: number | null; sum_xx: number | null; sum_yy: number | null; + // The extremes of the readings' own `value`, without the per-row range + // `min_value` / `max_value` widen to. See `MeasurementRollup.readingMinValue`. + reading_min_value: number | null; + reading_max_value: number | null; } /** @@ -546,7 +550,9 @@ async function runRollupAggregate( SUM(${REBASED_X})::double precision AS sum_x, SUM(${REBASED_X} * m."value")::double precision AS sum_xy, SUM(${REBASED_X} * ${REBASED_X})::double precision AS sum_xx, - SUM(m."value" * m."value")::double precision AS sum_yy + SUM(m."value" * m."value")::double precision AS sum_yy, + MIN(m."value")::double precision AS reading_min_value, + MAX(m."value")::double precision AS reading_max_value FROM measurements m WHERE m."user_id" = $1 AND m."type" IN (${typeList}) @@ -594,7 +600,9 @@ async function runRollupAggregate( SUM(${REBASED_X})::double precision AS sum_x, SUM(${REBASED_X} * m."value")::double precision AS sum_xy, SUM(${REBASED_X} * ${REBASED_X})::double precision AS sum_xx, - SUM(m."value" * m."value")::double precision AS sum_yy + SUM(m."value" * m."value")::double precision AS sum_yy, + MIN(m."value")::double precision AS reading_min_value, + MAX(m."value")::double precision AS reading_max_value FROM measurements m WHERE m."user_id" = $1 AND m."measured_at" >= $2 @@ -921,20 +929,24 @@ function buildRollupUpsert( row.sum_xy ?? null, row.sum_xx ?? null, row.sum_yy ?? null, + row.reading_min_value ?? null, + row.reading_max_value ?? null, computedAt, ); tuples.push( `($${base + 1}, $${base + 2}::"measurement_type", $${base + 3}::"measurement_rollup_granularity", ` + `$${base + 4}::timestamptz, $${base + 5}::"measurement_source", $${base + 6}::int, ` + `$${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, ` + - `$${base + 13}, $${base + 14}, $${base + 15}, $${base + 16}, $${base + 17}, $${base + 18}::timestamptz)`, + `$${base + 13}, $${base + 14}, $${base + 15}, $${base + 16}, $${base + 17}, $${base + 18}, ` + + `$${base + 19}, $${base + 20}::timestamptz)`, ); } const sql = ` INSERT INTO measurement_rollups (user_id, type, granularity, bucket_start, source, count, mean, min_value, max_value, sum_value, sd, slope, r2, - sum_x, sum_xy, sum_xx, sum_yy, computed_at) + sum_x, sum_xy, sum_xx, sum_yy, reading_min_value, reading_max_value, + computed_at) VALUES ${tuples.join(", ")} ON CONFLICT (user_id, type, granularity, bucket_start, source) DO UPDATE SET @@ -950,6 +962,8 @@ function buildRollupUpsert( sum_xy = EXCLUDED.sum_xy, sum_xx = EXCLUDED.sum_xx, sum_yy = EXCLUDED.sum_yy, + reading_min_value = EXCLUDED.reading_min_value, + reading_max_value = EXCLUDED.reading_max_value, computed_at = EXCLUDED.computed_at `; return { sql, params }; @@ -1201,7 +1215,9 @@ export async function enqueueBootTimeRollupBackfill( // catches up — mirrors the v1.4.39 sum_value rollout, which the // v1.4.41 note confirms converged via this same boot queue. Bounded // to the fold window so days outside it (un-foldable by design) - // never re-flag a user on every boot. + // never re-flag a user on every boot. The same arm refills the + // reading extremes of migration 0417 (`reading_min_value IS NULL`); + // the extremes reader treats a NULL as uncovered until it lands. const users = await prisma.$queryRaw>` SELECT DISTINCT id FROM ( SELECT DISTINCT mt."user_id" AS id @@ -1227,7 +1243,7 @@ export async function enqueueBootTimeRollupBackfill( FROM measurement_rollups r WHERE r."granularity" = 'DAY' AND r."bucket_start" >= ${foldWindowStart} - AND r."sum_xy" IS NULL + AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL) ) discovered `; diff --git a/tests/integration/all-time-extremes-tier-parity.test.ts b/tests/integration/all-time-extremes-tier-parity.test.ts new file mode 100644 index 000000000..74c16102a --- /dev/null +++ b/tests/integration/all-time-extremes-tier-parity.test.ts @@ -0,0 +1,321 @@ +/** + * The full-history extremes read the DAY rollup tier and give the figures the + * raw-row fold gave. + * + * `readAllTimeExtremes` used to fold every raw reading of the four types on + * each call. It now folds the DAY tier for the whole days inside the fold + * window and the raw rows for the rest. The reference below is that earlier + * query, verbatim, so the comparison is against what the function used to + * answer, not against a second derivation of the same idea. + * + * The seed has what the two paths could disagree on: two sources on the same + * days (weight by scale and by hand, pulse from a watch's hourly statistics + * with their own range and a strap's dense samples), readings older than the + * fold window, readings either side of a UTC midnight, soft-deleted rows, and + * a reading deleted after the tier was built (through the write path's own + * rollup recompute). + * + * Means are compared to nine significant digits: the tier sums the same day + * values in a different order. Extremes are compared exactly. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { Prisma, type MeasurementType } from "@/generated/prisma/client"; +import { readAllTimeExtremes } from "@/lib/insights/feature-blocks"; +import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; +import { + dayWeightedRowsSql, + foldMeanSql, + isHourlyMeanTypeSql, +} from "@/lib/measurements/day-mean"; +import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; +import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; + +const prisma = getPrismaClient(); +const USER = "all-time-tier-parity"; +const TYPES: MeasurementType[] = [ + "WEIGHT", + "BLOOD_PRESSURE_SYS", + "BLOOD_PRESSURE_DIA", + "PULSE", +]; +const HOUR = 3_600_000; +const DAY = 24 * HOUR; +const today = Date.UTC( + new Date().getUTCFullYear(), + new Date().getUTCMonth(), + new Date().getUTCDate(), +); + +/** The query `readAllTimeExtremes` ran before it read the DAY tier. */ +async function referenceExtremes( + userId: string, + types: readonly MeasurementType[], + floor: Date | null, +) { + const out = new Map(); + const since = + floor === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${floor}`; + const priorityJson = await loadUserSourcePriority(userId); + const rank = Prisma.raw( + buildSourceRankCase(priorityJson, 'p."type"', 'p."source"'), + ); + const typeList = Prisma.join( + types.map((t) => Prisma.sql`${t}::measurement_type`), + ); + const mean = Prisma.raw( + foldMeanSql({ + typeColumn: 'c."type"', + total: "c.total", + count: "c.cnt", + weightedSum: "c.wsum", + weightSum: "c.wdays", + }), + ); + const rows = await prisma.$queryRaw< + Array<{ type: string; mean: number; min: number; max: number }> + >` + WITH hourly_src AS ( + SELECT m.* + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList}) + ${since} + AND ${Prisma.raw(isHourlyMeanTypeSql('m."type"'))} + ), + weighted AS ( + SELECT + w."type", + w."source", + date_trunc('day', w."measured_at") AS day, + SUM(w."value" * w.day_weight)::double precision AS wsum, + SUM(w.day_weight)::double precision AS wdays + FROM ${dayWeightedRowsSql("hourly_src", null, { bySource: true })} w + GROUP BY w."type", w."source", 3 + ), + per_source AS ( + SELECT + m."type", + m."source", + date_trunc('day', m."measured_at") AS day, + COUNT(*)::int AS cnt, + SUM(m."value")::double precision AS total, + MIN(m."value")::double precision AS min_value, + MAX(m."value")::double precision AS max_value + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList}) + ${since} + GROUP BY m."type", m."source", 3 + ), + canon AS ( + SELECT DISTINCT ON (p."type", p.day) + p."type", p.cnt, p.total, p.min_value, p.max_value, w.wsum, w.wdays + FROM per_source p + LEFT JOIN weighted w + ON w."type" = p."type" AND w."source" = p."source" AND w.day = p.day + ORDER BY p."type", p.day, (${rank}), p."source"::text + ) + SELECT + c."type"::text AS type, + ${mean}::double precision AS mean, + MIN(c.min_value)::double precision AS min, + MAX(c.max_value)::double precision AS max + FROM canon c + GROUP BY c."type" + `; + for (const r of rows) { + out.set(r.type, { mean: r.mean, min: r.min, max: r.max }); + } + return out; +} + +/** A small deterministic generator, so a failure reproduces. */ +function lcg(seed: number): () => number { + let s = seed; + return () => { + s = (s * 1_103_515_245 + 12_345) % 2_147_483_648; + return s / 2_147_483_648; + }; +} + +function seed(): Prisma.MeasurementCreateManyInput[] { + const rnd = lcg(7); + const rows: Prisma.MeasurementCreateManyInput[] = []; + let n = 0; + const add = ( + type: MeasurementType, + unit: string, + source: Prisma.MeasurementCreateManyInput["source"], + at: number, + value: number, + extra: Partial = {}, + ) => { + n += 1; + rows.push({ + id: `${USER}-${n}`, + userId: USER, + type, + unit, + source, + value, + measuredAt: new Date(at), + ...extra, + }); + }; + // Weight and blood pressure across six years: older than the fold window + // too. Every fifth day a hand entry beside the scale's, every 23rd day a + // reading a minute either side of UTC midnight, and a few tombstones. + for (let d = 6 * 365; d >= 0; d -= 1) { + const day = today - d * DAY; + add("WEIGHT", "kg", "WITHINGS", day + 6 * HOUR, 70 + 10 * rnd()); + if (d % 5 === 0) + add("WEIGHT", "kg", "MANUAL", day + 20 * HOUR, 69 + 12 * rnd()); + for (const [type, base] of [ + ["BLOOD_PRESSURE_SYS", 110], + ["BLOOD_PRESSURE_DIA", 65], + ] as const) { + add(type, "mmHg", "WITHINGS", day + 7 * HOUR, base + 40 * rnd()); + if (d % 23 === 0) { + add(type, "mmHg", "WITHINGS", day - 60_000, base + 45 * rnd()); + add(type, "mmHg", "WITHINGS", day + 60_000, base - 5 + 50 * rnd()); + } + if (d % 31 === 0) { + add(type, "mmHg", "WITHINGS", day + 9 * HOUR, base + 80, { + deletedAt: new Date(), + }); + } + } + } + // Pulse over the last 400 days: a watch's hourly statistics carrying the + // hour's own min / max, and a strap's dense samples on every other day. + for (let d = 400; d >= 0; d -= 1) { + const day = today - d * DAY; + for (let h = 0; h < 24; h += 3) { + const avg = 55 + 30 * rnd(); + add("PULSE", "bpm", "APPLE_HEALTH", day + h * HOUR, avg, { + valueMin: avg - 15, + valueMax: avg + 40, + externalId: `stats:HKQuantityTypeIdentifierHeartRate:${day + h * HOUR}`, + }); + } + if (d % 2 === 0) { + for (let i = 0; i < 30; i += 1) { + add( + "PULSE", + "bpm", + "WHOOP", + day + 10 * HOUR + i * 120_000, + 50 + 70 * rnd(), + ); + } + } + } + return rows; +} + +async function expectParity(floor: Date | null) { + const [actual, reference] = await Promise.all([ + readAllTimeExtremes(USER, TYPES, floor), + referenceExtremes(USER, TYPES, floor), + ]); + expect([...actual.keys()].sort()).toEqual([...reference.keys()].sort()); + expect(reference.size).toBe(4); + for (const [type, ref] of reference) { + const got = actual.get(type as MeasurementType)!; + expect(got.min, type).toBe(ref.min); + expect(got.max, type).toBe(ref.max); + expect(got.mean! / ref.mean, type).toBeCloseTo(1, 9); + } +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: "Pacific/Auckland" }, + }); + const rows = seed(); + for (let i = 0; i < rows.length; i += 5000) { + await prisma.measurement.createMany({ data: rows.slice(i, i + 5000) }); + } + await recomputeUserRollups(USER, { granularities: ["DAY"] }); +}, 300_000); + +describe("readAllTimeExtremes over the DAY tier", () => { + it("reads the tier for every type", async () => { + const covered = await prisma.measurementRollup.groupBy({ + by: ["type"], + where: { + userId: USER, + granularity: "DAY", + readingMinValue: { not: null }, + }, + }); + expect(covered.map((c) => c.type).sort()).toEqual([...TYPES].sort()); + }); + + it("gives the raw fold's figures over the whole history", async () => { + await expectParity(null); + }); + + it("gives the raw fold's figures from a floor inside a day", async () => { + await expectParity(new Date(today - 200 * DAY + 13 * HOUR + 17 * 60_000)); + }); + + it("follows a deletion through the write path's recompute", async () => { + const hottest = await prisma.measurement.findFirst({ + where: { userId: USER, type: "PULSE", source: "WHOOP", deletedAt: null }, + orderBy: { value: "desc" }, + }); + await prisma.measurement.update({ + where: { id: hottest!.id }, + data: { deletedAt: new Date() }, + }); + await afterMeasurementMutation( + USER, + [{ type: "PULSE", measuredAt: hottest!.measuredAt }], + { source: "manual" }, + ); + await expectParity(null); + }); + + it("is the tier that answers, not the raw rows", async () => { + // Write it, then break it: a tier row raised past every reading must + // move the answer, or the parity above proved nothing about the tier. + const day = new Date(today - 3 * DAY); + await prisma.measurementRollup.updateMany({ + where: { + userId: USER, + type: "WEIGHT", + granularity: "DAY", + bucketStart: day, + }, + data: { readingMaxValue: 999 }, + }); + const out = await readAllTimeExtremes(USER, ["WEIGHT"]); + expect(out.get("WEIGHT")?.max).toBe(999); + }); + + it("reads the raw rows for a type with a tier row lacking the extremes", async () => { + const day = new Date(today - 3 * DAY); + await prisma.measurementRollup.updateMany({ + where: { + userId: USER, + type: "WEIGHT", + granularity: "DAY", + bucketStart: day, + }, + data: { readingMinValue: null }, + }); + const [actual, reference] = await Promise.all([ + readAllTimeExtremes(USER, ["WEIGHT"]), + referenceExtremes(USER, ["WEIGHT"], null), + ]); + expect(actual.get("WEIGHT")?.max).toBe(reference.get("WEIGHT")?.max); + expect(actual.get("WEIGHT")?.min).toBe(reference.get("WEIGHT")?.min); + }); +}); From c6c51be280e5de8809f59613de63642fb2273749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:26:30 +0200 Subject: [PATCH 08/23] perf(jobs): stop the geo backfill from retrying the same rows hourly An audit row whose address no database can place stayed without a location, so every hourly pass looked it up again for its whole 30-day window, and finding it walked every audit row of those 30 days through the created_at index. Migration 0418 adds audit_logs.geo_attempted_at and a partial index on the rows still missing a location or carrier. Every row a pass looks at is stamped, and a row tried less than seven days ago is not picked again, which still lets a range a later GeoLite2 release learns resolve. The candidate read goes through the partial index, which holds only the backlog. --- .../0418_audit_log_geo_attempt/migration.sql | 30 +++++ prisma/schema.prisma | 22 ++-- src/lib/jobs/__tests__/geo-backfill.test.ts | 45 ++++++-- src/lib/jobs/geo-backfill.ts | 44 +++++-- .../integration/geo-backfill-attempt.test.ts | 107 ++++++++++++++++++ 5 files changed, 224 insertions(+), 24 deletions(-) create mode 100644 prisma/migrations/0418_audit_log_geo_attempt/migration.sql create mode 100644 tests/integration/geo-backfill-attempt.test.ts diff --git a/prisma/migrations/0418_audit_log_geo_attempt/migration.sql b/prisma/migrations/0418_audit_log_geo_attempt/migration.sql new file mode 100644 index 000000000..cd5cd5d84 --- /dev/null +++ b/prisma/migrations/0418_audit_log_geo_attempt/migration.sql @@ -0,0 +1,30 @@ +-- The geo backfill notes when it last tried a row, and finds its candidates +-- through an index that holds only them. +-- +-- The hourly backfill (`src/lib/jobs/geo-backfill.ts`) re-resolves audit rows +-- whose location or carrier is missing. An address no database will ever +-- place (a private range that reached the log through a proxy) stays missing, +-- so the same few rows were looked up again on every pass, and finding them +-- meant walking thirty days of `audit_logs` through `created_at`, filtering +-- every row that had long been resolved. +-- +-- `geo_attempted_at` records the try; the backfill skips a row it tried less +-- than `GEO_BACKFILL_RETRY_DAYS` ago. NULL for every existing row: each is +-- tried once more, then waits like any other. +-- +-- `audit_logs_geo_backfill_idx` is partial on the rows still missing +-- something, so it holds the backlog and nothing else, and shrinks as rows +-- resolve. Prisma cannot express the predicate, so the index lives here only +-- (noted at the model in `schema.prisma`). A plain build: it takes SHARE on +-- `audit_logs` while it reads the table once, which `migrate deploy` does at +-- boot before the app serves. +-- +-- Reversibility: +-- DROP INDEX IF EXISTS "audit_logs_geo_backfill_idx"; +-- ALTER TABLE "audit_logs" DROP COLUMN IF EXISTS "geo_attempted_at"; + +ALTER TABLE "audit_logs" ADD COLUMN IF NOT EXISTS "geo_attempted_at" TIMESTAMP(3); + +CREATE INDEX IF NOT EXISTS "audit_logs_geo_backfill_idx" + ON "audit_logs" ("created_at") + WHERE "ip_address" IS NOT NULL AND ("location" IS NULL OR "carrier" IS NULL); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index f3337df8c..c802f3fd4 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5415,18 +5415,26 @@ model AuditLog { /// lives. actorUserId String? @map("actor_user_id") - action String // e.g. "auth.login", "measurement.create", "export.csv" - details String? // JSON string, no sensitive data - ipAddress String? @map("ip_address") - location String? // e.g. "Berlin, DE" — resolved from IP at log time + action String // e.g. "auth.login", "measurement.create", "export.csv" + details String? // JSON string, no sensitive data + ipAddress String? @map("ip_address") + location String? // e.g. "Berlin, DE" — resolved from IP at log time // v1.4.27 B3 — resolved at audit-creation time from the bundled // GeoLite2-ASN MMDB. `asn` is the autonomous-system number, // `carrier` is the organisation string (e.g. "Deutsche Telekom AG"). // Both nullable so older rows, private-IP rows, and offline-miss // rows stay valid. - asn Int? // autonomous-system number - carrier String? // autonomous-system organisation - createdAt DateTime @default(now()) @map("created_at") + asn Int? // autonomous-system number + carrier String? // autonomous-system organisation + /// When the geo backfill (`src/lib/jobs/geo-backfill.ts`) last tried this + /// row. A row it could not complete waits `GEO_BACKFILL_RETRY_DAYS` before + /// the next try, so an address no database will ever place (a private + /// range behind a proxy) is not resolved again on every hourly pass. NULL: + /// never tried. The backfill reads its candidates through the partial index + /// `audit_logs_geo_backfill_idx` (migration 0418, raw SQL only: Prisma + /// cannot express its predicate). + geoAttemptedAt DateTime? @map("geo_attempted_at") + createdAt DateTime @default(now()) @map("created_at") user User? @relation(fields: [userId], references: [id], onDelete: SetNull) diff --git a/src/lib/jobs/__tests__/geo-backfill.test.ts b/src/lib/jobs/__tests__/geo-backfill.test.ts index 0bd4bdec3..8a6b4a835 100644 --- a/src/lib/jobs/__tests__/geo-backfill.test.ts +++ b/src/lib/jobs/__tests__/geo-backfill.test.ts @@ -9,6 +9,7 @@ import { GEO_BACKFILL_BATCH_CAP, GEO_BACKFILL_CRON, GEO_BACKFILL_QUEUE, + GEO_BACKFILL_RETRY_DAYS, GEO_BACKFILL_WINDOW_DAYS, runGeoBackfill, } from "../geo-backfill"; @@ -65,6 +66,21 @@ describe("runGeoBackfill", () => { createdAt: { gt: new Date(now.getTime() - GEO_BACKFILL_WINDOW_DAYS * 86_400_000), }, + // A row tried less than the retry wait ago is not picked again. + AND: [ + { + OR: [ + { geoAttemptedAt: null }, + { + geoAttemptedAt: { + lt: new Date( + now.getTime() - GEO_BACKFILL_RETRY_DAYS * 86_400_000, + ), + }, + }, + ], + }, + ], }); expect(args?.take).toBe(GEO_BACKFILL_BATCH_CAP); }); @@ -91,7 +107,7 @@ describe("runGeoBackfill", () => { }); expect(prisma.auditLog.update).toHaveBeenCalledWith({ where: { id: "a1" }, - data: { location: "Berlin, DE" }, + data: { location: "Berlin, DE", geoAttemptedAt: expect.any(Date) }, }); }); @@ -117,6 +133,7 @@ describe("runGeoBackfill", () => { location: "München, DE", asn: 3320, carrier: "Deutsche Telekom AG", + geoAttemptedAt: expect.any(Date), }, }); }); @@ -136,7 +153,11 @@ describe("runGeoBackfill", () => { expect(summary.carrierResolved).toBe(1); expect(prisma.auditLog.update).toHaveBeenCalledWith({ where: { id: "a2b" }, - data: { location: "Bochum, DE", carrier: "Deutsche Telekom AG" }, + data: { + location: "Bochum, DE", + carrier: "Deutsche Telekom AG", + geoAttemptedAt: expect.any(Date), + }, }); }); @@ -158,15 +179,20 @@ describe("runGeoBackfill", () => { }); expect(prisma.auditLog.update).toHaveBeenCalledWith({ where: { id: "a3" }, - data: { carrier: "Vodafone GmbH", asn: 3209 }, + data: { + carrier: "Vodafone GmbH", + asn: 3209, + geoAttemptedAt: expect.any(Date), + }, }); }); - it("skips the update when the resolver misses entirely, counts as still-unresolved", async () => { + it("stamps only the attempt when the resolver misses entirely, counts as still-unresolved", async () => { const prisma = makePrismaMock([{ id: "a4", ipAddress: "192.0.2.1" }]); vi.mocked(lookupIpGeo).mockResolvedValueOnce(EMPTY_GEO); + const now = new Date("2026-05-15T12:00:00Z"); - const summary = await runGeoBackfill(prisma); + const summary = await runGeoBackfill(prisma, now); expect(summary).toEqual({ scanned: 1, @@ -174,7 +200,12 @@ describe("runGeoBackfill", () => { carrierResolved: 0, stillUnresolved: 1, }); - expect(prisma.auditLog.update).not.toHaveBeenCalled(); + // The try is recorded, so the next passes leave the row alone until + // the retry wait has passed. + expect(prisma.auditLog.update).toHaveBeenCalledWith({ + where: { id: "a4" }, + data: { geoAttemptedAt: now }, + }); }); it("handles a mixed batch of hits and misses", async () => { @@ -204,7 +235,7 @@ describe("runGeoBackfill", () => { carrierResolved: 2, stillUnresolved: 1, }); - expect(prisma.auditLog.update).toHaveBeenCalledTimes(2); + expect(prisma.auditLog.update).toHaveBeenCalledTimes(3); }); it("treats a Prisma update throw as still-unresolved without aborting the pass", async () => { diff --git a/src/lib/jobs/geo-backfill.ts b/src/lib/jobs/geo-backfill.ts index f0e7bd12d..919cb8a01 100644 --- a/src/lib/jobs/geo-backfill.ts +++ b/src/lib/jobs/geo-backfill.ts @@ -34,9 +34,16 @@ * * Idempotent: a row that the resolver still cannot match (private IP * sneaking through CF egress, freshly-allocated range the GeoLite2 - * release hasn't picked up yet) stays NULL and the next pass tries - * again. The cap is per-call, not per-row, so the helper can be - * scheduled at any cadence without doubling up. + * release hasn't picked up yet) stays NULL. Every row a pass looks at is + * stamped `geoAttemptedAt`, and a row stamped less than + * `GEO_BACKFILL_RETRY_DAYS` ago is not picked again: an address no + * database will ever place used to be looked up on every hourly pass for + * its whole 30-day window, and finding it walked every audit row of those + * 30 days. The wait still lets a range a later GeoLite2 release learns + * resolve on a later pass. The candidates are read through the partial + * index `audit_logs_geo_backfill_idx` (migration 0418), which holds only + * the rows still missing something. The cap is per-call, not per-row, so + * the helper can be scheduled at any cadence without doubling up. * * No notifications, no event ladder — the maintainer can read the * counts off the returned summary and decide whether to keep @@ -47,6 +54,8 @@ import { lookupIpGeo } from "@/lib/geo"; export const GEO_BACKFILL_BATCH_CAP = 500; export const GEO_BACKFILL_WINDOW_DAYS = 30; +/** How long a row the backfill could not complete waits for its next try. */ +export const GEO_BACKFILL_RETRY_DAYS = 7; /** * v1.4.37 — pg-boss queue name + cron expression for the recurring @@ -74,6 +83,9 @@ export async function runGeoBackfill( const cutoff = new Date( now.getTime() - GEO_BACKFILL_WINDOW_DAYS * 86_400_000, ); + const retryBefore = new Date( + now.getTime() - GEO_BACKFILL_RETRY_DAYS * 86_400_000, + ); const rows = await prisma.auditLog.findMany({ where: { @@ -85,6 +97,14 @@ export async function runGeoBackfill( OR: [{ location: null }, { carrier: null }], ipAddress: { not: null }, createdAt: { gt: cutoff }, + AND: [ + { + OR: [ + { geoAttemptedAt: null }, + { geoAttemptedAt: { lt: retryBefore } }, + ], + }, + ], }, select: { id: true, ipAddress: true }, orderBy: { createdAt: "desc" }, @@ -109,8 +129,14 @@ export async function runGeoBackfill( // ASN MMDB. The online fallback bounds itself with a request timeout. const { location, asn, carrier } = await lookupIpGeo(ip); - const data: { location?: string; asn?: number; carrier?: string | null } = - {}; + // Stamped whatever the outcome: a row that resolved leaves the candidate + // set anyway, and one that did not waits for its next try. + const data: { + location?: string; + asn?: number; + carrier?: string | null; + geoAttemptedAt: Date; + } = { geoAttemptedAt: now }; if (location) { data.location = location; summary.located += 1; @@ -121,10 +147,8 @@ export async function runGeoBackfill( summary.carrierResolved += 1; } - if (Object.keys(data).length === 0) { - summary.stillUnresolved += 1; - continue; - } + const resolved = data.location !== undefined || data.carrier !== undefined; + if (!resolved) summary.stillUnresolved += 1; try { await prisma.auditLog.update({ @@ -135,7 +159,7 @@ export async function runGeoBackfill( // The row may have been deleted by the retention sweeper between // findMany and update. Treat as unresolved; the next pass will // skip it because the row no longer exists. - summary.stillUnresolved += 1; + if (resolved) summary.stillUnresolved += 1; } } diff --git a/tests/integration/geo-backfill-attempt.test.ts b/tests/integration/geo-backfill-attempt.test.ts new file mode 100644 index 000000000..8c71bf735 --- /dev/null +++ b/tests/integration/geo-backfill-attempt.test.ts @@ -0,0 +1,107 @@ +/** + * The geo backfill tries a row once per retry wait, and finds its candidates + * through the partial index that holds only them. + * + * Before migration 0418 an address no database could place was looked up on + * every hourly pass for thirty days, and every pass walked thirty days of + * audit rows to find it. Run against the real table: the second pass must not + * pick the row again, a pass past the wait must, and the candidate read must + * go through `audit_logs_geo_backfill_idx` (read off the index's own scan + * counter, so the statement under test is the one Prisma actually sends). + */ +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/geo", () => ({ + lookupIpGeo: vi.fn(async (ip: string) => + ip.startsWith("10.") + ? { location: null, asn: null, carrier: null } + : { location: "Bochum, DE", asn: 3320, carrier: "Deutsche Telekom AG" }, + ), +})); + +const { runGeoBackfill, GEO_BACKFILL_RETRY_DAYS } = + await import("@/lib/jobs/geo-backfill"); +const { lookupIpGeo } = await import("@/lib/geo"); + +const prisma = getPrismaClient(); +const USER = "geo-backfill-attempt"; +const DAY = 86_400_000; +const NOW = new Date(); + +async function indexScans(): Promise { + const [row] = await prisma.$queryRaw>` + SELECT idx_scan AS n FROM pg_stat_user_indexes + WHERE indexrelname = 'audit_logs_geo_backfill_idx'`; + return Number(row?.n ?? 0); +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ data: { id: USER, username: USER } }); + // Enough resolved history that a sequential walk is the costlier plan. + await prisma.$executeRaw` + INSERT INTO audit_logs (id, user_id, action, ip_address, location, asn, carrier, created_at) + SELECT 'resolved-' || g, ${USER}, 'measurement.create', '84.131.0.1', + 'Bochum, DE', 3320, 'Deutsche Telekom AG', + ${NOW}::timestamp - (g * interval '5 seconds') + FROM generate_series(1, 20000) g`; + await prisma.auditLog.createMany({ + data: [ + // Never placeable: a private address. + { + id: "private", + userId: USER, + action: "auth.login", + ipAddress: "10.1.2.3", + createdAt: new Date(NOW.getTime() - 2 * DAY), + }, + // Placeable, missing only its carrier. + { + id: "carrierless", + userId: USER, + action: "auth.login", + ipAddress: "84.131.0.9", + location: "Bochum, DE", + createdAt: new Date(NOW.getTime() - 3 * DAY), + }, + ], + }); + await prisma.$executeRawUnsafe("ANALYZE audit_logs"); +}, 120_000); + +describe("runGeoBackfill", () => { + it("reads its candidates through the partial index", async () => { + const before = await indexScans(); + const summary = await runGeoBackfill(prisma, NOW); + expect(summary.scanned).toBe(2); + expect(summary.carrierResolved).toBe(1); + expect(summary.stillUnresolved).toBe(1); + // Statistics reach the view asynchronously; give them a moment. + let after = await indexScans(); + for (let i = 0; i < 40 && after === before; i += 1) { + await new Promise((r) => setTimeout(r, 250)); + after = await indexScans(); + } + expect(after).toBeGreaterThan(before); + }); + + it("does not try the unplaceable row again inside the wait", async () => { + vi.mocked(lookupIpGeo).mockClear(); + const summary = await runGeoBackfill( + prisma, + new Date(NOW.getTime() + 60 * 60_000), + ); + expect(summary.scanned).toBe(0); + expect(lookupIpGeo).not.toHaveBeenCalled(); + }); + + it("tries it again once the wait has passed", async () => { + const later = new Date(NOW.getTime() + (GEO_BACKFILL_RETRY_DAYS + 1) * DAY); + const summary = await runGeoBackfill(prisma, later); + expect(summary.scanned).toBe(1); + const row = await prisma.auditLog.findUnique({ where: { id: "private" } }); + expect(row?.geoAttemptedAt?.getTime()).toBe(later.getTime()); + }); +}); From 73f2004c54af428103e826bd77acd76e185562b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:27:24 +0200 Subject: [PATCH 09/23] feat(observability): break the heap sample down by V8 space The heap sample named old_space alone, so a heap that stood well above it could not be explained from the log: a buffer past V8's large-object threshold lives in large_object_space, compiled code and bytecode in the code and trusted spaces. The sampled and pressure lines now carry the used size of every space, with any space the runtime reports that is not named summed into other_spaces_mb, so the fields account for the whole used heap whatever the Node version. --- docs/ops/heap-diagnostics.md | 2 +- .../__tests__/heap-monitor.test.ts | 54 ++++++++++++++++ src/lib/observability/heap-monitor.ts | 63 +++++++++++++++++-- 3 files changed, 112 insertions(+), 7 deletions(-) diff --git a/docs/ops/heap-diagnostics.md b/docs/ops/heap-diagnostics.md index 87b12052f..7652079a3 100644 --- a/docs/ops/heap-diagnostics.md +++ b/docs/ops/heap-diagnostics.md @@ -6,7 +6,7 @@ A Node process that reaches its heap limit stops with `JavaScript heap out of me Two background wide events, no configuration needed: -- `runtime.heap.sampled` (info), every five minutes, and every minute while the heap is at or above 70 % of its limit. It goes to stdout and Loki but not to Admin > App logs, so it cannot push other events out of that page's 500-entry buffer. It carries `heap_used_mb`, `heap_total_mb`, `heap_limit_mb`, `heap_used_ratio`, `old_space_mb`, `rss_mb`, `external_mb`, `array_buffers_mb`, `active_requests`, `active_jobs`, `in_flight_dropped` and `uptime_s`. +- `runtime.heap.sampled` (info), every five minutes, and every minute while the heap is at or above 70 % of its limit. It goes to stdout and Loki but not to Admin > App logs, so it cannot push other events out of that page's 500-entry buffer. It carries `heap_used_mb`, `heap_total_mb`, `heap_limit_mb`, `heap_used_ratio`, the used size of each heap space (`new_space_mb`, `old_space_mb`, `code_space_mb`, `large_object_space_mb`, `new_large_object_space_mb`, `code_large_object_space_mb`, `trusted_space_mb`, `read_only_space_mb`, and `other_spaces_mb` for any space not named), `rss_mb`, `external_mb`, `array_buffers_mb`, `active_requests`, `active_jobs`, `in_flight_dropped` and `uptime_s`. The spaces add up to `heap_used_mb`: a large buffer or string (a profile picture held in memory, a long response body) shows in `large_object_space_mb`, not in `old_space_mb`, and memory outside the V8 heap shows in `external_mb` and `array_buffers_mb`. - `runtime.heap.pressure` (warn), the first time the heap reaches 85 % of its limit. Besides the figures above it lists the 50 oldest requests and jobs in flight (`in_flight_oldest`: kind, method, route, age in seconds, whether it was a server action) and the routes most of them share (`in_flight_by_route`). Routes are reduced to their shape: no query string, secret path segments replaced by `[REDACTED]`, ids by `:id`, anything unusual by `:x`. It writes again only after the heap has dropped below 70 % and thirty minutes have passed. One exception: when the heap reaches 95 %, the same report is written again at once, with `pressure_ratio: 0.95`, whatever the cooldown; that tier re-arms as soon as the heap is back under 85 %. A heap that climbs to its limit shortly after an 85 % report therefore still leaves a report naming the work that pushed it over. The heap is read every five seconds, a single call that takes microseconds. Tracking requests costs one map insert and one delete per request. diff --git a/src/lib/observability/__tests__/heap-monitor.test.ts b/src/lib/observability/__tests__/heap-monitor.test.ts index 9e686a52d..9df0ae3bb 100644 --- a/src/lib/observability/__tests__/heap-monitor.test.ts +++ b/src/lib/observability/__tests__/heap-monitor.test.ts @@ -10,6 +10,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { getHeapSpaceStatistics } from "node:v8"; import { describe, expect, it, vi } from "vitest"; @@ -34,6 +35,7 @@ import { type HeapFigures, type HeapMonitorDeps, emitHeapLine, + heapSpaceFigures, isPrivateDirectory, writeDiagnosticHeapSnapshot, } from "../heap-monitor"; @@ -46,7 +48,15 @@ const FIGURE_KEYS: Array = [ "heap_total_mb", "heap_limit_mb", "heap_used_ratio", + "new_space_mb", "old_space_mb", + "code_space_mb", + "large_object_space_mb", + "new_large_object_space_mb", + "code_large_object_space_mb", + "trusted_space_mb", + "read_only_space_mb", + "other_spaces_mb", "rss_mb", "external_mb", "array_buffers_mb", @@ -125,6 +135,50 @@ describe("heap monitor sampling", () => { }); }); +describe("heap space breakdown", () => { + const MB = 1024 * 1024; + + it("reports each named space and sums the rest into other_spaces_mb", () => { + const figures = heapSpaceFigures([ + { space_name: "old_space", space_used_size: 300 * MB }, + { space_name: "large_object_space", space_used_size: 120 * MB }, + { space_name: "trusted_space", space_used_size: 40 * MB }, + { space_name: "shared_space", space_used_size: 2 * MB }, + { space_name: "a_space_v8_adds_later", space_used_size: 3 * MB }, + ]); + expect(figures).toEqual({ + new_space_mb: 0, + old_space_mb: 300, + code_space_mb: 0, + large_object_space_mb: 120, + new_large_object_space_mb: 0, + code_large_object_space_mb: 0, + trusted_space_mb: 40, + read_only_space_mb: 0, + other_spaces_mb: 5, + }); + }); + + it("accounts for every space this runtime reports", () => { + const spaces = getHeapSpaceStatistics(); + expect(spaces.length).toBeGreaterThan(0); + const figures = heapSpaceFigures(spaces); + expect(Object.keys(figures).sort()).toEqual( + FIGURE_KEYS.filter( + (k) => k.endsWith("space_mb") || k === "other_spaces_mb", + ) + .map(String) + .sort(), + ); + const total = spaces.reduce((sum, s) => sum + s.space_used_size, 0); + const reported = Object.values(figures).reduce((a, b) => a + b, 0); + // Each space is rounded to the megabyte on its own. + expect(Math.abs(reported - total / MB)).toBeLessThanOrEqual( + Object.keys(figures).length, + ); + }); +}); + describe("heap pressure report", () => { it("fires once at 85 % with the work in flight, at warn", () => { const h = harness(); diff --git a/src/lib/observability/heap-monitor.ts b/src/lib/observability/heap-monitor.ts index 362a9f7ae..5e049763d 100644 --- a/src/lib/observability/heap-monitor.ts +++ b/src/lib/observability/heap-monitor.ts @@ -72,12 +72,66 @@ export const HEAP_CRITICAL_RATIO = 0.95; export const HEAP_PRESSURE_COOLDOWN_MS = 30 * 60_000; export const HEAP_PRESSURE_LIST_MAX = 50; -export interface HeapFigures { +/** + * The used size of each V8 heap space, by the field it is reported under. + * + * `old_space` alone left most of a large heap unexplained: a buffer or string + * past V8's large-object threshold (a profile picture read into memory, a + * long JSON payload) lives in `large_object_space`, compiled code in the code + * spaces, and since V8 12 bytecode and other sandboxed objects in + * `trusted_space`. Every space a runtime reports that is not named here + * (the shared spaces, a space a later V8 adds) is summed into + * `other_spaces_mb`, so the fields account for the whole used heap and the + * line keeps one shape whatever the Node version. + */ +const HEAP_SPACE_FIELDS = { + new_space: "new_space_mb", + old_space: "old_space_mb", + code_space: "code_space_mb", + large_object_space: "large_object_space_mb", + new_large_object_space: "new_large_object_space_mb", + code_large_object_space: "code_large_object_space_mb", + trusted_space: "trusted_space_mb", + read_only_space: "read_only_space_mb", +} as const; + +type HeapSpaceField = + | (typeof HEAP_SPACE_FIELDS)[keyof typeof HEAP_SPACE_FIELDS] + | "other_spaces_mb"; + +export type HeapSpaceFigures = Record; + +/** The per-space breakdown of `getHeapSpaceStatistics()`, in megabytes. */ +export function heapSpaceFigures( + spaces: ReadonlyArray<{ space_name: string; space_used_size: number }>, +): HeapSpaceFigures { + const bytes: Record = { + new_space_mb: 0, + old_space_mb: 0, + code_space_mb: 0, + large_object_space_mb: 0, + new_large_object_space_mb: 0, + code_large_object_space_mb: 0, + trusted_space_mb: 0, + read_only_space_mb: 0, + other_spaces_mb: 0, + }; + for (const space of spaces) { + const field = + HEAP_SPACE_FIELDS[space.space_name as keyof typeof HEAP_SPACE_FIELDS] ?? + "other_spaces_mb"; + bytes[field] += space.space_used_size; + } + return Object.fromEntries( + Object.entries(bytes).map(([field, size]) => [field, megabytes(size)]), + ) as HeapSpaceFigures; +} + +export interface HeapFigures extends HeapSpaceFigures { heap_used_mb: number; heap_total_mb: number; heap_limit_mb: number; heap_used_ratio: number; - old_space_mb: number; rss_mb: number; external_mb: number; array_buffers_mb: number; @@ -115,9 +169,6 @@ function defaultReadHeap(): { used: number; limit: number } { function defaultReadFigures(registry: InFlightRegistry): HeapFigures { const memory = process.memoryUsage(); const limit = getHeapStatistics().heap_size_limit; - const oldSpace = getHeapSpaceStatistics().find( - (space) => space.space_name === "old_space", - ); const counts = registry.counts(); return { heap_used_mb: megabytes(memory.heapUsed), @@ -125,7 +176,7 @@ function defaultReadFigures(registry: InFlightRegistry): HeapFigures { heap_limit_mb: megabytes(limit), heap_used_ratio: limit > 0 ? Math.round((memory.heapUsed / limit) * 1000) / 1000 : 0, - old_space_mb: megabytes(oldSpace?.space_used_size ?? 0), + ...heapSpaceFigures(getHeapSpaceStatistics()), rss_mb: megabytes(memory.rss), external_mb: megabytes(memory.external), array_buffers_mb: megabytes(memory.arrayBuffers), From 05ae8b1e995fc408f54653ade00276b944fcf2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:27:32 +0200 Subject: [PATCH 10/23] perf(dashboard): count all-time readings from the DAY rollups The summary's all-time aggregate grouped every live reading of the account by type: one index entry per reading, millions on an account with a watch, two to four seconds whenever they were not in memory and the slowest read of the request on every cache miss. The count now sums the DAY buckets that already hold a per-day count of every source, counts live only the readings before the fold window and any type not folded yet, and takes lastSeenAt from the newest reading the latest-reading read already returns. Parity with the live count is pinned against a real Postgres. --- .../dashboard-summary-read-plan-guard.test.ts | 2 +- .../dense-measurement-read-guard.test.ts | 2 + .../dashboard/summary/__tests__/route.test.ts | 85 ++++++++--------- src/app/api/dashboard/summary/route.ts | 25 ++--- src/lib/dashboard/summary-reads.ts | 77 ++++++++++++++++ .../dashboard-summary-reads.test.ts | 92 +++++++++++++++++++ 6 files changed, 229 insertions(+), 54 deletions(-) diff --git a/src/__tests__/dashboard-summary-read-plan-guard.test.ts b/src/__tests__/dashboard-summary-read-plan-guard.test.ts index b1b9a14e1..ce7035eb5 100644 --- a/src/__tests__/dashboard-summary-read-plan-guard.test.ts +++ b/src/__tests__/dashboard-summary-read-plan-guard.test.ts @@ -85,7 +85,7 @@ describe("dashboard summary reads — sparkline rollup buckets", () => { // defeat statement reuse. const sql = code(READS); const casts = sql.match(/\}::"measurement_type"\[\]/g) ?? []; - expect(casts.length).toBe(2); + expect(casts.length).toBe(3); expect(sql).not.toMatch(/\$queryRawUnsafe/); }); }); diff --git a/src/__tests__/dense-measurement-read-guard.test.ts b/src/__tests__/dense-measurement-read-guard.test.ts index 75c083e73..07c9d441b 100644 --- a/src/__tests__/dense-measurement-read-guard.test.ts +++ b/src/__tests__/dense-measurement-read-guard.test.ts @@ -134,6 +134,8 @@ const ALLOWED_SQL: Record = { "lib/rollups/measurement-coverage.ts::1": "Selects from `unnest` of the caller's type list; `measurements` is read inside `EXISTS`, one row per type at most.", "lib/dashboard/summary-reads.ts::1": + "Selects from `unnest` of the type list; `measurements` is read only inside COUNT scalar subqueries, before the fold window for a type the DAY rollups cover.", + "lib/dashboard/summary-reads.ts::2": "Selects from `generate_series` of the streak window's local days; `measurements` is read inside `EXISTS`, one row per day at most.", }; diff --git a/src/app/api/dashboard/summary/__tests__/route.test.ts b/src/app/api/dashboard/summary/__tests__/route.test.ts index b74b91c07..c6fd2ca31 100644 --- a/src/app/api/dashboard/summary/__tests__/route.test.ts +++ b/src/app/api/dashboard/summary/__tests__/route.test.ts @@ -78,6 +78,11 @@ vi.mock("@/lib/rollups/measurement-read", () => ({ // covered by `src/lib/analytics/__tests__/mood-series.test.ts`. Pinned to an // empty series by default so the legacy assertions keep their "no mood" // expectation, and overridden per test below. +vi.mock("@/lib/dashboard/summary-reads", async (importOriginal) => ({ + ...(await importOriginal()), + readAllTimeCounts: vi.fn(), +})); + vi.mock("@/lib/analytics/mood-series", () => ({ buildMoodDailySeries: vi.fn(async () => ({ entries: [], @@ -106,6 +111,7 @@ import { getSession } from "@/lib/auth/session"; import { __resetAllCachesForTests } from "@/lib/cache/server-cache"; import { buildMoodDailySeries } from "@/lib/analytics/mood-series"; import { userDayKey } from "@/lib/tz/format"; +import { readAllTimeCounts } from "@/lib/dashboard/summary-reads"; const SESSION_OK = { session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, @@ -133,10 +139,10 @@ beforeEach(() => { // between tests to keep cross-test isolation. __resetAllCachesForTests(); vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); - // v1.4.33 — the route now issues a `groupBy` for - // per-type all-time count + most-recent timestamp. Default to an - // empty aggregate so legacy tests keep their "no data" expectations. - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([] as never); + // The per-type all-time counts (`readAllTimeCounts`). Default to none so + // legacy tests keep their "no data" expectations; the newest reading per + // type, which `lastSeenAt` carries, comes from the latest-reading read. + vi.mocked(readAllTimeCounts).mockResolvedValue([] as never); // v1.11.4 — the sleep tile reads the raw per-stage SLEEP_DURATION rows // via `measurement.findMany` to reconstruct the night total. Default to // no rows so legacy tests keep their "no sleep" expectation. @@ -249,22 +255,26 @@ describe("GET /api/dashboard/summary", () => { } }); - it("surfaces lastSeenAt from the historical aggregate even when the 7-day window is empty", async () => { + it("surfaces lastSeenAt from the newest reading even when the 7-day window is empty", async () => { // Power-user path — the user has logged weight for years but - // hasn't touched the app in two weeks. `groupBy` returns the - // all-time count + most-recent timestamp; the 7-day `findMany` - // returns empty. The route should still emit the weight card + // hasn't touched the app in two weeks. The all-time count and the + // newest reading come back; the sparkline is empty. The route should + // still emit the weight card // with the historical `lastSeenAt` so the iOS client renders the // staleness caption. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const twoWeeksAgo = new Date(Date.now() - 14 * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "WEIGHT", - _count: { _all: 312 }, - _max: { measuredAt: twoWeeksAgo }, + count: 312, }, ] as never); + // The newest reading, which `lastSeenAt` carries (first `$queryRaw` of + // the round: the latest-reading read). + vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([ + { type: "WEIGHT", value: 81.2, measured_at: twoWeeksAgo }, + ] as never); const res = await callGet(makeReq()); const body = (await res.json()) as { data: { @@ -281,10 +291,9 @@ describe("GET /api/dashboard/summary", () => { expect(weight, "weight card must be present").toBeDefined(); expect(weight?.allTimeCount).toBe(312); expect(weight?.lastSeenAt).toBe(twoWeeksAgo.toISOString()); - // No 7-day reading → latestValue stays null but updatedAt falls - // through to the historical timestamp so the iOS client can - // build the relative-age caption from a single field. - expect(weight?.latestValue).toBeNull(); + // The tile shows the historical reading (REG-11) and dates it, so the + // iOS client builds the relative-age caption from one field. + expect(weight?.latestValue).toBe(81.2); expect(weight?.updatedAt).toBe(twoWeeksAgo.toISOString()); }); @@ -299,16 +308,14 @@ describe("GET /api/dashboard/summary", () => { vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const sixtyDaysAgo = new Date(Date.now() - 60 * 86_400_000); const sixtyOneDaysAgo = new Date(Date.now() - 61 * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "BLOOD_PRESSURE_SYS", - _count: { _all: 14 }, - _max: { measuredAt: sixtyDaysAgo }, + count: 14, }, { type: "BLOOD_PRESSURE_DIA", - _count: { _all: 14 }, - _max: { measuredAt: sixtyDaysAgo }, + count: 14, }, ] as never); // Promise.all order: [latestEver, sparkline, streakDays]. @@ -406,11 +413,10 @@ describe("GET /api/dashboard/summary", () => { // don't accidentally regress the fresh-data case. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const twoDaysAgo = new Date(Date.now() - 2 * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "WEIGHT", - _count: { _all: 200 }, - _max: { measuredAt: twoDaysAgo }, + count: 200, }, ] as never); vi.mocked(prisma.$queryRaw) @@ -457,11 +463,10 @@ describe("GET /api/dashboard/summary", () => { // oldest bucket first. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const days = (n: number) => new Date(Date.now() - n * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "PULSE", - _count: { _all: 42 }, - _max: { measuredAt: days(30) }, + count: 42, }, ] as never); vi.mocked(prisma.$queryRaw) @@ -547,13 +552,15 @@ describe("GET /api/dashboard/summary", () => { // hint). vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const tenDaysAgo = new Date(Date.now() - 10 * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "BLOOD_GLUCOSE", - _count: { _all: 4 }, - _max: { measuredAt: tenDaysAgo }, + count: 4, }, ] as never); + vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([ + { type: "BLOOD_GLUCOSE", value: 104, measured_at: tenDaysAgo }, + ] as never); const res = await callGet(makeReq()); const body = (await res.json()) as { data: { @@ -579,11 +586,10 @@ describe("GET /api/dashboard/summary", () => { // hours, NOT surface a single stage. Excludes IN_BED + AWAKE. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const wake = new Date("2026-06-04T06:00:00.000Z"); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "SLEEP_DURATION", - _count: { _all: 5 }, - _max: { measuredAt: wake }, + count: 5, }, ] as never); // Raw per-stage rows for the night (the sleep findMany read). @@ -654,11 +660,10 @@ describe("GET /api/dashboard/summary", () => { // total; the additive annotation surfaces the disagreement. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const wake = new Date("2026-06-04T06:00:00.000Z"); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "SLEEP_DURATION", - _count: { _all: 4 }, - _max: { measuredAt: wake }, + count: 4, }, ] as never); vi.mocked(prisma.measurement.findMany).mockResolvedValue([ @@ -780,11 +785,10 @@ describe("GET /api/dashboard/summary", () => { // returned shape proves the route hands `sum_value` through // instead of reconstructing from `mean * count`. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "ACTIVITY_STEPS", - _count: { _all: 7 }, - _max: { measuredAt: new Date() }, + count: 7, }, ] as never); // First call: latest7d. Second call: sparkline. Third call: @@ -824,11 +828,10 @@ describe("GET /api/dashboard/summary", () => { it("falls back to mean * count when the legacy sum_value is null (v1.4.39 W-SUM)", async () => { vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "ACTIVITY_STEPS", - _count: { _all: 7 }, - _max: { measuredAt: new Date() }, + count: 7, }, ] as never); vi.mocked(prisma.$queryRaw) diff --git a/src/app/api/dashboard/summary/route.ts b/src/app/api/dashboard/summary/route.ts index 41a60ae4a..4526bf946 100644 --- a/src/app/api/dashboard/summary/route.ts +++ b/src/app/api/dashboard/summary/route.ts @@ -79,6 +79,7 @@ import { buildMedsTodayBlock } from "@/lib/dashboard/meds-today"; import { computeStreak } from "@/lib/dashboard/streak"; import { readActivityDayKeys, + readAllTimeCounts, readLatestEver, readSparkBuckets, } from "@/lib/dashboard/summary-reads"; @@ -468,7 +469,7 @@ async function buildDashboardSummary( // - sparkBuckets: ≤ SPARK_DAYS × N metric types (typically <80, // REG-11: taken via ROW_NUMBER window, no calendar // filter, so a 60-day-old metric still paints a chart) - // - allTimeAggregate: ≤ N metric types (unchanged) + // - allTimeAggregate: ≤ N metric types, summed from the DAY rollups // - medsToday: today-window tally via the shared projector // (≤ daily intake schedule count, unchanged semantics) // - streakActivity: ≤ daily intake count × 365 (unchanged) @@ -492,13 +493,10 @@ async function buildDashboardSummary( time("sparkline", () => readSparkBuckets(prisma, userId, measurementTypes, SPARK_DAYS), ), + // Counted from the DAY rollups where they cover the days, not by one + // index entry per reading (see `readAllTimeCounts`). time("allTime", () => - prisma.measurement.groupBy({ - by: ["type"], - where: { userId, type: { in: measurementTypes }, deletedAt: null }, - _count: { _all: true }, - _max: { measuredAt: true }, - }), + readAllTimeCounts(prisma, userId, measurementTypes, now), ), // v1.4.39 — the dashboard compliance tile pulled // `MedicationIntakeEvent` rows for the today-window only. Daily @@ -583,17 +581,20 @@ async function buildDashboardSummary( // Per-type metadata lookup — typed Map so a metric with no readings // at all falls through `metaForType` to the `{ allTimeCount: 0, - // lastSeenAt: null }` default. The aggregate row's `_count._all` is - // always `number` per Prisma's runtime contract; defending against - // undefined keeps the helper's narrow signature stable. + // lastSeenAt: null }` default. const allTimeByType = new Map< MeasurementType, { allTimeCount: number; lastSeenAt: Date | null } >(); + // The newest reading is the one `latestEver` already read per type; its + // instant is the metric's last-seen time. + const lastSeenByType = new Map( + latestEver.map((row) => [row.type, row.measured_at]), + ); for (const row of allTimeAggregate) { allTimeByType.set(row.type, { - allTimeCount: row._count?._all ?? 0, - lastSeenAt: row._max?.measuredAt ?? null, + allTimeCount: row.count, + lastSeenAt: lastSeenByType.get(row.type) ?? null, }); } function metaForType(type: MeasurementType): { diff --git a/src/lib/dashboard/summary-reads.ts b/src/lib/dashboard/summary-reads.ts index 1a0322d63..8f8550160 100644 --- a/src/lib/dashboard/summary-reads.ts +++ b/src/lib/dashboard/summary-reads.ts @@ -17,6 +17,7 @@ */ import { Prisma, type PrismaClient } from "@/generated/prisma/client"; import type { MeasurementType } from "@/generated/prisma/client"; +import { ROLLUP_FOLD_WINDOW_MS } from "@/lib/rollups/measurement-rollups"; /** v1.4.38 — the single most recent reading per measurement * type, irrespective of age. One row per metric the user has ever @@ -166,6 +167,82 @@ export function readSparkBuckets( `; } +/** One metric's all-time reading count, for a type that has any. */ +export interface AllTimeCountRow { + type: MeasurementType; + count: number; +} + +/** + * The first UTC day the DAY rollups are trusted to hold in full: one day + * inside the fold window. Every fold that ever ran for the account began at + * or before it (its window was anchored on an earlier `now`), and the write + * hooks keep every later day current. + */ +export function allTimeRollupBoundary(now: Date): Date { + const DAY_MS = 24 * 60 * 60 * 1000; + const start = now.getTime() - ROLLUP_FOLD_WINDOW_MS; + return new Date(Math.floor(start / DAY_MS) * DAY_MS + DAY_MS); +} + +/** + * Total live readings per measurement type, all time. + * + * Counting them on `measurements` reads one index entry per reading: on an + * account with a watch that is millions of entries per request, two to four + * seconds whenever they are not in memory, which is the dashboard's slowest + * read by far. The DAY rollups already carry a count per (type, day, source) + * — every live reading of the day, every source — so the sum of a type's DAY + * buckets is its count over the days they cover. Readings before the fold + * window, which the fold never buckets, are counted live; that range is a + * small, bounded index slice. A type with no DAY bucket at all (not folded + * yet) is counted live in full, exactly as before. Types with no readings + * are absent. + */ +export async function readAllTimeCounts( + db: PrismaClient, + userId: string, + measurementTypes: readonly MeasurementType[], + now: Date = new Date(), +): Promise { + const boundary = allTimeRollupBoundary(now); + const rows = await db.$queryRaw< + Array<{ type: MeasurementType; count: number }> + >` + SELECT wanted."type" AS type, + (CASE + WHEN EXISTS ( + SELECT 1 FROM measurement_rollups r + WHERE r."user_id" = ${userId} + AND r."type" = wanted."type" + AND r."granularity" = 'DAY' + ) + THEN COALESCE(( + SELECT SUM(r."count") FROM measurement_rollups r + WHERE r."user_id" = ${userId} + AND r."type" = wanted."type" + AND r."granularity" = 'DAY' + AND r."bucket_start" >= ${boundary} + ), 0) + ( + SELECT COUNT(*) FROM measurements m + WHERE m."user_id" = ${userId} + AND m."type" = wanted."type" + AND m."deleted_at" IS NULL + AND m."measured_at" < ${boundary} + ) + ELSE ( + SELECT COUNT(*) FROM measurements m + WHERE m."user_id" = ${userId} + AND m."type" = wanted."type" + AND m."deleted_at" IS NULL + ) + END)::int AS count + FROM unnest(${[...measurementTypes]}::"measurement_type"[]) AS wanted("type") + ORDER BY wanted."type" + `; + return rows.filter((row) => row.count > 0); +} + /** * The local days in `[since, now]` that hold at least one live reading, as * `YYYY-MM-DD` keys in `timeZone`: the measurement half of the streak. diff --git a/tests/integration/dashboard-summary-reads.test.ts b/tests/integration/dashboard-summary-reads.test.ts index 6dcf1f840..979c6746f 100644 --- a/tests/integration/dashboard-summary-reads.test.ts +++ b/tests/integration/dashboard-summary-reads.test.ts @@ -27,12 +27,15 @@ import { beforeEach, describe, expect, it } from "vitest"; import { getPrismaClient, truncateAllTables } from "./setup"; import { + allTimeRollupBoundary, + readAllTimeCounts, readLatestEver, readSparkBuckets, type LatestEverRow, type SparklineRow, } from "@/lib/dashboard/summary-reads"; import { measurementTypeEnum } from "@/lib/validations/measurement"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; import type { MeasurementType, RollupGranularity, @@ -245,3 +248,92 @@ describe("dashboard summary reads — parity with the pre-v1.38.5 SQL", () => { } }); }); + +describe("dashboard all-time counts — parity with the live count", () => { + /** The pre-rollup read: one index entry per live reading. */ + async function legacyAllTime(userId: string) { + const groups = await getPrismaClient().measurement.groupBy({ + by: ["type"], + where: { userId, type: { in: TYPES }, deletedAt: null }, + _count: { _all: true }, + }); + return new Map(groups.map((g) => [g.type, g._count._all])); + } + + it("counts what the live count counts, from the rollups", async () => { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username: "all-time-counts", email: "all-time@example.test" }, + }); + const now = new Date(); + const boundary = allTimeRollupBoundary(now); + + // A watch-dense pulse history from two sources: a reading every four + // minutes for two years, written set-wise the way an import lands it. + const twoYears = 2 * 365 * DAY; + for (const [source, offsetMinutes] of [ + ["APPLE_HEALTH", 0], + ["WITHINGS", 2], + ] as const) { + await prisma.$executeRawUnsafe( + `INSERT INTO measurements + (id, user_id, type, value, unit, source, measured_at, created_at, updated_at) + SELECT md5($1 || $2 || g::text), $1, 'PULSE'::"measurement_type", + 60 + (g % 30), 'bpm', $2::"measurement_source", + $3::timestamptz + make_interval(mins => g * 4 + $4), now(), now() + FROM generate_series(0, $5::int) AS g`, + user.id, + source, + new Date(now.getTime() - twoYears), + offsetMinutes, + Math.floor(twoYears / (4 * 60_000)) - 1, + ); + } + // Weigh-ins: some older than the fold window, one tombstoned. + await prisma.measurement.createMany({ + data: [ + ...[1, 2, 3].map((years) => ({ + userId: user.id, + type: "WEIGHT" as MeasurementType, + value: 80 + years, + unit: "kg", + measuredAt: new Date(boundary.getTime() - years * 30 * DAY), + })), + ...[3, 2, 1].map((days) => ({ + userId: user.id, + type: "WEIGHT" as MeasurementType, + value: 79, + unit: "kg", + measuredAt: new Date(now.getTime() - days * DAY), + deletedAt: days === 2 ? now : null, + })), + ], + }); + await recomputeUserRollups(user.id, { granularities: ["DAY"] }); + // A type written after the fold, never bucketed: counted live. + await prisma.measurement.createMany({ + data: [5, 4].map((days) => ({ + userId: user.id, + type: "BLOOD_GLUCOSE" as MeasurementType, + value: 98, + unit: "mg/dL", + measuredAt: new Date(now.getTime() - days * DAY), + })), + }); + + let started = performance.now(); + const legacy = await legacyAllTime(user.id); + const legacyMs = performance.now() - started; + started = performance.now(); + const shipped = await readAllTimeCounts(prisma, user.id, TYPES, now); + const shippedMs = performance.now() - started; + console.log( + `all-time counts over ${legacy.get("PULSE")} pulse rows: live ${legacyMs.toFixed(0)} ms, rollups ${shippedMs.toFixed(0)} ms`, + ); + + expect(new Map(shipped.map((r) => [r.type, r.count]))).toEqual(legacy); + expect(legacy.get("PULSE")).toBeGreaterThan(500_000); + expect(legacy.get("WEIGHT")).toBe(5); + expect(legacy.get("BLOOD_GLUCOSE")).toBe(2); + }); +}); From 0534123d0d947453e8bac6a90d8ad37f1be00691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:28:21 +0200 Subject: [PATCH 11/23] perf(integrations): read the status envelope in one round The provider rows, the unified connection health and the per-metric freshness were awaited one after another although none depends on the others, so a cold first visit to the settings card paid their latencies end to end. They now run in one round. --- .../status/__tests__/route.test.ts | 23 ++ src/app/api/integrations/status/route.ts | 251 +++++++++--------- 2 files changed, 153 insertions(+), 121 deletions(-) diff --git a/src/app/api/integrations/status/__tests__/route.test.ts b/src/app/api/integrations/status/__tests__/route.test.ts index 6b8d0e68d..43596e00d 100644 --- a/src/app/api/integrations/status/__tests__/route.test.ts +++ b/src/app/api/integrations/status/__tests__/route.test.ts @@ -485,3 +485,26 @@ describe("/api/integrations/status — the unified connection health", () => { } }); }); + +describe("/api/integrations/status — one round of reads", () => { + it("asks for the connection health and the freshness without waiting on the provider rows", async () => { + userFind.mockResolvedValue({}); + // Hold the user row back: under the old shape nothing else was asked + // until it (and the rest of the first round) had answered. + let release: (row: unknown) => void = () => {}; + userFind.mockImplementationOnce( + () => new Promise((resolve) => (release = resolve)), + ); + const { loadConnectionHealth } = + await import("@/lib/integrations/connection-health-load"); + + const pending = fetchEnvelope(); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(loadConnectionHealth).toHaveBeenCalledTimes(1); + expect(listLiveMeasurementTypes).toHaveBeenCalled(); + release({}); + const envelope = await pending; + expect(envelope.integrations.length).toBeGreaterThan(0); + }); +}); diff --git a/src/app/api/integrations/status/route.ts b/src/app/api/integrations/status/route.ts index 1c0d439c3..73396103d 100644 --- a/src/app/api/integrations/status/route.ts +++ b/src/app/api/integrations/status/route.ts @@ -64,124 +64,138 @@ export const GET = apiHandler(async () => { const { user } = await requireAuth(); annotate({ action: { name: "integrations.status" } }); + // One round: the per-provider rows, the unified connection health and the + // per-metric freshness do not depend on each other. Awaited one after + // another, a cold first visit paid the three latencies end to end. + let metricFreshnessDegraded = false; const [ - withingsStatus, - whoopStatus, - fitbitStatus, - googleHealthStatus, - polarStatus, - ouraStatus, - stravaStatus, - nightscoutStatus, - dawarichStatus, - dbUser, - withingsConn, - whoopConn, - fitbitConn, - googleHealthConn, - // v1.17.1 — `available` reports whether usable OAuth credentials resolve - // (per-user BYO first, then the shared env app), mirroring the per-card - // /api//status the consolidated envelope now replaces. - polarAvailable, - ouraAvailable, - stravaAvailable, + [ + withingsStatus, + whoopStatus, + fitbitStatus, + googleHealthStatus, + polarStatus, + ouraStatus, + stravaStatus, + nightscoutStatus, + dawarichStatus, + dbUser, + withingsConn, + whoopConn, + fitbitConn, + googleHealthConn, + // v1.17.1 — `available` reports whether usable OAuth credentials resolve + // (per-user BYO first, then the shared env app), mirroring the per-card + // /api//status the consolidated envelope now replaces. + polarAvailable, + ouraAvailable, + stravaAvailable, + ], + health, + metricFreshness, ] = await Promise.all([ - getIntegrationStatus(user.id, "withings"), - getIntegrationStatus(user.id, "whoop"), - getIntegrationStatus(user.id, "fitbit"), - getIntegrationStatus(user.id, "google-health"), - getIntegrationStatus(user.id, "polar"), - getIntegrationStatus(user.id, "oura"), - getIntegrationStatus(user.id, "strava"), - getIntegrationStatus(user.id, "nightscout"), - getIntegrationStatus(user.id, "dawarich"), - prisma.user.findUnique({ - where: { id: user.id }, - select: { - withingsClientIdEncrypted: true, - withingsClientSecretEncrypted: true, - whoopClientIdEncrypted: true, - whoopClientSecretEncrypted: true, - fitbitClientIdEncrypted: true, - fitbitClientSecretEncrypted: true, - googleHealthClientIdEncrypted: true, - googleHealthClientSecretEncrypted: true, - polarAccessTokenEncrypted: true, - polarClientIdEncrypted: true, - polarClientSecretEncrypted: true, - ouraAccessTokenEncrypted: true, - ouraClientIdEncrypted: true, - ouraClientSecretEncrypted: true, - ouraScopeDenied: true, - stravaAccessTokenEncrypted: true, - stravaClientIdEncrypted: true, - stravaClientSecretEncrypted: true, - // Nightscout folds onto the envelope; the card reads these three off - // it instead of its own /api/nightscout/status round-trip. - nightscoutUrlEncrypted: true, - nightscoutTokenEncrypted: true, - nightscoutAllowPrivateHost: true, - dawarichUrlEncrypted: true, - }, + Promise.all([ + getIntegrationStatus(user.id, "withings"), + getIntegrationStatus(user.id, "whoop"), + getIntegrationStatus(user.id, "fitbit"), + getIntegrationStatus(user.id, "google-health"), + getIntegrationStatus(user.id, "polar"), + getIntegrationStatus(user.id, "oura"), + getIntegrationStatus(user.id, "strava"), + getIntegrationStatus(user.id, "nightscout"), + getIntegrationStatus(user.id, "dawarich"), + prisma.user.findUnique({ + where: { id: user.id }, + select: { + withingsClientIdEncrypted: true, + withingsClientSecretEncrypted: true, + whoopClientIdEncrypted: true, + whoopClientSecretEncrypted: true, + fitbitClientIdEncrypted: true, + fitbitClientSecretEncrypted: true, + googleHealthClientIdEncrypted: true, + googleHealthClientSecretEncrypted: true, + polarAccessTokenEncrypted: true, + polarClientIdEncrypted: true, + polarClientSecretEncrypted: true, + ouraAccessTokenEncrypted: true, + ouraClientIdEncrypted: true, + ouraClientSecretEncrypted: true, + ouraScopeDenied: true, + stravaAccessTokenEncrypted: true, + stravaClientIdEncrypted: true, + stravaClientSecretEncrypted: true, + // Nightscout folds onto the envelope; the card reads these three off + // it instead of its own /api/nightscout/status round-trip. + nightscoutUrlEncrypted: true, + nightscoutTokenEncrypted: true, + nightscoutAllowPrivateHost: true, + dawarichUrlEncrypted: true, + }, + }), + prisma.withingsConnection.findUnique({ + where: { userId: user.id }, + select: { + tokenExpiresAt: true, + lastSyncedAt: true, + createdAt: true, + scope: true, + apiPlanState: true, + }, + }), + prisma.whoopConnection.findUnique({ + where: { userId: user.id }, + select: { + tokenExpiresAt: true, + whoopUserId: true, + lastSyncedAt: true, + createdAt: true, + backfillCompletedAt: true, + }, + }), + prisma.fitbitConnection.findUnique({ + where: { userId: user.id }, + select: { + tokenExpiresAt: true, + lastSyncedAt: true, + createdAt: true, + backfillCompletedAt: true, + }, + }), + prisma.googleHealthConnection.findUnique({ + where: { userId: user.id }, + select: { + tokenExpiresAt: true, + lastSyncedAt: true, + createdAt: true, + backfillCompletedAt: true, + backfillProgress: true, + needsReauth: true, + }, + }), + getPolarClientCredentials(user.id).then((c) => !!c), + getOuraClientCredentials(user.id).then((c) => !!c), + getStravaClientCredentials(user.id).then((c) => !!c), + ]), + // The unified per-connection state. Resolved by the same loader the + // per-provider routes and the integration watchdog use, so the envelope + // and every other surface say the same thing about a connection. + loadConnectionHealth(user.id, [ + "withings", + "whoop", + "fitbit", + "google-health", + "polar", + "oura", + "strava", + "nightscout", + "dawarich", + ]), + // See `metricFreshnessDegraded` below. + getSourceMetricFreshness(user.id).catch(() => { + metricFreshnessDegraded = true; + return {} as Partial>; }), - prisma.withingsConnection.findUnique({ - where: { userId: user.id }, - select: { - tokenExpiresAt: true, - lastSyncedAt: true, - createdAt: true, - scope: true, - apiPlanState: true, - }, - }), - prisma.whoopConnection.findUnique({ - where: { userId: user.id }, - select: { - tokenExpiresAt: true, - whoopUserId: true, - lastSyncedAt: true, - createdAt: true, - backfillCompletedAt: true, - }, - }), - prisma.fitbitConnection.findUnique({ - where: { userId: user.id }, - select: { - tokenExpiresAt: true, - lastSyncedAt: true, - createdAt: true, - backfillCompletedAt: true, - }, - }), - prisma.googleHealthConnection.findUnique({ - where: { userId: user.id }, - select: { - tokenExpiresAt: true, - lastSyncedAt: true, - createdAt: true, - backfillCompletedAt: true, - backfillProgress: true, - needsReauth: true, - }, - }), - getPolarClientCredentials(user.id).then((c) => !!c), - getOuraClientCredentials(user.id).then((c) => !!c), - getStravaClientCredentials(user.id).then((c) => !!c), - ]); - - // The unified per-connection state. Resolved by the same loader the - // per-provider routes and the integration watchdog use, so the envelope and - // every other surface say the same thing about a connection. - const health = await loadConnectionHealth(user.id, [ - "withings", - "whoop", - "fitbit", - "google-health", - "polar", - "oura", - "strava", - "nightscout", - "dawarich", ]); // Per-metric-type last-value timestamps so the card can show that a @@ -196,12 +210,7 @@ export const GET = apiHandler(async () => { // the wire, and the honesty signal quietly became the thing it was built to // prevent. `metricFreshnessDegraded` separates them. One flag rather than a // nullable array per entry, because this is ONE query for all eight providers - // — it fails for all of them or for none. - let metricFreshnessDegraded = false; - const metricFreshness = await getSourceMetricFreshness(user.id).catch(() => { - metricFreshnessDegraded = true; - return {} as Partial>; - }); + // — it fails for all of them or for none. (Read in the round above.) const now = Date.now(); const whoopConnected = !!whoopConn?.whoopUserId; From 7759729aca71d124030e2a121a874476c1c6a638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:29:46 +0200 Subject: [PATCH 12/23] fix(proxy): refuse Server Action calls the app never makes HealthLog defines no Server Actions, so no bundle of it, current or stale, sends a Next-Action header. The requests that do arrive come from the nightly scan probing that surface, and Next answered each with an unstructured "Failed to find Server Action" at error level. The proxy now refuses them with a 404 and one structured warn line, and a guard keeps "use server" out of the tree so the refusal stays safe. --- src/__tests__/no-server-actions-guard.test.ts | 37 +++++++++ src/__tests__/proxy-server-action.test.ts | 77 +++++++++++++++++++ src/proxy.ts | 40 ++++++++++ 3 files changed, 154 insertions(+) create mode 100644 src/__tests__/no-server-actions-guard.test.ts create mode 100644 src/__tests__/proxy-server-action.test.ts diff --git a/src/__tests__/no-server-actions-guard.test.ts b/src/__tests__/no-server-actions-guard.test.ts new file mode 100644 index 000000000..8dd2b7700 --- /dev/null +++ b/src/__tests__/no-server-actions-guard.test.ts @@ -0,0 +1,37 @@ +/** + * HealthLog defines no Server Actions. + * + * The proxy refuses every request carrying `Next-Action` on the strength of + * that (`src/proxy.ts`): the only callers that send one are scanners. A module + * that declares `"use server"` would make the refusal break a real feature, so + * adding one has to come with revisiting it. This guard is the reminder. + */ +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import { describe, expect, it } from "vitest"; + +const SRC = join(__dirname, ".."); + +function sourceFiles(dir: string): string[] { + const out: string[] = []; + for (const name of readdirSync(dir)) { + const path = join(dir, name); + if (name === "generated" || name === "__tests__") continue; + if (statSync(path).isDirectory()) out.push(...sourceFiles(path)); + else if (/\.(ts|tsx)$/.test(name)) out.push(path); + } + return out; +} + +describe("no Server Actions", () => { + it('has no module or function that declares "use server"', () => { + const files = sourceFiles(SRC); + expect(files.length).toBeGreaterThan(100); + const declaring = files + .filter((file) => + /^\s*["']use server["'];?\s*$/m.test(readFileSync(file, "utf8")), + ) + .map((file) => relative(SRC, file)); + expect(declaring).toEqual([]); + }); +}); diff --git a/src/__tests__/proxy-server-action.test.ts b/src/__tests__/proxy-server-action.test.ts new file mode 100644 index 000000000..a41e8a129 --- /dev/null +++ b/src/__tests__/proxy-server-action.test.ts @@ -0,0 +1,77 @@ +/** + * A Server Action call is refused at the proxy. + * + * HealthLog defines no Server Actions, so the only requests that carry + * `Next-Action` are scanners probing for them. Next used to answer each with + * an unstructured "Failed to find Server Action" at error level; the proxy now + * refuses it with a 404 and one structured `warn` line, and a request without + * the header is untouched. + */ +import { NextRequest } from "next/server"; +import { + afterEach, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from "vitest"; + +vi.mock("@/lib/process-type", () => ({ shouldRunWeb: () => true })); + +import { proxy } from "../proxy"; + +let warn: MockInstance<(...args: unknown[]) => void>; + +beforeEach(() => { + warn = vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + warn.mockRestore(); +}); + +function probe(path: string, headers: Record = {}) { + return new NextRequest(`https://health.example.test${path}`, { + method: "POST", + headers: { "content-type": "text/plain", ...headers }, + body: "[]", + }); +} + +describe("proxy on a Server Action call", () => { + it("refuses it with 404 and one warn line", async () => { + const response = proxy( + probe("/", { "next-action": "7f3a9c", "user-agent": "scanner" }), + ); + + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ + data: null, + error: "Not found", + meta: { errorCode: "server_action.unsupported" }, + }); + expect(response.headers.get("X-Content-Type-Options")).toBe("nosniff"); + expect(warn).toHaveBeenCalledTimes(1); + expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({ + level: "warn", + action: { name: "proxy.server_action.refused" }, + http: { method: "POST", path: "/", status: 404, user_agent: "scanner" }, + }); + }); + + it("never logs a secret path segment the probe aimed at", () => { + proxy(probe("/api/withings/webhook/s3cr3t-value", { "next-action": "x" })); + + const line = String(warn.mock.calls[0]![0]); + expect(line).not.toContain("s3cr3t-value"); + }); + + it("leaves a request without the header alone", () => { + const response = proxy(probe("/api/health")); + + expect(response.status).not.toBe(404); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/src/proxy.ts b/src/proxy.ts index 1d54b9f39..d2f8e2e81 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -8,6 +8,7 @@ import { import { setBaselineSecurityHeaders } from "@/lib/http/proxy-bypass-routes"; import { DEMO_REFUSAL_BODY } from "@/lib/demo-refusal"; import { resolveRequestId } from "@/lib/http/request-id"; +import { redactSecrets } from "@/lib/logging/redact"; /** * Paths that do NOT require a session cookie (public pages + external webhooks). @@ -321,6 +322,9 @@ function applyBaselineSecurityHeaders(response: NextResponse): NextResponse { */ const NOTIFICATION_ACTION_PREFIX = "/api/notifications/action/"; +/** The request header that marks a Next.js Server Action call. */ +const SERVER_ACTION_HEADER = "next-action"; + type NotificationActionRefusal = "method_not_allowed" | "worker_only" | "demo_mode"; @@ -371,6 +375,42 @@ export function proxy(request: NextRequest) { ); } + // A Server Action call. HealthLog defines no Server Actions (no module + // carries `"use server"`, which `no-server-actions-guard.test.ts` holds), so + // no bundle of this app, old or new, ever sends `Next-Action`: the requests + // that do are scanners probing the Server Action surface, nightly and in + // bursts. Left to Next, each one printed "Failed to find Server Action" at + // error level, unstructured, an error with no user behind it. Refused here + // instead, with one countable `warn` line: a refusal by design. + if (request.headers.has(SERVER_ACTION_HEADER)) { + console.warn( + JSON.stringify({ + timestamp: new Date().toISOString(), + level: "warn", + kind: "http", + message: "Server Action call refused: this app defines none.", + action: { name: "proxy.server_action.refused" }, + http: { + method: request.method, + // A probe may aim at a path that carries a secret segment. + path: redactSecrets(pathname), + status: 404, + user_agent: request.headers.get("user-agent") ?? undefined, + }, + }), + ); + return applyBaselineSecurityHeaders( + NextResponse.json( + { + data: null, + error: "Not found", + meta: { errorCode: "server_action.unsupported" }, + }, + { status: 404 }, + ), + ); + } + // 301 redirects for renamed routes const redirect = LEGACY_REDIRECTS[pathname]; if (redirect) { From 63a3e354c8c9e5a2f987d769587a4ae6a60c8638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:46:34 +0200 Subject: [PATCH 13/23] perf(boot): load route modules on first request, not all at start Next preloads every route and page module right after the server starts, about 670 here. The load runs on the event loop, so every container start showed one stall of several seconds, the first requests waited behind it, and the heap stood at about 430 MB before the first sign-in, most of it code for routes an instance rarely serves. With preloadEntriesOnStart off, each module loads on its route's first request. The same image started locally at 185 MB of heap instead of 428, with no stall, and a route's first request paid 30 to 60 ms once. --- next.config.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/next.config.ts b/next.config.ts index 6658934f0..0af3eb7aa 100644 --- a/next.config.ts +++ b/next.config.ts @@ -329,9 +329,20 @@ const nextConfig: NextConfig = { // their own body after authenticating (`src/lib/http/proxy-bypass-routes.ts`), // and every other route's own cap is at or below this. Large exports still // want a matching reverse-proxy body limit (docs/self-hosting/reverse-proxy.md). + // + // `preloadEntriesOnStart` is off. Next turns it on by default and then + // requires every route and page module (about 670 here) right after the + // server starts. That load runs on the event loop: a stall of several + // seconds in the first minute of every container, the first requests + // queued behind it, and a heap of about 430 MB before anyone has signed in, + // most of it code for routes a given instance rarely or never serves. Off, + // each module loads on its route's first request, which costs that one + // request a few tens of milliseconds; the same image measured locally + // started at 185 MB of heap instead of 428 and without the stall. experimental: { optimizePackageImports: ["recharts", "lucide-react"], proxyClientMaxBodySize: "1mb", + preloadEntriesOnStart: false, }, }; From 652dd6943b8d3e14b2db88409366814b031adab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:56:08 +0200 Subject: [PATCH 14/23] test(auth): read the password hash off the stored row in proof tests The proof gates now read whether an account has a password from its own row rather than from the per-request user, so the suites that drive them mock that read instead of carrying the hash on the session user. --- .../grants/__tests__/invite-recent-proof.test.ts | 7 +++++-- .../api/dashboard/snapshot/__tests__/route.test.ts | 4 ++-- .../fhir/__tests__/capability-and-everything.test.ts | 2 ++ src/app/api/mcp/oauth/__tests__/authorize.test.ts | 2 ++ src/app/api/mcp/tokens/__tests__/route.test.ts | 2 ++ src/app/api/share-links/__tests__/route.test.ts | 2 ++ src/lib/__tests__/require-fresh-mfa.test.ts | 11 ++++++++++- src/lib/__tests__/require-recent-proof.test.ts | 6 +++++- 8 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/app/api/account/grants/__tests__/invite-recent-proof.test.ts b/src/app/api/account/grants/__tests__/invite-recent-proof.test.ts index e4c987074..f450f87e6 100644 --- a/src/app/api/account/grants/__tests__/invite-recent-proof.test.ts +++ b/src/app/api/account/grants/__tests__/invite-recent-proof.test.ts @@ -14,7 +14,11 @@ vi.mock("@/lib/db", () => ({ session: { findUnique: vi.fn() }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, - user: { findFirst: vi.fn() }, + user: { + findFirst: vi.fn(), + // The stored row: the password hash is not on the session user. + findUnique: vi.fn(async () => ({ passwordHash: "argon" })), + }, accountGrant: { create: vi.fn(), findFirst: vi.fn() }, }, })); @@ -50,7 +54,6 @@ const OWNER = { role: "USER", username: "owner", totpConfirmedAt: null, - passwordHash: "argon", }; function invite(): NextRequest { diff --git a/src/app/api/dashboard/snapshot/__tests__/route.test.ts b/src/app/api/dashboard/snapshot/__tests__/route.test.ts index a2226125c..e161146af 100644 --- a/src/app/api/dashboard/snapshot/__tests__/route.test.ts +++ b/src/app/api/dashboard/snapshot/__tests__/route.test.ts @@ -17,7 +17,8 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; vi.mock("@/lib/db", () => ({ - prisma: {}, + // The cached briefing text is read off its own row, not the session user. + prisma: { user: { findUnique: async () => null } }, })); vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn(), @@ -71,7 +72,6 @@ const SESSION_OK = { glucoseUnit: "mg/dL", onboardingTourCompleted: true, disableCoach: false, - insightsCachedText: null, insightsCachedAt: null, dashboardWidgetsJson: null, }, diff --git a/src/app/api/fhir/__tests__/capability-and-everything.test.ts b/src/app/api/fhir/__tests__/capability-and-everything.test.ts index c402cf539..d3709b9dd 100644 --- a/src/app/api/fhir/__tests__/capability-and-everything.test.ts +++ b/src/app/api/fhir/__tests__/capability-and-everything.test.ts @@ -28,6 +28,8 @@ vi.mock("@/lib/db", () => ({ }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, + // The password hash is read off the stored row; none here. + user: { findUnique: vi.fn(async () => null) }, }, })); vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: vi.fn() })); diff --git a/src/app/api/mcp/oauth/__tests__/authorize.test.ts b/src/app/api/mcp/oauth/__tests__/authorize.test.ts index 164da251e..24f12bb88 100644 --- a/src/app/api/mcp/oauth/__tests__/authorize.test.ts +++ b/src/app/api/mcp/oauth/__tests__/authorize.test.ts @@ -35,6 +35,8 @@ vi.mock("@/lib/db", () => ({ }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, + // The password hash is read off the stored row; none here. + user: { findUnique: vi.fn(async () => null) }, }, })); vi.mock("@/lib/app-settings", () => ({ diff --git a/src/app/api/mcp/tokens/__tests__/route.test.ts b/src/app/api/mcp/tokens/__tests__/route.test.ts index e389130e3..98c3e136e 100644 --- a/src/app/api/mcp/tokens/__tests__/route.test.ts +++ b/src/app/api/mcp/tokens/__tests__/route.test.ts @@ -19,6 +19,8 @@ vi.mock("@/lib/db", () => ({ }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, + // The password hash is read off the stored row; none here. + user: { findUnique: vi.fn(async () => null) }, }, })); vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); diff --git a/src/app/api/share-links/__tests__/route.test.ts b/src/app/api/share-links/__tests__/route.test.ts index 33509c3c9..bc5e3ad24 100644 --- a/src/app/api/share-links/__tests__/route.test.ts +++ b/src/app/api/share-links/__tests__/route.test.ts @@ -28,6 +28,8 @@ vi.mock("@/lib/db", () => ({ }, webauthnMfaCredential: { count: vi.fn(async () => 0) }, passkey: { count: vi.fn(async () => 0) }, + // The password hash is read off the stored row; none here. + user: { findUnique: vi.fn(async () => null) }, }, })); diff --git a/src/lib/__tests__/require-fresh-mfa.test.ts b/src/lib/__tests__/require-fresh-mfa.test.ts index 28104c009..90001aea4 100644 --- a/src/lib/__tests__/require-fresh-mfa.test.ts +++ b/src/lib/__tests__/require-fresh-mfa.test.ts @@ -5,6 +5,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; vi.mock("@/lib/db", () => ({ prisma: { session: { findUnique: vi.fn() }, + user: { findUnique: vi.fn() }, webauthnMfaCredential: { count: vi.fn() }, passkey: { count: vi.fn() }, }, @@ -50,7 +51,15 @@ beforeEach(() => { vi.resetAllMocks(); }); -function mockSession(user: unknown) { +function mockSession(user: { + id: string; + totpConfirmedAt: Date | null; + passwordHash?: string | null; +}) { + // The password hash is read off the stored row, not the session user. + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + passwordHash: user.passwordHash ?? null, + } as never); vi.mocked(getSession).mockResolvedValue({ session: { id: "sess-1", expiresAt: new Date(Date.now() + 1e6) }, user, diff --git a/src/lib/__tests__/require-recent-proof.test.ts b/src/lib/__tests__/require-recent-proof.test.ts index 8c0d09fd1..136797db7 100644 --- a/src/lib/__tests__/require-recent-proof.test.ts +++ b/src/lib/__tests__/require-recent-proof.test.ts @@ -72,13 +72,17 @@ beforeEach(() => { }); function cookie( - user: unknown, + user: { id: string; passwordHash?: string | null }, stamps: { mfaVerifiedAt?: Date | null; createdAt?: Date; reproofAt?: Date | null; }, ) { + // The password hash is read off the stored row, not the session user. + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + passwordHash: user.passwordHash ?? null, + } as never); vi.mocked(getSession).mockResolvedValue({ session: { id: "sess-1", expiresAt: new Date(Date.now() + 1e6) }, user, From 52c6c8dc9bb6c158abd9b4f87e4a8d355b643607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 21:08:16 +0200 Subject: [PATCH 15/23] fix(demo): keep the demo dashboard to the measurement charts The shared demo account carries onboarding answers, so the setup checklist stayed on its dashboard for every visitor although none of its rows can be done there. Under DEMO_MODE the dashboard leaves it out, as the shell already does with the tour. The demo seed hides the mood and medication cards, achievements, recent workouts and check-ups, opens weight, BMI and blood pressure on 90 days, and resting heart rate on 30 with its 7-day trend and trend line. --- scripts/seed-demo.ts | 31 ++++++++++++++-- .../dashboard-demo-checklist.test.ts | 35 +++++++++++++++++++ src/app/page-client.tsx | 9 ++++- src/app/page.tsx | 11 ++++-- 4 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 src/app/__tests__/dashboard-demo-checklist.test.ts diff --git a/scripts/seed-demo.ts b/scripts/seed-demo.ts index 57abb04a7..aad6f50f2 100644 --- a/scripts/seed-demo.ts +++ b/scripts/seed-demo.ts @@ -90,6 +90,7 @@ import { import { ENVIRONMENT_LOCATION_AAD } from "../src/lib/crypto/encrypted-columns"; import { sealInsightsCachedText } from "../src/lib/insights/cached-text-cipher"; import { + DEFAULT_CHART_OVERLAY_PREFS, DEFAULT_DASHBOARD_LAYOUT, serializeDashboardLayout, } from "../src/lib/dashboard-layout"; @@ -497,16 +498,40 @@ async function seed() { // One row of six tiles (weight, the two blood-pressure values, resting // heart rate, HRV and mood) and the weight chart right below it, so the // first screen of the public demo shows the summary, the tiles and the - // start of a chart together. Every other tile stays one switch away in - // the layout settings; its data is seeded either way. Writing the layout - // verbatim keeps the demo's home page stable across default tweaks. + // start of a chart together. Below it only the measurement charts: the + // mood and medication cards, achievements, recent workouts and check-ups + // stay one switch away in the layout settings, their data seeded either + // way. Weight, BMI and blood pressure open on 90 days, resting heart rate + // on 30 with its 7-day trend and trend line, so each chart shows a shape + // rather than a flat week. Writing the layout verbatim keeps the demo's + // home page stable across default tweaks. const SHOWCASE_TILE_IDS = new Set(["weight", "bp", "pulse", "hrv", "mood"]); + const HIDDEN_CARD_IDS = new Set([ + "mood", + "medications", + "achievements", + "recentWorkouts", + "vorsorge", + ]); const demoDashboardLayout = serializeDashboardLayout({ ...DEFAULT_DASHBOARD_LAYOUT, widgets: DEFAULT_DASHBOARD_LAYOUT.widgets.map((w) => ({ ...w, + visible: w.visible && !HIDDEN_CARD_IDS.has(w.id), tileVisible: SHOWCASE_TILE_IDS.has(w.id), })), + chartOverlayPrefs: { + weight: { ...DEFAULT_CHART_OVERLAY_PREFS, rangePoints: 90 }, + bmi: { ...DEFAULT_CHART_OVERLAY_PREFS, rangePoints: 90 }, + bp: { ...DEFAULT_CHART_OVERLAY_PREFS, rangePoints: 90 }, + // The resting heart rate chart is the dashboard's `pulse` chart. + pulse: { + ...DEFAULT_CHART_OVERLAY_PREFS, + rangePoints: 30, + showTrendIndicator: true, + showTrendArrow: true, + }, + }, }); await client.query( `UPDATE users SET dashboard_widgets_json = $1 WHERE id = $2`, diff --git a/src/app/__tests__/dashboard-demo-checklist.test.ts b/src/app/__tests__/dashboard-demo-checklist.test.ts new file mode 100644 index 000000000..0e7f0b50f --- /dev/null +++ b/src/app/__tests__/dashboard-demo-checklist.test.ts @@ -0,0 +1,35 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +/** + * The setup checklist on a demo instance. + * + * The demo is one shared, read-only account carrying onboarding answers, so + * `isStillInSetup` keeps the checklist on screen for every visitor although + * none of its rows can be done there. The shell already leaves out the tour + * under `DEMO_MODE`; the dashboard leaves out the checklist the same way. + * Pinned here: every render path of the page threads the flag, and the + * client mounts the checklist only without it. + */ +const PAGE_PATH = join(process.cwd(), "src/app/page.tsx"); +const CLIENT_PATH = join(process.cwd(), "src/app/page-client.tsx"); + +describe("dashboard setup checklist on a demo instance", () => { + it("threads DEMO_MODE into every render of the dashboard client", () => { + const src = readFileSync(PAGE_PATH, "utf8"); + expect(src).toContain('process.env.DEMO_MODE === "true"'); + const renders = src.match(/]*>/g) ?? []; + expect(renders.length).toBeGreaterThan(0); + for (const render of renders) { + expect(render).toContain("demoMode={demoMode}"); + } + }); + + it("mounts the checklist only outside demo mode", () => { + const src = readFileSync(CLIENT_PATH, "utf8"); + const mounts = src.match(/]*\/>/g) ?? []; + expect(mounts).toHaveLength(1); + expect(src).toContain("{demoMode ? null : }"); + }); +}); diff --git a/src/app/page-client.tsx b/src/app/page-client.tsx index cb3aab93e..b72534282 100644 --- a/src/app/page-client.tsx +++ b/src/app/page-client.tsx @@ -193,6 +193,7 @@ import { export default function DashboardPageClient({ batchWindow: batchWindowProp, + demoMode = false, }: { /** * v1.30.9 — the batched-series ISO window computed server-side from the @@ -204,6 +205,12 @@ export default function DashboardPageClient({ * under, so the prefetched slice lands instead of triggering a refetch. */ batchWindow?: BatchWindow; + /** + * True on a `DEMO_MODE` instance. The demo is one shared, read-only + * account a visitor cannot set up, so the setup checklist has nothing to + * offer there, as the shell already leaves out the tour. + */ + demoMode?: boolean; } = {}) { const { isAuthenticated } = useAuth(); const mounted = useMounted(); @@ -2312,7 +2319,7 @@ export default function DashboardPageClient({ * content (tiles, charts) keeps a stable position, and the * near-empty dashboards of the new accounts the card targets keep * it visible without scrolling anyway. */} - + {demoMode ? null : } ); } diff --git a/src/app/page.tsx b/src/app/page.tsx index 493c3f27e..48af2ec52 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -76,12 +76,15 @@ const DIGEST_SOFT_BUDGET_MS = 400; * others. */ export default async function DashboardPage() { + // `DEMO_MODE` is a server-only env var; resolved here and threaded in, as + // the layout does for the shell. + const demoMode = process.env.DEMO_MODE === "true"; // Operator/test escape hatch: `DASHBOARD_SSR_PREFETCH=false` renders the // pure client-fetch dashboard (pre-prefetch behaviour). The e2e server // sets it so Playwright route mocks — which only see CLIENT fetches — // keep governing what the dashboard paints. It gates EVERY prefetch below. if (process.env.DASHBOARD_SSR_PREFETCH === "false") { - return ; + return ; } let dehydratedState = null; let batchWindow: BatchWindow | undefined; @@ -171,11 +174,13 @@ export default async function DashboardPage() { } if (dehydratedState === null) { - return ; + return ( + + ); } return ( - + ); } From 7bbd7c38116ac8a949f020e1212cbed25f0a4c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 22:48:24 +0200 Subject: [PATCH 16/23] fix(insights): read the all-time extremes from the tier only where it matches The DAY tier is not recomputed on every path that removes readings: a hard delete with no rollup recompute leaves a bucket counting readings that are gone, and the extremes then repeated it. Those figures are what the Coach snapshot checks its coarse history against, so they must stay true to the live record. A type is now folded from the tier only when the tier accounts for the live record day by day: the same days, and per day the same reading count. The live side is a count per day served by an index-only scan of the live index, so the check reads no measurement rows. A type that fails it is read raw, as before. --- src/lib/insights/feature-blocks.ts | 51 +++++++++++++++---- .../all-time-extremes-tier-parity.test.ts | 23 +++++++++ 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/src/lib/insights/feature-blocks.ts b/src/lib/insights/feature-blocks.ts index 70024c404..8e5af6078 100644 --- a/src/lib/insights/feature-blocks.ts +++ b/src/lib/insights/feature-blocks.ts @@ -66,10 +66,11 @@ interface AllTimeExtremes { * * - the DAY tier, for every whole UTC day from `tierFrom` on, for each type * whose tier rows in that range all carry the reading extremes - * (migration 0417; the boot re-fold fills older rows); + * (migration 0417; the boot re-fold fills older rows) and account for + * the live record day by day (same days, same reading counts); * - the raw rows for the rest: the days before `tierFrom` (older than the * fold window, or the partial day the `floor` falls in), and every day of - * a type the tier does not cover yet. + * a type the tier does not cover yet or no longer matches. * * The split is at a UTC midnight, so no day is folded from both, and the * canonical pick per day is the same on both sides. The tier's pulse day @@ -104,17 +105,45 @@ export async function readAllTimeExtremes( const [priorityJson, tierRows] = await Promise.all([ loadUserSourcePriority(userId), + // A type is read from the tier only when the tier accounts for the live + // record day by day: every day with a live reading has DAY rows whose + // counts add up to that day's live count, and every DAY row has its day + // of live readings. A bucket that outlived its rows (a hard delete with no + // rollup recompute) or a day the tier never got fails the comparison and + // the type is read raw. The figures from here are the anchor the Coach + // snapshot checks its coarse history against, so they must never repeat + // a stale bucket. The live side is a count per day off the partial + // `(user_id, type, measured_at)` live index, not a read of the rows. prisma.$queryRaw>` - SELECT r."type"::text AS type - FROM measurement_rollups r - WHERE r."user_id" = ${userId} - AND r."granularity" = 'DAY' - AND r."type" IN (${typeList(types)}) - AND r."bucket_start" >= ${tierFrom} - GROUP BY r."type" - HAVING bool_and( - r."reading_min_value" IS NOT NULL AND r."reading_max_value" IS NOT NULL + WITH live AS ( + SELECT m."type", date_trunc('day', m."measured_at") AS day, + COUNT(*)::bigint AS n + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList(types)}) + AND m."measured_at" >= ${tierFrom} + GROUP BY 1, 2 + ), + tier AS ( + SELECT r."type", r."bucket_start" AT TIME ZONE 'UTC' AS day, + SUM(r."count")::bigint AS n, + bool_and( + r."reading_min_value" IS NOT NULL + AND r."reading_max_value" IS NOT NULL + ) AS complete + FROM measurement_rollups r + WHERE r."user_id" = ${userId} + AND r."granularity" = 'DAY' + AND r."type" IN (${typeList(types)}) + AND r."bucket_start" >= ${tierFrom} + GROUP BY 1, 2 ) + SELECT COALESCE(l."type", t."type")::text AS type + FROM live l + FULL JOIN tier t ON t."type" = l."type" AND t.day = l.day + GROUP BY 1 + HAVING bool_and(COALESCE(l.n = t.n AND t.complete, false)) `, ]); const tierTypes = new Set(tierRows.map((r) => r.type)); diff --git a/tests/integration/all-time-extremes-tier-parity.test.ts b/tests/integration/all-time-extremes-tier-parity.test.ts index 74c16102a..c73ad99b0 100644 --- a/tests/integration/all-time-extremes-tier-parity.test.ts +++ b/tests/integration/all-time-extremes-tier-parity.test.ts @@ -318,4 +318,27 @@ describe("readAllTimeExtremes over the DAY tier", () => { expect(actual.get("WEIGHT")?.max).toBe(reference.get("WEIGHT")?.max); expect(actual.get("WEIGHT")?.min).toBe(reference.get("WEIGHT")?.min); }); + + it("reads the raw rows for a type whose tier outlived a hard delete", async () => { + // Rebuild a clean tier, then delete the hottest pulse reading with no + // rollup recompute: its DAY row now counts a reading that is gone. + await recomputeUserRollups(USER, { granularities: ["DAY"] }); + const hottest = await prisma.measurement.findFirst({ + where: { userId: USER, type: "PULSE", deletedAt: null }, + orderBy: { value: "desc" }, + }); + await prisma.measurement.delete({ where: { id: hottest!.id } }); + const stale = await prisma.measurementRollup.findMany({ + where: { + userId: USER, + type: "PULSE", + granularity: "DAY", + readingMaxValue: hottest!.value, + }, + }); + expect(stale.length).toBeGreaterThan(0); + await expectParity(null); + const out = await readAllTimeExtremes(USER, ["PULSE"]); + expect(out.get("PULSE")?.max).toBeLessThan(hottest!.value); + }); }); From 23bc58660d33d90ba7e6e031e59c3f1de8b3bc57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 00:03:08 +0200 Subject: [PATCH 17/23] test(integration): keep two cases independent of file order and random ids The backup schedule case assumed the shared database had no queue schema, but files that drive the queue create it and one leaves a finished backup run behind, so the case failed whenever it ran after them. It now clears that queue's history first. The delegate refusal case scanned the raw response for a digit run that the random request id can contain; the id is taken out before the scan. --- tests/integration/admin-backups-audit.test.ts | 16 +++++++++++++--- tests/integration/mood-context-linkage.test.ts | 10 ++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/integration/admin-backups-audit.test.ts b/tests/integration/admin-backups-audit.test.ts index 1417525e0..0a68fd32a 100644 --- a/tests/integration/admin-backups-audit.test.ts +++ b/tests/integration/admin-backups-audit.test.ts @@ -225,6 +225,16 @@ describe("GET /api/admin/backups — schedule health", () => { data: { userId: admin.id, type: "MANUAL_UPLOAD_1", data: blob }, }); + // Files that drive the queue create the `pgboss` schema in the shared + // database, and one of them leaves a finished backup run behind. Clear + // that queue's history so this case reads the same in every file order. + const [queueTable] = await prisma.$queryRaw>` + SELECT to_regclass('pgboss.job')::text AS t + `; + if (queueTable?.t) { + await prisma.$executeRaw`DELETE FROM pgboss.job WHERE name = 'data-backup'`; + } + // The listing handler declares no parameters; `apiHandler` reads the // request defensively out of its arguments, so calling it bare is what a // zero-argument handler supports. @@ -255,9 +265,9 @@ describe("GET /api/admin/backups — schedule health", () => { expect(body.data.schedule.lastSuccessAgeDays).toBe(46); expect(body.data.schedule.stale).toBe(true); - // No `pgboss` schema in the test database — the run reader fails soft to - // null rather than taking the whole page down, which is the arm a web-only - // deployment lives on permanently. + // No remembered run, or no `pgboss` schema at all — the run reader fails + // soft to null rather than taking the whole page down, which is the arm a + // web-only deployment lives on permanently. expect(body.data.schedule.lastRun).toBeNull(); expect(body.data.schedule.lastRunFailed).toBe(false); }); diff --git a/tests/integration/mood-context-linkage.test.ts b/tests/integration/mood-context-linkage.test.ts index 13db69e77..e3230ec3f 100644 --- a/tests/integration/mood-context-linkage.test.ts +++ b/tests/integration/mood-context-linkage.test.ts @@ -524,10 +524,12 @@ describe("mood context linkage (real Postgres)", () => { expect(res.status).toBe(403); const raw = await res.text(); // Asserted on the bytes, not on a parsed field: the point is that not one - // of the owner's figures reached the response. - expect(raw).not.toContain("7777"); - expect(raw).not.toContain("55"); - expect(raw).not.toContain("functionalImpact"); + // of the owner's figures reached the response. The request id is random + // and can contain any digit run, so it is taken out before the check. + const bytes = raw.replace(/"requestId":"[^"]*"/, ""); + expect(bytes).not.toContain("7777"); + expect(bytes).not.toContain("55"); + expect(bytes).not.toContain("functionalImpact"); }); it("refuses a malformed day", async () => { From 62244096788b163b110460a7ec4d9881238c7dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 00:05:22 +0200 Subject: [PATCH 18/23] fix(oura): request the stress scope and exchange a repeated callback once Resilience sits behind Oura's `stress` scope, not `heart_health`, so a connection that granted every scope HealthLog asked for still answered 401 on `daily_resilience`. `src/lib/oura/scope.ts` now maps every collection the sync reads to the scope the live API enforces and is the one source for the authorize request, the per-collection permission detection, the reconnect hint and the settings card. The request asks for `stress` as well. Oura names a missing scope in its 401 body ("Token is not authorized access stress scope."), while a dead token reads differently. A collection refusal now carries that scope: it spends no token refresh, never counts toward the reconnect threshold and marks only the collections it names. A 403 (a lapsed membership) is a persistent failure rather than a reconnect. The callback also records what the token response says was granted, so an unticked permission shows on the card before the first sync. A browser can deliver an OAuth callback twice with the same code. Both requests exchanged it, and Oura answers a reused code by revoking the token the first exchange issued. Every connect callback (Oura, Polar, Strava, WHOOP, Withings, Fitbit, Google Health) now claims its state in `oauth_callback_claims` before the exchange: one request exchanges, the duplicate waits for it and redirects to the same page without touching any token. The ledger flows claim inside the transaction that consumes their state row. A failed exchange settles with its error page, and a new connect mints a new state. Oura token refreshes are serialised per user with an advisory lock, so overlapping syncs no longer spend the same single-use refresh token, and the write stays conditional so a reconnect's new grant is never overwritten by a refresh of the old one. --- .env.production.example | 8 +- docs/api/openapi.yaml | 5 +- messages/de.json | 2 +- messages/en.json | 2 +- messages/es.json | 2 +- messages/fr.json | 2 +- messages/it.json | 2 +- messages/ko.json | 2 +- messages/nl.json | 2 +- messages/pl.json | 2 +- .../0419_oauth_callback_claims/migration.sql | 28 ++ prisma/schema.prisma | 28 ++ src/app/api/fitbit/callback/route.ts | 214 +++++++------ src/app/api/google-health/callback/route.ts | 234 ++++++++------- .../api/oura/callback/__tests__/route.test.ts | 20 ++ src/app/api/oura/callback/route.ts | 54 +++- .../polar/callback/__tests__/route.test.ts | 20 ++ src/app/api/polar/callback/route.ts | 29 +- .../strava/callback/__tests__/route.test.ts | 20 ++ src/app/api/strava/callback/route.ts | 29 +- .../whoop/callback/__tests__/route.test.ts | 22 +- src/app/api/whoop/callback/route.ts | 216 +++++++------ .../withings/callback/__tests__/route.test.ts | 22 +- src/app/api/withings/callback/route.ts | 230 +++++++------- .../__tests__/recovery-empty-state.test.tsx | 2 +- src/lib/data-wipe/wipe-plan.ts | 2 + src/lib/insights/recovery-empty.ts | 7 +- src/lib/integrations/oauth-refresh.ts | 2 +- src/lib/oauth/callback-claim.ts | 207 +++++++++++++ src/lib/oauth/signed-state.ts | 2 +- src/lib/openapi/routes/integrations.ts | 2 +- src/lib/oura/__tests__/client.test.ts | 47 ++- src/lib/oura/__tests__/scope.test.ts | 101 +++++++ src/lib/oura/__tests__/sync.test.ts | 163 +++++++++- src/lib/oura/client.ts | 74 +++-- src/lib/oura/credentials.ts | 118 +++++--- src/lib/oura/response-classifier.ts | 22 +- src/lib/oura/scope.ts | 165 +++++++++- src/lib/oura/sync.ts | 184 +++++++----- .../oauth-callback-single-use.test.ts | 283 ++++++++++++++++++ .../oura-refresh-serialization.test.ts | 156 ++++++++++ tests/integration/setup.ts | 3 + tests/integration/withings-oauth-flow.test.ts | 43 +-- 43 files changed, 2154 insertions(+), 624 deletions(-) create mode 100644 prisma/migrations/0419_oauth_callback_claims/migration.sql create mode 100644 src/lib/oauth/callback-claim.ts create mode 100644 src/lib/oura/__tests__/scope.test.ts create mode 100644 tests/integration/oauth-callback-single-use.test.ts create mode 100644 tests/integration/oura-refresh-serialization.test.ts diff --git a/.env.production.example b/.env.production.example index 2d2884592..0428201a7 100644 --- a/.env.production.example +++ b/.env.production.example @@ -153,9 +153,11 @@ API_TOKEN_HMAC_KEY="" # Oura Cloud v2 -- optional integration (full OAuth, shared app from env) # ----------------------------------------------------------------------------- # Register an app at https://cloud.ouraring.com/oauth/applications and set its -# redirect URI to `${NEXT_PUBLIC_APP_URL}/api/oura/callback`. Both vars together -# enable the Oura connect button. Oura uses refresh tokens; the sync refreshes -# reactively on a 401. OURA_REDIRECT_URI overrides the derived callback. +# redirect URI to `${NEXT_PUBLIC_APP_URL}/api/oura/callback`. Enable the Daily, +# Personal, SpO2, Heart health and Stress permissions on it; HealthLog asks for +# exactly those. Both vars together enable the Oura connect button. Oura uses +# refresh tokens; the sync refreshes reactively on a 401. OURA_REDIRECT_URI +# overrides the derived callback. # OURA_CLIENT_ID="" # OURA_CLIENT_SECRET="" # OURA_REDIRECT_URI="" diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index c17830d99..0ce16651c 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -79303,8 +79303,9 @@ components: type: boolean scopeReconnect: deprecated: true - description: "Deprecated: removed in v1.44, read `health`. Oura only. Collections the stored grant does not cover: they - answered 401 while sibling collections succeeded with the same token. Not a failure and not counted toward + description: "Deprecated: removed in v1.44, read `health`. Oura only. Collections the stored grant does not cover: Oura + refused them for a missing scope (named in its 401, or inferred when sibling collections succeeded with the + same token), or the scopes Oura reported at connect leave theirs out. Not a failure and not counted toward the reconnect threshold; one reconnect asks for the missing scopes. Empty when the grant covers everything HealthLog reads." type: array diff --git a/messages/de.json b/messages/de.json index 030c254cc..9a2c80392 100644 --- a/messages/de.json +++ b/messages/de.json @@ -4657,7 +4657,7 @@ "scoreCrossLink": "Den zusammengesetzten Erholungs-Score ansehen", "emptyTitle": "Noch keine Erholungsdaten", "empty": "Diese Seite füllt sich mit den Erholungs-, Belastungs- und Trainingslastwerten von WHOOP oder Polar und mit Ouras Resilienz-Stufe.", - "emptyOuraPermission": "Ouras Resilienz-Stufe braucht Zugriff auf Herzgesundheitsdaten, also verbinde Oura einmal neu unter Integrationen und erlaube ihn.", + "emptyOuraPermission": "Ouras Resilienz-Stufe braucht die Berechtigung für Stressdaten, also verbinde Oura einmal neu unter Integrationen und erlaube sie.", "emptyOuraNoResilience": "Oura ist verbunden, und seine Resilienz-Stufe erscheint hier, sobald Oura eine liefert.", "emptyAction": "Integrationen öffnen", "block": { diff --git a/messages/en.json b/messages/en.json index a486f636a..8d6ab2cd4 100644 --- a/messages/en.json +++ b/messages/en.json @@ -4657,7 +4657,7 @@ "scoreCrossLink": "See the composite recovery score", "emptyTitle": "No recovery data yet", "empty": "This page fills with the recharge, strain and load readings from WHOOP or Polar and with Oura's resilience level.", - "emptyOuraPermission": "Oura's resilience level needs heart-health access, so reconnect Oura once under Integrations and allow it.", + "emptyOuraPermission": "Oura's resilience level needs the stress permission, so reconnect Oura once under Integrations and allow it.", "emptyOuraNoResilience": "Oura is connected, and its resilience level appears here as soon as Oura provides one.", "emptyAction": "Open integrations", "block": { diff --git a/messages/es.json b/messages/es.json index a154ba9bd..f86b42a52 100644 --- a/messages/es.json +++ b/messages/es.json @@ -4657,7 +4657,7 @@ "scoreCrossLink": "Ver la puntuación de recuperación combinada", "emptyTitle": "Aún no hay datos de recuperación", "empty": "Esta página se llena con los valores de recuperación, esfuerzo y carga de WHOOP o Polar y con el nivel de resiliencia de Oura.", - "emptyOuraPermission": "El nivel de resiliencia de Oura necesita acceso a los datos de salud cardíaca, así que vuelve a conectar Oura una vez en Integraciones y permítelo.", + "emptyOuraPermission": "El nivel de resiliencia de Oura necesita el permiso de datos de estrés, así que vuelve a conectar Oura una vez en Integraciones y permítelo.", "emptyOuraNoResilience": "Oura está conectado y su nivel de resiliencia aparecerá aquí en cuanto Oura proporcione uno.", "emptyAction": "Abrir integraciones", "block": { diff --git a/messages/fr.json b/messages/fr.json index 0807d9df1..3b3a92f2f 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -4657,7 +4657,7 @@ "scoreCrossLink": "Voir le score de récupération composite", "emptyTitle": "Pas encore de données de récupération", "empty": "Cette page se remplit avec les valeurs de récupération, d'effort et de charge de WHOOP ou Polar et avec le niveau de résilience d'Oura.", - "emptyOuraPermission": "Le niveau de résilience d'Oura demande l'accès aux données de santé cardiaque : reconnectez Oura une fois dans Intégrations et autorisez-le.", + "emptyOuraPermission": "Le niveau de résilience d'Oura demande l'autorisation d'accès aux données de stress : reconnectez Oura une fois dans Intégrations et autorisez-la.", "emptyOuraNoResilience": "Oura est connecté, et son niveau de résilience apparaîtra ici dès qu'Oura en fournira un.", "emptyAction": "Ouvrir les intégrations", "block": { diff --git a/messages/it.json b/messages/it.json index 52629264f..85e7d2cf8 100644 --- a/messages/it.json +++ b/messages/it.json @@ -4657,7 +4657,7 @@ "scoreCrossLink": "Vedi il punteggio di recupero composito", "emptyTitle": "Ancora nessun dato di recupero", "empty": "Questa pagina si riempie con i valori di recupero, sforzo e carico di WHOOP o Polar e con il livello di resilienza di Oura.", - "emptyOuraPermission": "Il livello di resilienza di Oura richiede l'accesso ai dati sulla salute del cuore, quindi ricollega Oura una volta in Integrazioni e consentilo.", + "emptyOuraPermission": "Il livello di resilienza di Oura richiede l'autorizzazione ai dati sullo stress, quindi ricollega Oura una volta in Integrazioni e consentila.", "emptyOuraNoResilience": "Oura è collegato e il suo livello di resilienza apparirà qui non appena Oura ne fornirà uno.", "emptyAction": "Apri integrazioni", "block": { diff --git a/messages/ko.json b/messages/ko.json index 3b245cf71..cc848901c 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -4657,7 +4657,7 @@ "scoreCrossLink": "종합 회복 점수 보기", "emptyTitle": "아직 회복 데이터가 없어요", "empty": "이 페이지에는 WHOOP 또는 Polar의 회복, 부하, 훈련 부하 값과 Oura의 회복탄력성 수준이 표시돼요.", - "emptyOuraPermission": "Oura의 회복탄력성 수준에는 심장 건강 데이터 접근 권한이 필요하니, 연동에서 Oura를 한 번 다시 연결하고 허용해 주세요.", + "emptyOuraPermission": "Oura의 회복탄력성 수준에는 스트레스 데이터 권한이 필요하니, 연동에서 Oura를 한 번 다시 연결하고 허용해 주세요.", "emptyOuraNoResilience": "Oura가 연결되어 있으며, Oura가 회복탄력성 수준을 제공하는 즉시 여기에 표시돼요.", "emptyAction": "통합 열기", "block": { diff --git a/messages/nl.json b/messages/nl.json index 6732ea4dd..745094620 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -4668,7 +4668,7 @@ }, "emptyAction": "Integraties openen", "emptyOuraNoResilience": "Oura is verbonden, en het veerkrachtniveau verschijnt hier zodra Oura er een levert.", - "emptyOuraPermission": "Het veerkrachtniveau van Oura heeft toegang tot hartgezondheid nodig, dus verbind Oura één keer opnieuw onder Integraties en sta dat toe.", + "emptyOuraPermission": "Het veerkrachtniveau van Oura heeft toestemming voor stressgegevens nodig, dus verbind Oura één keer opnieuw onder Integraties en sta die toe.", "loadError": "Je herstelwaarden konden niet worden geladen." }, "bodyTempDeviation": { diff --git a/messages/pl.json b/messages/pl.json index 21304ed2c..d87da90db 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -4657,7 +4657,7 @@ "scoreCrossLink": "Zobacz łączny wynik regeneracji", "emptyTitle": "Brak danych o regeneracji", "empty": "Ta strona wypełnia się wartościami regeneracji, obciążenia i obciążenia treningowego z WHOOP lub Polar oraz poziomem odporności z Oura.", - "emptyOuraPermission": "Poziom odporności Oura wymaga dostępu do danych o zdrowiu serca, więc połącz Oura ponownie raz w Integracjach i zezwól na niego.", + "emptyOuraPermission": "Poziom odporności Oura wymaga uprawnienia do danych o stresie, więc połącz Oura ponownie raz w Integracjach i zezwól na nie.", "emptyOuraNoResilience": "Oura jest połączona, a jej poziom odporności pojawi się tutaj, gdy tylko Oura go dostarczy.", "emptyAction": "Otwórz integracje", "block": { diff --git a/prisma/migrations/0419_oauth_callback_claims/migration.sql b/prisma/migrations/0419_oauth_callback_claims/migration.sql new file mode 100644 index 000000000..5f2a45ab4 --- /dev/null +++ b/prisma/migrations/0419_oauth_callback_claims/migration.sql @@ -0,0 +1,28 @@ +-- Single-use OAuth callbacks. +-- +-- A browser can deliver an OAuth callback twice with the same code and state, +-- a few hundred milliseconds apart. The connect flows that sign their state +-- (Oura, Polar, Strava) kept no record of it, so both requests exchanged the +-- code; the second was refused, and Oura answers a reused code by revoking the +-- token the first exchange had just issued. A row here is the claim on one +-- callback: the first request inserts it before the exchange, a duplicate +-- finds it taken and is sent where the first one went. No user id, token or +-- code is stored; the key is a hash of the provider and the state. +-- +-- Reversibility: +-- DROP TABLE IF EXISTS "oauth_callback_claims"; + +-- CreateTable +CREATE TABLE "oauth_callback_claims" ( + "key" TEXT NOT NULL, + "provider" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "expires_at" TIMESTAMP(3) NOT NULL, + "settled_at" TIMESTAMP(3), + "location" TEXT, + + CONSTRAINT "oauth_callback_claims_pkey" PRIMARY KEY ("key") +); + +-- CreateIndex +CREATE INDEX "oauth_callback_claims_expires_at_idx" ON "oauth_callback_claims"("expires_at"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c802f3fd4..e972d56af 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -4549,6 +4549,34 @@ model GoogleHealthOAuthState { @@map("google_health_oauth_states") } +/// One row per OAuth callback the connect flows have begun to act on, keyed on +/// a hash of the provider and the `state` it returned with. Inserting it is +/// the claim: a second request carrying the same `state` (a browser that sends +/// the callback twice, a reload, a retry) finds the row taken and never +/// exchanges the code again. An authorization code works once, and Oura +/// revokes the token the first exchange issued when it sees the code reused. +/// +/// The claiming request records where it sent the browser, so the duplicate +/// sends it to the same place instead of to an error page. Holds no user id, +/// token or code: `location` is the app's own result URL (`?oura=connected`, +/// `?whoop=error&reason=…`). Rows outlive the state they guard by a few +/// minutes and are swept by expiry on the next settle +/// (`src/lib/oauth/callback-claim.ts`). +model OAuthCallbackClaim { + /// sha256(provider + NUL + state), hex. + key String @id + provider String + createdAt DateTime @default(now()) @map("created_at") + expiresAt DateTime @map("expires_at") + /// When the claiming request finished, with or without a result URL. + settledAt DateTime? @map("settled_at") + /// The redirect the claiming request answered with. Null when it threw. + location String? + + @@index([expiresAt]) + @@map("oauth_callback_claims") +} + // ─── Mood Entries ───────────────────────────────────────────── model MoodEntry { diff --git a/src/app/api/fitbit/callback/route.ts b/src/app/api/fitbit/callback/route.ts index 984698e3c..84af494a0 100644 --- a/src/app/api/fitbit/callback/route.ts +++ b/src/app/api/fitbit/callback/route.ts @@ -15,6 +15,11 @@ import { FITBIT_BACKFILL_QUEUE } from "@/lib/jobs/fitbit-backfill"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; +import { + consumeLedgerState, + duplicateOAuthCallback, + settleOAuthCallback, +} from "@/lib/oauth/callback-claim"; import { NextRequest, NextResponse } from "next/server"; import { timingSafeEqual } from "node:crypto"; @@ -37,6 +42,12 @@ import { timingSafeEqual } from "node:crypto"; * On success: exchange the code, fetch the Fitbit profile for the external * `fitbitUserId`, persist the encrypted `FitbitConnection`, clear any prior * reauth state, and enqueue the self-converging history backfill. + * + * A repeat of a consumed nonce is told apart from a replay by the callback + * claim taken in the same transaction (`src/lib/oauth/callback-claim.ts`): a + * browser that delivers the callback twice gets one exchange, and the repeat + * follows the first request to the same page. Only a nonce with no claim left + * lands on `replay`. */ const ERR = (reason: string) => NextResponse.redirect( @@ -79,14 +90,24 @@ export const GET = apiHandler(async (request: NextRequest) => { codeVerifier: string | null; } | null = null; try { - stateRow = await prisma.fitbitOAuthState.delete({ - where: { nonce: state }, - }); + stateRow = await consumeLedgerState("fitbit", state, (tx) => + tx.fitbitOAuthState.delete({ where: { nonce: state } }), + ); } catch (err) { if ( err instanceof Prisma.PrismaClientKnownRequestError && err.code === "P2025" ) { + // A duplicate of a callback that is consuming or has consumed this + // row: follow that request's result instead of showing an error. + const duplicate = await duplicateOAuthCallback("fitbit", state, { + fallbackUrl: new URL( + "/settings/integrations", + process.env.NEXT_PUBLIC_APP_URL!, + ).toString(), + stateCookie: FITBIT_OAUTH_STATE_COOKIE, + }); + if (duplicate) return duplicate; annotate({ meta: { reason: "replay" } }); return ERR("replay"); } @@ -96,105 +117,108 @@ export const GET = apiHandler(async (request: NextRequest) => { return ERR("state"); } - if (stateRow.expiresAt <= new Date()) { - annotate({ meta: { reason: "expired" } }); - return ERR("expired"); - } - if (stateRow.userId !== user.id) { - annotate({ meta: { reason: "cross_user" } }); - return ERR("cross_user"); - } - - if (!code) { - return ERR("nocode"); - } - - // The PKCE verifier minted at connect rides on the ledger row. Its absence - // means a malformed / legacy handshake — refuse rather than send a code - // exchange Fitbit will reject for a missing verifier. - if (!stateRow.codeVerifier) { - annotate({ meta: { reason: "no_verifier" } }); - return ERR("no_verifier"); - } - - try { - const creds = await getUserFitbitCredentials(user.id); - if (!creds) { - return ERR("nocreds"); + const consumed = stateRow; + return settleOAuthCallback("fitbit", state, async () => { + if (consumed.expiresAt <= new Date()) { + annotate({ meta: { reason: "expired" } }); + return ERR("expired"); + } + if (consumed.userId !== user.id) { + annotate({ meta: { reason: "cross_user" } }); + return ERR("cross_user"); } - const tokens = await exchangeCode(code, stateRow.codeVerifier, creds); - - // The initial code exchange MUST return a refresh token. If Fitbit ever - // omits it, refuse to persist — a stored empty refresh token silently bricks - // every future token refresh (the refresh POST then 400s and the connection - // dies the moment the access token expires). Error out cleanly so the user - // re-consents rather than landing a dead connection. - if (!tokens.refresh_token) { - getEvent()?.addWarning( - `fitbit callback: token response carried no refresh_token for user ${user.id}`, - ); - annotate({ meta: { reason: "no_refresh_token" } }); - return ERR("no_refresh_token"); + if (!code) { + return ERR("nocode"); } - const profile = await fetchProfile(tokens.access_token); - const fitbitUserId = resolveFitbitUserId(profile); - const expiresAt = new Date(Date.now() + tokens.expires_in * 1000); + // The PKCE verifier minted at connect rides on the ledger row. Its absence + // means a malformed / legacy handshake — refuse rather than send a code + // exchange Fitbit will reject for a missing verifier. + if (!consumed.codeVerifier) { + annotate({ meta: { reason: "no_verifier" } }); + return ERR("no_verifier"); + } - await prisma.fitbitConnection.upsert({ - where: { userId: user.id }, - update: { - fitbitUserId, - accessToken: encrypt(tokens.access_token), - refreshToken: encrypt(tokens.refresh_token), - tokenExpiresAt: expiresAt, - scope: tokens.scope ?? FITBIT_OAUTH_SCOPE, - backfillCompletedAt: null, - }, - create: { - userId: user.id, - fitbitUserId, - accessToken: encrypt(tokens.access_token), - refreshToken: encrypt(tokens.refresh_token), - tokenExpiresAt: expiresAt, - scope: tokens.scope ?? FITBIT_OAUTH_SCOPE, - }, - }); + try { + const creds = await getUserFitbitCredentials(user.id); + if (!creds) { + return ERR("nocreds"); + } - await auditLog("fitbit.connect", { - userId: user.id, - details: { fitbitUserId }, - }); + const tokens = await exchangeCode(code, consumed.codeVerifier, creds); - // Re-completing OAuth clears any prior reauth-required state. - await markReconnected(user.id, "fitbit"); - - // Enqueue the self-converging history backfill. Best-effort: the boot-time - // discovery query (`backfillCompletedAt IS NULL`) is the safety net, so a - // missing boss instance here doesn't strand the connection. - const boss = getGlobalBoss(); - if (boss) { - await boss - .send(FITBIT_BACKFILL_QUEUE, { - userId: user.id, - enqueuedAt: new Date().toISOString(), - }) - .catch((err) => - getEvent()?.addWarning(`fitbit-backfill enqueue failed: ${err}`), + // The initial code exchange MUST return a refresh token. If Fitbit ever + // omits it, refuse to persist — a stored empty refresh token silently bricks + // every future token refresh (the refresh POST then 400s and the connection + // dies the moment the access token expires). Error out cleanly so the user + // re-consents rather than landing a dead connection. + if (!tokens.refresh_token) { + getEvent()?.addWarning( + `fitbit callback: token response carried no refresh_token for user ${user.id}`, ); - } + annotate({ meta: { reason: "no_refresh_token" } }); + return ERR("no_refresh_token"); + } - const response = NextResponse.redirect( - new URL( - "/settings/integrations?fitbit=connected", - process.env.NEXT_PUBLIC_APP_URL!, - ), - ); - response.cookies.delete(FITBIT_OAUTH_STATE_COOKIE); - return response; - } catch (err) { - getEvent()?.setError(err); - return ERR("token"); - } + const profile = await fetchProfile(tokens.access_token); + const fitbitUserId = resolveFitbitUserId(profile); + const expiresAt = new Date(Date.now() + tokens.expires_in * 1000); + + await prisma.fitbitConnection.upsert({ + where: { userId: user.id }, + update: { + fitbitUserId, + accessToken: encrypt(tokens.access_token), + refreshToken: encrypt(tokens.refresh_token), + tokenExpiresAt: expiresAt, + scope: tokens.scope ?? FITBIT_OAUTH_SCOPE, + backfillCompletedAt: null, + }, + create: { + userId: user.id, + fitbitUserId, + accessToken: encrypt(tokens.access_token), + refreshToken: encrypt(tokens.refresh_token), + tokenExpiresAt: expiresAt, + scope: tokens.scope ?? FITBIT_OAUTH_SCOPE, + }, + }); + + await auditLog("fitbit.connect", { + userId: user.id, + details: { fitbitUserId }, + }); + + // Re-completing OAuth clears any prior reauth-required state. + await markReconnected(user.id, "fitbit"); + + // Enqueue the self-converging history backfill. Best-effort: the boot-time + // discovery query (`backfillCompletedAt IS NULL`) is the safety net, so a + // missing boss instance here doesn't strand the connection. + const boss = getGlobalBoss(); + if (boss) { + await boss + .send(FITBIT_BACKFILL_QUEUE, { + userId: user.id, + enqueuedAt: new Date().toISOString(), + }) + .catch((err) => + getEvent()?.addWarning(`fitbit-backfill enqueue failed: ${err}`), + ); + } + + const response = NextResponse.redirect( + new URL( + "/settings/integrations?fitbit=connected", + process.env.NEXT_PUBLIC_APP_URL!, + ), + ); + response.cookies.delete(FITBIT_OAUTH_STATE_COOKIE); + return response; + } catch (err) { + getEvent()?.setError(err); + return ERR("token"); + } + }); }); diff --git a/src/app/api/google-health/callback/route.ts b/src/app/api/google-health/callback/route.ts index 6c8c8d8f2..9f87a2fcf 100644 --- a/src/app/api/google-health/callback/route.ts +++ b/src/app/api/google-health/callback/route.ts @@ -15,6 +15,11 @@ import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "@/lib/jobs/google-health-backfill" import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; +import { + consumeLedgerState, + duplicateOAuthCallback, + settleOAuthCallback, +} from "@/lib/oauth/callback-claim"; import { NextRequest, NextResponse } from "next/server"; import { timingSafeEqual } from "node:crypto"; @@ -36,6 +41,12 @@ import { timingSafeEqual } from "node:crypto"; * Health profile for the external `googleUserId`, persist the encrypted * `GoogleHealthConnection`, clear any prior reauth state (`needsReauth:false` + * the status ledger), and enqueue the self-converging history backfill. + * + * A repeat of a consumed nonce is told apart from a replay by the callback + * claim taken in the same transaction (`src/lib/oauth/callback-claim.ts`): a + * browser that delivers the callback twice gets one exchange, and the repeat + * follows the first request to the same page. Only a nonce with no claim left + * lands on `replay`. */ const ERR = (reason: string) => NextResponse.redirect( @@ -81,14 +92,24 @@ export const GET = apiHandler(async (request: NextRequest) => { codeVerifier: string | null; } | null = null; try { - stateRow = await prisma.googleHealthOAuthState.delete({ - where: { nonce: state }, - }); + stateRow = await consumeLedgerState("google-health", state, (tx) => + tx.googleHealthOAuthState.delete({ where: { nonce: state } }), + ); } catch (err) { if ( err instanceof Prisma.PrismaClientKnownRequestError && err.code === "P2025" ) { + // A duplicate of a callback that is consuming or has consumed this + // row: follow that request's result instead of showing an error. + const duplicate = await duplicateOAuthCallback("google-health", state, { + fallbackUrl: new URL( + "/settings/integrations", + process.env.NEXT_PUBLIC_APP_URL!, + ).toString(), + stateCookie: GOOGLE_HEALTH_OAUTH_STATE_COOKIE, + }); + if (duplicate) return duplicate; annotate({ meta: { reason: "replay" } }); return ERR("replay"); } @@ -98,114 +119,117 @@ export const GET = apiHandler(async (request: NextRequest) => { return ERR("state"); } - if (stateRow.expiresAt <= new Date()) { - annotate({ meta: { reason: "expired" } }); - return ERR("expired"); - } - if (stateRow.userId !== user.id) { - annotate({ meta: { reason: "cross_user" } }); - return ERR("cross_user"); - } - if (!stateRow.codeVerifier) { - annotate({ meta: { reason: "no_verifier" } }); - return ERR("no_verifier"); - } - - if (!code) { - return ERR("nocode"); - } - - try { - const creds = await getUserGoogleHealthCredentials(user.id); - if (!creds) { - return ERR("nocreds"); + const consumed = stateRow; + return settleOAuthCallback("google-health", state, async () => { + if (consumed.expiresAt <= new Date()) { + annotate({ meta: { reason: "expired" } }); + return ERR("expired"); + } + if (consumed.userId !== user.id) { + annotate({ meta: { reason: "cross_user" } }); + return ERR("cross_user"); + } + if (!consumed.codeVerifier) { + annotate({ meta: { reason: "no_verifier" } }); + return ERR("no_verifier"); } - const tokens = await exchangeCode(code, stateRow.codeVerifier, creds); - - // The initial code exchange MUST return a refresh token (the connect route - // forces `prompt=consent` + `access_type=offline`). If Google ever omits - // it, refuse to persist — a stored empty refresh token silently bricks - // every future token refresh (the refresh POST then 400s and the - // connection dies the moment the access token expires). Error out cleanly - // so the user re-consents rather than landing a dead connection. - if (!tokens.refresh_token) { - getEvent()?.addWarning( - `google-health callback: token response carried no refresh_token for user ${user.id}`, - ); - annotate({ meta: { reason: "no_refresh_token" } }); - return ERR("no_refresh_token"); + if (!code) { + return ERR("nocode"); } - const profile = await fetchProfile(tokens.access_token); - const googleUserId = resolveGoogleHealthUserId(profile); - const expiresAt = new Date(Date.now() + tokens.expires_in * 1000); - const scope = tokens.scope ?? getGoogleHealthScopeString(); + try { + const creds = await getUserGoogleHealthCredentials(user.id); + if (!creds) { + return ERR("nocreds"); + } - await prisma.googleHealthConnection.upsert({ - where: { userId: user.id }, - update: { - googleUserId, - accessToken: encrypt(tokens.access_token), - refreshToken: encrypt(tokens.refresh_token), - tokenExpiresAt: expiresAt, - scope, - // Fresh consent clears the soft-disconnect flag so the card drops the - // reconnect CTA and the sync path stops short-circuiting. - needsReauth: false, - backfillCompletedAt: null, - // A new grant walks history from the start: what an earlier - // backfill finished was read under the old one. - backfillProgress: Prisma.DbNull, - }, - create: { - userId: user.id, - googleUserId, - accessToken: encrypt(tokens.access_token), - refreshToken: encrypt(tokens.refresh_token), - tokenExpiresAt: expiresAt, - scope, - needsReauth: false, - }, - }); + const tokens = await exchangeCode(code, consumed.codeVerifier, creds); - await auditLog("google_health.connect", { - userId: user.id, - details: { googleUserId }, - }); - - // Re-completing OAuth clears any prior reauth-required state on the status - // ledger too (the `needsReauth` column above is the connection-local flag; - // this resets the integration-status streak). - await markReconnected(user.id, "google-health"); - - // Enqueue the self-converging history backfill. Best-effort: the boot-time - // discovery query (`backfillCompletedAt IS NULL`) is the safety net, so a - // missing boss instance here doesn't strand the connection. - const boss = getGlobalBoss(); - if (boss) { - await boss - .send(GOOGLE_HEALTH_BACKFILL_QUEUE, { - userId: user.id, - enqueuedAt: new Date().toISOString(), - }) - .catch((err) => - getEvent()?.addWarning( - `google-health-backfill enqueue failed: ${err}`, - ), + // The initial code exchange MUST return a refresh token (the connect route + // forces `prompt=consent` + `access_type=offline`). If Google ever omits + // it, refuse to persist — a stored empty refresh token silently bricks + // every future token refresh (the refresh POST then 400s and the + // connection dies the moment the access token expires). Error out cleanly + // so the user re-consents rather than landing a dead connection. + if (!tokens.refresh_token) { + getEvent()?.addWarning( + `google-health callback: token response carried no refresh_token for user ${user.id}`, ); - } + annotate({ meta: { reason: "no_refresh_token" } }); + return ERR("no_refresh_token"); + } - const response = NextResponse.redirect( - new URL( - "/settings/integrations?googleHealth=connected", - process.env.NEXT_PUBLIC_APP_URL!, - ), - ); - response.cookies.delete(GOOGLE_HEALTH_OAUTH_STATE_COOKIE); - return response; - } catch (err) { - getEvent()?.setError(err); - return ERR("token"); - } + const profile = await fetchProfile(tokens.access_token); + const googleUserId = resolveGoogleHealthUserId(profile); + const expiresAt = new Date(Date.now() + tokens.expires_in * 1000); + const scope = tokens.scope ?? getGoogleHealthScopeString(); + + await prisma.googleHealthConnection.upsert({ + where: { userId: user.id }, + update: { + googleUserId, + accessToken: encrypt(tokens.access_token), + refreshToken: encrypt(tokens.refresh_token), + tokenExpiresAt: expiresAt, + scope, + // Fresh consent clears the soft-disconnect flag so the card drops the + // reconnect CTA and the sync path stops short-circuiting. + needsReauth: false, + backfillCompletedAt: null, + // A new grant walks history from the start: what an earlier + // backfill finished was read under the old one. + backfillProgress: Prisma.DbNull, + }, + create: { + userId: user.id, + googleUserId, + accessToken: encrypt(tokens.access_token), + refreshToken: encrypt(tokens.refresh_token), + tokenExpiresAt: expiresAt, + scope, + needsReauth: false, + }, + }); + + await auditLog("google_health.connect", { + userId: user.id, + details: { googleUserId }, + }); + + // Re-completing OAuth clears any prior reauth-required state on the status + // ledger too (the `needsReauth` column above is the connection-local flag; + // this resets the integration-status streak). + await markReconnected(user.id, "google-health"); + + // Enqueue the self-converging history backfill. Best-effort: the boot-time + // discovery query (`backfillCompletedAt IS NULL`) is the safety net, so a + // missing boss instance here doesn't strand the connection. + const boss = getGlobalBoss(); + if (boss) { + await boss + .send(GOOGLE_HEALTH_BACKFILL_QUEUE, { + userId: user.id, + enqueuedAt: new Date().toISOString(), + }) + .catch((err) => + getEvent()?.addWarning( + `google-health-backfill enqueue failed: ${err}`, + ), + ); + } + + const response = NextResponse.redirect( + new URL( + "/settings/integrations?googleHealth=connected", + process.env.NEXT_PUBLIC_APP_URL!, + ), + ); + response.cookies.delete(GOOGLE_HEALTH_OAUTH_STATE_COOKIE); + return response; + } catch (err) { + getEvent()?.setError(err); + return ERR("token"); + } + }); }); diff --git a/src/app/api/oura/callback/__tests__/route.test.ts b/src/app/api/oura/callback/__tests__/route.test.ts index efaf97c1d..898390087 100644 --- a/src/app/api/oura/callback/__tests__/route.test.ts +++ b/src/app/api/oura/callback/__tests__/route.test.ts @@ -1,5 +1,25 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +// Single-use claiming is covered against Postgres in +// tests/integration/oauth-callback-single-use.test.ts; here every callback +// is the first and only one. +vi.mock("@/lib/oauth/callback-claim", () => ({ + runOAuthCallbackOnce: ( + _p: string, + _s: string, + _o: unknown, + run: () => Promise, + ) => run(), + settleOAuthCallback: (_p: string, _s: string, run: () => Promise) => + run(), + consumeLedgerState: async ( + _p: string, + _s: string, + consume: (tx: unknown) => Promise, + ) => consume((await import("@/lib/db")).prisma), + duplicateOAuthCallback: async () => null, +})); + vi.mock("@/lib/api-handler", () => ({ apiHandler: unknown>(fn: T) => fn, })); diff --git a/src/app/api/oura/callback/route.ts b/src/app/api/oura/callback/route.ts index a68ff917d..8871b3a12 100644 --- a/src/app/api/oura/callback/route.ts +++ b/src/app/api/oura/callback/route.ts @@ -12,6 +12,8 @@ import { stateMatchesCookie, verifySignedState, } from "@/lib/oauth/signed-state"; +import { runOAuthCallbackOnce } from "@/lib/oauth/callback-claim"; +import { ouraCollectionsMissingFromGrant } from "@/lib/oura/scope"; import { markReconnected } from "@/lib/integrations/status"; import { OURA_SYNC_QUEUE } from "@/lib/jobs/integration-poll-queues"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; @@ -26,9 +28,22 @@ import { NextRequest, NextResponse } from "next/server"; * binding. The `userId` is read from inside the verified token, cross-checked * against the web session when present. * + * The callback is single-use: the verified state is claimed before anything + * else (`src/lib/oauth/callback-claim.ts`). A browser that delivers it twice + * gets one exchange; the duplicate is sent where the first request went and + * never touches the code, because Oura revokes the first exchange's token + * when it sees the code reused. + * * On success: exchange the code, persist the encrypted access + refresh token - * on `User`, clear any prior reauth state, and redirect back to Settings. + * on `User`, note the collections the granted scopes leave out, clear any + * prior reauth state, and redirect back to Settings. */ +const SETTINGS_URL = () => + new URL( + "/settings/integrations", + process.env.NEXT_PUBLIC_APP_URL!, + ).toString(); + const ERR = (reason: string) => NextResponse.redirect( new URL( @@ -40,9 +55,7 @@ const ERR = (reason: string) => export const GET = apiHandler(async (request: NextRequest) => { annotate({ action: { name: "oura.callback" } }); - const { searchParams } = new URL(request.url); - const code = searchParams.get("code"); - const state = searchParams.get("state"); + const state = new URL(request.url).searchParams.get("state"); const cookieState = request.cookies.get(oauthStateCookieName("oura"))?.value; if (!stateMatchesCookie(state, cookieState)) { @@ -57,6 +70,24 @@ export const GET = apiHandler(async (request: NextRequest) => { } const userId = verified.userId; + return runOAuthCallbackOnce( + "oura", + state, + { + fallbackUrl: SETTINGS_URL(), + stateCookie: oauthStateCookieName("oura"), + }, + () => completeCallback(request, userId), + ); +}); + +async function completeCallback( + request: NextRequest, + userId: string, +): Promise { + const { searchParams } = new URL(request.url); + const code = searchParams.get("code"); + const evt = getEvent(); if (evt) evt.setAuth({ user_id: userId, auth_method: "session" }); @@ -83,14 +114,21 @@ export const GET = apiHandler(async (request: NextRequest) => { const tokens = await exchangeCode(code, creds, iss); + // The token response names the granted scopes when Oura sends them; a + // permission the person unticked on the consent screen is known from it + // at once. Without them, the first sync finds out per collection. + const missing = ouraCollectionsMissingFromGrant(tokens.scope); + if (missing && missing.length > 0) { + annotate({ meta: { oura_scope_denied: missing } }); + } + await prisma.user.update({ where: { id: userId }, data: { ouraAccessTokenEncrypted: encrypt(tokens.access_token), ouraRefreshTokenEncrypted: encrypt(tokens.refresh_token), - // A new grant carries the scopes asked for now; whatever the old one - // lacked is found out again by the first sync. - ouraScopeDenied: Prisma.DbNull, + ouraScopeDenied: + missing && missing.length > 0 ? missing : Prisma.DbNull, }, }); @@ -124,4 +162,4 @@ export const GET = apiHandler(async (request: NextRequest) => { getEvent()?.setError(err); return ERR("token"); } -}); +} diff --git a/src/app/api/polar/callback/__tests__/route.test.ts b/src/app/api/polar/callback/__tests__/route.test.ts index ac69bba3e..18e5c6847 100644 --- a/src/app/api/polar/callback/__tests__/route.test.ts +++ b/src/app/api/polar/callback/__tests__/route.test.ts @@ -22,6 +22,26 @@ const { bossMock: vi.fn(), })); +// Single-use claiming is covered against Postgres in +// tests/integration/oauth-callback-single-use.test.ts; here every callback +// is the first and only one. +vi.mock("@/lib/oauth/callback-claim", () => ({ + runOAuthCallbackOnce: ( + _p: string, + _s: string, + _o: unknown, + run: () => Promise, + ) => run(), + settleOAuthCallback: (_p: string, _s: string, run: () => Promise) => + run(), + consumeLedgerState: async ( + _p: string, + _s: string, + consume: (tx: unknown) => Promise, + ) => consume((await import("@/lib/db")).prisma), + duplicateOAuthCallback: async () => null, +})); + vi.mock("@/lib/api-handler", () => ({ apiHandler: unknown>(fn: T) => fn, })); diff --git a/src/app/api/polar/callback/route.ts b/src/app/api/polar/callback/route.ts index bb52eed61..b18b1be34 100644 --- a/src/app/api/polar/callback/route.ts +++ b/src/app/api/polar/callback/route.ts @@ -11,6 +11,7 @@ import { stateMatchesCookie, verifySignedState, } from "@/lib/oauth/signed-state"; +import { runOAuthCallbackOnce } from "@/lib/oauth/callback-claim"; import { markReconnected } from "@/lib/integrations/status"; import { POLAR_SYNC_QUEUE } from "@/lib/jobs/integration-poll-queues"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; @@ -42,9 +43,7 @@ const ERR = (reason: string) => export const GET = apiHandler(async (request: NextRequest) => { annotate({ action: { name: "polar.callback" } }); - const { searchParams } = new URL(request.url); - const code = searchParams.get("code"); - const state = searchParams.get("state"); + const state = new URL(request.url).searchParams.get("state"); const cookieState = request.cookies.get(oauthStateCookieName("polar"))?.value; // CSRF double-submit: the URL state must equal the cookie (constant-time). @@ -61,6 +60,28 @@ export const GET = apiHandler(async (request: NextRequest) => { } const userId = verified.userId; + // Single-use: a callback delivered twice exchanges its code once, and the + // duplicate follows the first request's result (src/lib/oauth/callback-claim.ts). + return runOAuthCallbackOnce( + "polar", + state, + { + fallbackUrl: new URL( + "/settings/integrations", + process.env.NEXT_PUBLIC_APP_URL!, + ).toString(), + stateCookie: oauthStateCookieName("polar"), + }, + () => completeCallback(request, userId), + ); +}); + +async function completeCallback( + request: NextRequest, + userId: string, +): Promise { + const code = new URL(request.url).searchParams.get("code"); + const evt = getEvent(); if (evt) evt.setAuth({ user_id: userId, auth_method: "session" }); @@ -129,4 +150,4 @@ export const GET = apiHandler(async (request: NextRequest) => { getEvent()?.setError(err); return ERR("token"); } -}); +} diff --git a/src/app/api/strava/callback/__tests__/route.test.ts b/src/app/api/strava/callback/__tests__/route.test.ts index ec238315d..407cff3b5 100644 --- a/src/app/api/strava/callback/__tests__/route.test.ts +++ b/src/app/api/strava/callback/__tests__/route.test.ts @@ -25,6 +25,26 @@ const { markReconnectedMock: vi.fn(async () => {}), })); +// Single-use claiming is covered against Postgres in +// tests/integration/oauth-callback-single-use.test.ts; here every callback +// is the first and only one. +vi.mock("@/lib/oauth/callback-claim", () => ({ + runOAuthCallbackOnce: ( + _p: string, + _s: string, + _o: unknown, + run: () => Promise, + ) => run(), + settleOAuthCallback: (_p: string, _s: string, run: () => Promise) => + run(), + consumeLedgerState: async ( + _p: string, + _s: string, + consume: (tx: unknown) => Promise, + ) => consume((await import("@/lib/db")).prisma), + duplicateOAuthCallback: async () => null, +})); + vi.mock("@/lib/api-handler", () => ({ apiHandler: unknown>(fn: T) => fn, })); diff --git a/src/app/api/strava/callback/route.ts b/src/app/api/strava/callback/route.ts index 40934189e..807c36d4c 100644 --- a/src/app/api/strava/callback/route.ts +++ b/src/app/api/strava/callback/route.ts @@ -11,6 +11,7 @@ import { stateMatchesCookie, verifySignedState, } from "@/lib/oauth/signed-state"; +import { runOAuthCallbackOnce } from "@/lib/oauth/callback-claim"; import { markReconnected } from "@/lib/integrations/status"; import { STRAVA_BACKFILL_QUEUE } from "@/lib/jobs/strava-backfill"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; @@ -42,9 +43,7 @@ const ERR = (reason: string) => export const GET = apiHandler(async (request: NextRequest) => { annotate({ action: { name: "strava.callback" } }); - const { searchParams } = new URL(request.url); - const code = searchParams.get("code"); - const state = searchParams.get("state"); + const state = new URL(request.url).searchParams.get("state"); const cookieState = request.cookies.get( oauthStateCookieName("strava"), )?.value; @@ -61,6 +60,28 @@ export const GET = apiHandler(async (request: NextRequest) => { } const userId = verified.userId; + // Single-use: a callback delivered twice exchanges its code once, and the + // duplicate follows the first request's result (src/lib/oauth/callback-claim.ts). + return runOAuthCallbackOnce( + "strava", + state, + { + fallbackUrl: new URL( + "/settings/integrations", + process.env.NEXT_PUBLIC_APP_URL!, + ).toString(), + stateCookie: oauthStateCookieName("strava"), + }, + () => completeCallback(request, userId), + ); +}); + +async function completeCallback( + request: NextRequest, + userId: string, +): Promise { + const code = new URL(request.url).searchParams.get("code"); + const evt = getEvent(); if (evt) evt.setAuth({ user_id: userId, auth_method: "session" }); @@ -139,4 +160,4 @@ export const GET = apiHandler(async (request: NextRequest) => { getEvent()?.setError(err); return ERR("token"); } -}); +} diff --git a/src/app/api/whoop/callback/__tests__/route.test.ts b/src/app/api/whoop/callback/__tests__/route.test.ts index a335131d7..2f8d632fb 100644 --- a/src/app/api/whoop/callback/__tests__/route.test.ts +++ b/src/app/api/whoop/callback/__tests__/route.test.ts @@ -1,15 +1,29 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +// Single-use claiming is covered against Postgres in +// tests/integration/oauth-callback-single-use.test.ts. Here the state row is +// consumed for real against the mocked client, the claim always wins, and +// every callback is the first and only one. +vi.mock("@/lib/oauth/callback-claim", async (importOriginal) => ({ + ...(await importOriginal()), + settleOAuthCallback: (_p: string, _s: string, run: () => Promise) => + run(), + duplicateOAuthCallback: async () => null, +})); + vi.mock("@/lib/api-handler", () => ({ apiHandler: unknown>(fn: T) => fn, })); -vi.mock("@/lib/db", () => ({ - prisma: { +vi.mock("@/lib/db", () => { + const prisma = { whoopOAuthState: { delete: vi.fn() }, whoopConnection: { findUnique: vi.fn(), upsert: vi.fn() }, - }, -})); + oAuthCallbackClaim: { createMany: vi.fn(async () => ({ count: 1 })) }, + $transaction: async (run: (tx: unknown) => unknown) => run(prisma), + }; + return { prisma }; +}); vi.mock("@/lib/auth/session", () => ({ // Native ticket path → no web session present. diff --git a/src/app/api/whoop/callback/route.ts b/src/app/api/whoop/callback/route.ts index 418ab11f5..63d190764 100644 --- a/src/app/api/whoop/callback/route.ts +++ b/src/app/api/whoop/callback/route.ts @@ -17,6 +17,11 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; import { isP2002 } from "@/lib/prisma-errors"; +import { + consumeLedgerState, + duplicateOAuthCallback, + settleOAuthCallback, +} from "@/lib/oauth/callback-claim"; import { NextRequest, NextResponse } from "next/server"; import { timingSafeEqual } from "node:crypto"; @@ -51,6 +56,12 @@ import { timingSafeEqual } from "node:crypto"; * On success: exchange the code, fetch the WHOOP profile for `whoopUserId`, * persist the encrypted `WhoopConnection`, clear any prior reauth state, and * enqueue the self-converging history backfill. + * + * A repeat of a consumed nonce is told apart from a replay by the callback + * claim taken in the same transaction (`src/lib/oauth/callback-claim.ts`): a + * browser that delivers the callback twice gets one exchange, and the repeat + * follows the first request to the same page. Only a nonce with no claim left + * lands on `replay`. */ /** Web-URL redirect (default + every pre-row-resolution path). */ @@ -119,12 +130,24 @@ export const GET = apiHandler(async (request: NextRequest) => { returnScheme: string | null; } | null = null; try { - stateRow = await prisma.whoopOAuthState.delete({ where: { nonce: state } }); + stateRow = await consumeLedgerState("whoop", state, (tx) => + tx.whoopOAuthState.delete({ where: { nonce: state } }), + ); } catch (err) { if ( err instanceof Prisma.PrismaClientKnownRequestError && err.code === "P2025" ) { + // A duplicate of a callback that is consuming or has consumed this + // row: follow that request's result instead of showing an error. + const duplicate = await duplicateOAuthCallback("whoop", state, { + fallbackUrl: new URL( + "/settings/integrations", + process.env.NEXT_PUBLIC_APP_URL!, + ).toString(), + stateCookie: WHOOP_OAUTH_STATE_COOKIE, + }); + if (duplicate) return duplicate; annotate({ meta: { reason: "replay" } }); return WEB_ERR("replay"); } @@ -134,109 +157,112 @@ export const GET = apiHandler(async (request: NextRequest) => { return WEB_ERR("state"); } - // The validated native return scheme (if any) is now known; every redirect - // below honours it. Identity is the row's userId (bound at connect time from - // the session or the one-time ticket). - const returnScheme = stateRow.returnScheme; - const userId = stateRow.userId; + const consumed = stateRow; + return settleOAuthCallback("whoop", state, async () => { + // The validated native return scheme (if any) is now known; every redirect + // below honours it. Identity is the row's userId (bound at connect time from + // the session or the one-time ticket). + const returnScheme = consumed.returnScheme; + const userId = consumed.userId; - const evt = getEvent(); - if (evt) { - // Identity resolved from the consumed state row (bound at connect time - // from the session or the one-time ticket); no role is asserted here. - evt.setAuth({ user_id: userId, auth_method: "session" }); - } - - if (stateRow.expiresAt <= new Date()) { - annotate({ meta: { reason: "expired" } }); - return outcomeRedirect(returnScheme, "error", "expired"); - } - - // Optional session cross-check: only when a web session cookie is present. - // The native ticket path carries no session — the nonce cookie + atomic - // single-use delete already bind the row to its originator. - const sessionData = await getSession(); - if (sessionData && sessionData.user.id !== userId) { - annotate({ meta: { reason: "cross_user" } }); - return outcomeRedirect(returnScheme, "error", "cross_user"); - } - - if (!code) { - return outcomeRedirect(returnScheme, "error", "nocode"); - } - - try { - const creds = await getUserWhoopCredentials(userId); - if (!creds) { - return outcomeRedirect(returnScheme, "error", "nocreds"); + const evt = getEvent(); + if (evt) { + // Identity resolved from the consumed state row (bound at connect time + // from the session or the one-time ticket); no role is asserted here. + evt.setAuth({ user_id: userId, auth_method: "session" }); } - const tokens = await exchangeCode(code, creds); - const profile = await fetchProfile(tokens.access_token); - const whoopUserId = String(profile.user_id); - const expiresAt = new Date(Date.now() + tokens.expires_in * 1000); - const owner = await prisma.whoopConnection.findUnique({ - where: { whoopUserId }, - select: { userId: true }, - }); - if (owner && owner.userId !== userId) { - annotate({ meta: { reason: "owner_conflict" } }); - return outcomeRedirect(returnScheme, "error", "owner_conflict"); + if (consumed.expiresAt <= new Date()) { + annotate({ meta: { reason: "expired" } }); + return outcomeRedirect(returnScheme, "error", "expired"); + } + + // Optional session cross-check: only when a web session cookie is present. + // The native ticket path carries no session — the nonce cookie + atomic + // single-use delete already bind the row to its originator. + const sessionData = await getSession(); + if (sessionData && sessionData.user.id !== userId) { + annotate({ meta: { reason: "cross_user" } }); + return outcomeRedirect(returnScheme, "error", "cross_user"); + } + + if (!code) { + return outcomeRedirect(returnScheme, "error", "nocode"); } try { - await prisma.whoopConnection.upsert({ - where: { userId }, - update: { - whoopUserId, - accessToken: encrypt(tokens.access_token), - refreshToken: encrypt(tokens.refresh_token), - tokenExpiresAt: expiresAt, - scope: tokens.scope ?? WHOOP_OAUTH_SCOPE, - backfillCompletedAt: null, - }, - create: { - userId, - whoopUserId, - accessToken: encrypt(tokens.access_token), - refreshToken: encrypt(tokens.refresh_token), - tokenExpiresAt: expiresAt, - scope: tokens.scope ?? WHOOP_OAUTH_SCOPE, - }, + const creds = await getUserWhoopCredentials(userId); + if (!creds) { + return outcomeRedirect(returnScheme, "error", "nocreds"); + } + + const tokens = await exchangeCode(code, creds); + const profile = await fetchProfile(tokens.access_token); + const whoopUserId = String(profile.user_id); + const expiresAt = new Date(Date.now() + tokens.expires_in * 1000); + const owner = await prisma.whoopConnection.findUnique({ + where: { whoopUserId }, + select: { userId: true }, }); - } catch (err) { - if (isP2002(err)) { + if (owner && owner.userId !== userId) { annotate({ meta: { reason: "owner_conflict" } }); return outcomeRedirect(returnScheme, "error", "owner_conflict"); } - throw err; + + try { + await prisma.whoopConnection.upsert({ + where: { userId }, + update: { + whoopUserId, + accessToken: encrypt(tokens.access_token), + refreshToken: encrypt(tokens.refresh_token), + tokenExpiresAt: expiresAt, + scope: tokens.scope ?? WHOOP_OAUTH_SCOPE, + backfillCompletedAt: null, + }, + create: { + userId, + whoopUserId, + accessToken: encrypt(tokens.access_token), + refreshToken: encrypt(tokens.refresh_token), + tokenExpiresAt: expiresAt, + scope: tokens.scope ?? WHOOP_OAUTH_SCOPE, + }, + }); + } catch (err) { + if (isP2002(err)) { + annotate({ meta: { reason: "owner_conflict" } }); + return outcomeRedirect(returnScheme, "error", "owner_conflict"); + } + throw err; + } + + await auditLog("whoop.connect", { userId }); + + // Re-completing OAuth clears any prior reauth-required state. + await markReconnected(userId, "whoop"); + + // Enqueue the self-converging history backfill. Best-effort: the boot-time + // discovery query (`backfillCompletedAt IS NULL`) is the safety net, so a + // missing boss instance here doesn't strand the connection. + const boss = getGlobalBoss(); + if (boss) { + await boss + .send(WHOOP_BACKFILL_QUEUE, { + userId, + enqueuedAt: new Date().toISOString(), + }) + .catch((err) => + getEvent()?.addWarning(`whoop-backfill enqueue failed: ${err}`), + ); + } + + const response = outcomeRedirect(returnScheme, "connected"); + response.cookies.delete(WHOOP_OAUTH_STATE_COOKIE); + return response; + } catch (err) { + getEvent()?.setError(err); + return outcomeRedirect(returnScheme, "error", "token"); } - - await auditLog("whoop.connect", { userId }); - - // Re-completing OAuth clears any prior reauth-required state. - await markReconnected(userId, "whoop"); - - // Enqueue the self-converging history backfill. Best-effort: the boot-time - // discovery query (`backfillCompletedAt IS NULL`) is the safety net, so a - // missing boss instance here doesn't strand the connection. - const boss = getGlobalBoss(); - if (boss) { - await boss - .send(WHOOP_BACKFILL_QUEUE, { - userId, - enqueuedAt: new Date().toISOString(), - }) - .catch((err) => - getEvent()?.addWarning(`whoop-backfill enqueue failed: ${err}`), - ); - } - - const response = outcomeRedirect(returnScheme, "connected"); - response.cookies.delete(WHOOP_OAUTH_STATE_COOKIE); - return response; - } catch (err) { - getEvent()?.setError(err); - return outcomeRedirect(returnScheme, "error", "token"); - } + }); }); diff --git a/src/app/api/withings/callback/__tests__/route.test.ts b/src/app/api/withings/callback/__tests__/route.test.ts index f63707762..12a5da528 100644 --- a/src/app/api/withings/callback/__tests__/route.test.ts +++ b/src/app/api/withings/callback/__tests__/route.test.ts @@ -20,6 +20,17 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; +// Single-use claiming is covered against Postgres in +// tests/integration/oauth-callback-single-use.test.ts. Here the state row is +// consumed for real against the mocked client, the claim always wins, and +// every callback is the first and only one. +vi.mock("@/lib/oauth/callback-claim", async (importOriginal) => ({ + ...(await importOriginal()), + settleOAuthCallback: (_p: string, _s: string, run: () => Promise) => + run(), + duplicateOAuthCallback: async () => null, +})); + vi.mock("@/lib/api-handler", () => ({ apiHandler: unknown>(fn: T) => fn, requireAuth: vi.fn(async () => ({ @@ -36,16 +47,19 @@ vi.mock("@/lib/api-handler", () => ({ }, })); -vi.mock("@/lib/db", () => ({ - prisma: { +vi.mock("@/lib/db", () => { + const prisma = { withingsOAuthState: { delete: vi.fn(), }, withingsConnection: { upsert: vi.fn(), }, - }, -})); + oAuthCallbackClaim: { createMany: vi.fn(async () => ({ count: 1 })) }, + $transaction: async (run: (tx: unknown) => unknown) => run(prisma), + }; + return { prisma }; +}); vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() })); diff --git a/src/app/api/withings/callback/route.ts b/src/app/api/withings/callback/route.ts index e3c7d169c..faed894ae 100644 --- a/src/app/api/withings/callback/route.ts +++ b/src/app/api/withings/callback/route.ts @@ -9,6 +9,11 @@ import { WITHINGS_OAUTH_STATE_COOKIE } from "@/lib/withings/oauth-state"; import { setupWebhook } from "@/lib/withings/sync"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; +import { + consumeLedgerState, + duplicateOAuthCallback, + settleOAuthCallback, +} from "@/lib/oauth/callback-claim"; import { NextRequest, NextResponse } from "next/server"; import { timingSafeEqual } from "node:crypto"; @@ -39,6 +44,12 @@ import { timingSafeEqual } from "node:crypto"; * carries only the opaque nonce, so a future refactor flipping it to * non-httpOnly cannot leak the user id into request logs / network * captures. + * + * A repeat of a consumed nonce is told apart from a replay by the callback + * claim taken in the same transaction (`src/lib/oauth/callback-claim.ts`): a + * browser that delivers the callback twice gets one exchange, and the repeat + * follows the first request to the same page. Only a nonce with no claim left + * lands on `replay`. */ export const GET = apiHandler(async (request: NextRequest) => { const { user } = await requireAuth(); @@ -88,15 +99,25 @@ export const GET = apiHandler(async (request: NextRequest) => { // lookup before either issues the delete. let stateRow: { userId: string; expiresAt: Date } | null = null; try { - stateRow = await prisma.withingsOAuthState.delete({ - where: { nonce: state }, - }); + stateRow = await consumeLedgerState("withings", state, (tx) => + tx.withingsOAuthState.delete({ where: { nonce: state } }), + ); } catch (err) { if ( err instanceof Prisma.PrismaClientKnownRequestError && err.code === "P2025" ) { // Row already consumed (replay) or never existed. + // A duplicate of a callback that is consuming or has consumed this + // row: follow that request's result instead of showing an error. + const duplicate = await duplicateOAuthCallback("withings", state, { + fallbackUrl: new URL( + "/settings/integrations", + process.env.NEXT_PUBLIC_APP_URL!, + ).toString(), + stateCookie: WITHINGS_OAUTH_STATE_COOKIE, + }); + if (duplicate) return duplicate; annotate({ meta: { reason: "replay" } }); return NextResponse.redirect( new URL( @@ -125,116 +146,119 @@ export const GET = apiHandler(async (request: NextRequest) => { ); } - // Validity checks against the consumed row. The row is already - // gone from the ledger by this point — single-use semantics — - // whether the expiry tripped, the userId mismatched, or - // everything looked good. - if (stateRow.expiresAt <= new Date()) { - annotate({ meta: { reason: "expired" } }); - return NextResponse.redirect( - new URL( - "/settings/integrations?withings=error&reason=expired", - process.env.NEXT_PUBLIC_APP_URL!, - ), - ); - } - if (stateRow.userId !== user.id) { - annotate({ meta: { reason: "cross_user" } }); - return NextResponse.redirect( - new URL( - "/settings/integrations?withings=error&reason=cross_user", - process.env.NEXT_PUBLIC_APP_URL!, - ), - ); - } - - if (!code) { - return NextResponse.redirect( - new URL( - "/settings/integrations?withings=error&reason=nocode", - process.env.NEXT_PUBLIC_APP_URL!, - ), - ); - } - - try { - const creds = await getUserWithingsCredentials(user.id); - if (!creds) { + const consumed = stateRow; + return settleOAuthCallback("withings", state, async () => { + // Validity checks against the consumed row. The row is already + // gone from the ledger by this point — single-use semantics — + // whether the expiry tripped, the userId mismatched, or + // everything looked good. + if (consumed.expiresAt <= new Date()) { + annotate({ meta: { reason: "expired" } }); return NextResponse.redirect( new URL( - "/settings/integrations?withings=error&reason=nocreds", + "/settings/integrations?withings=error&reason=expired", + process.env.NEXT_PUBLIC_APP_URL!, + ), + ); + } + if (consumed.userId !== user.id) { + annotate({ meta: { reason: "cross_user" } }); + return NextResponse.redirect( + new URL( + "/settings/integrations?withings=error&reason=cross_user", process.env.NEXT_PUBLIC_APP_URL!, ), ); } - const tokens = await exchangeCode(code, creds); + if (!code) { + return NextResponse.redirect( + new URL( + "/settings/integrations?withings=error&reason=nocode", + process.env.NEXT_PUBLIC_APP_URL!, + ), + ); + } - const expiresAt = new Date(Date.now() + tokens.expires_in * 1000); - const subscriptionRetryAt = new Date(); + try { + const creds = await getUserWithingsCredentials(user.id); + if (!creds) { + return NextResponse.redirect( + new URL( + "/settings/integrations?withings=error&reason=nocreds", + process.env.NEXT_PUBLIC_APP_URL!, + ), + ); + } - // v1.4.25 — persist the OAuth scope we requested. The token - // response doesn't echo the granted scopes back as a top-level - // field, but Withings honours the scope param verbatim — there's - // no scope downgrade unless the user explicitly revokes one in - // their Health Mate account, which would fail the token exchange - // entirely. Storing the requested set is therefore a safe - // representation of what the connection holds. - await prisma.withingsConnection.upsert({ - where: { userId: user.id }, - update: { - withingsUserId: tokens.userid, - accessToken: encrypt(tokens.access_token), - refreshToken: encrypt(tokens.refresh_token), - tokenExpiresAt: expiresAt, - scope: WITHINGS_OAUTH_SCOPE, - webhookSubscriptionState: Prisma.DbNull, - webhookSubscriptionRetryAt: subscriptionRetryAt, - // A reconnect may come with another developer app (another plan), so - // what the old grant's plan refused is asked for again. - apiPlanState: Prisma.DbNull, - }, - create: { + const tokens = await exchangeCode(code, creds); + + const expiresAt = new Date(Date.now() + tokens.expires_in * 1000); + const subscriptionRetryAt = new Date(); + + // v1.4.25 — persist the OAuth scope we requested. The token + // response doesn't echo the granted scopes back as a top-level + // field, but Withings honours the scope param verbatim — there's + // no scope downgrade unless the user explicitly revokes one in + // their Health Mate account, which would fail the token exchange + // entirely. Storing the requested set is therefore a safe + // representation of what the connection holds. + await prisma.withingsConnection.upsert({ + where: { userId: user.id }, + update: { + withingsUserId: tokens.userid, + accessToken: encrypt(tokens.access_token), + refreshToken: encrypt(tokens.refresh_token), + tokenExpiresAt: expiresAt, + scope: WITHINGS_OAUTH_SCOPE, + webhookSubscriptionState: Prisma.DbNull, + webhookSubscriptionRetryAt: subscriptionRetryAt, + // A reconnect may come with another developer app (another plan), so + // what the old grant's plan refused is asked for again. + apiPlanState: Prisma.DbNull, + }, + create: { + userId: user.id, + withingsUserId: tokens.userid, + accessToken: encrypt(tokens.access_token), + refreshToken: encrypt(tokens.refresh_token), + tokenExpiresAt: expiresAt, + scope: WITHINGS_OAUTH_SCOPE, + webhookSubscriptionState: Prisma.DbNull, + webhookSubscriptionRetryAt: subscriptionRetryAt, + }, + }); + + // Reconcile every category from a clean state after connect/reconnect. + setupWebhook(user.id).catch(() => + getEvent()?.addWarning("Webhook setup failed"), + ); + + await auditLog("withings.connect", { userId: user.id, - withingsUserId: tokens.userid, - accessToken: encrypt(tokens.access_token), - refreshToken: encrypt(tokens.refresh_token), - tokenExpiresAt: expiresAt, - scope: WITHINGS_OAUTH_SCOPE, - webhookSubscriptionState: Prisma.DbNull, - webhookSubscriptionRetryAt: subscriptionRetryAt, - }, - }); + details: { withingsUserId: tokens.userid }, + }); - // Reconcile every category from a clean state after connect/reconnect. - setupWebhook(user.id).catch(() => - getEvent()?.addWarning("Webhook setup failed"), - ); + // Re-completing OAuth clears any prior `error_reauth` state. We + // don't write a fresh success-time — that's the next sync's job. + await markReconnected(user.id, "withings"); - await auditLog("withings.connect", { - userId: user.id, - details: { withingsUserId: tokens.userid }, - }); - - // Re-completing OAuth clears any prior `error_reauth` state. We - // don't write a fresh success-time — that's the next sync's job. - await markReconnected(user.id, "withings"); - - const response = NextResponse.redirect( - new URL( - "/settings/integrations?withings=connected", - process.env.NEXT_PUBLIC_APP_URL!, - ), - ); - response.cookies.delete(WITHINGS_OAUTH_STATE_COOKIE); - return response; - } catch { - getEvent()?.setError(new Error("Withings callback failed")); - return NextResponse.redirect( - new URL( - "/settings/integrations?withings=error&reason=token", - process.env.NEXT_PUBLIC_APP_URL!, - ), - ); - } + const response = NextResponse.redirect( + new URL( + "/settings/integrations?withings=connected", + process.env.NEXT_PUBLIC_APP_URL!, + ), + ); + response.cookies.delete(WITHINGS_OAUTH_STATE_COOKIE); + return response; + } catch { + getEvent()?.setError(new Error("Withings callback failed")); + return NextResponse.redirect( + new URL( + "/settings/integrations?withings=error&reason=token", + process.env.NEXT_PUBLIC_APP_URL!, + ), + ); + } + }); }); diff --git a/src/components/insights/__tests__/recovery-empty-state.test.tsx b/src/components/insights/__tests__/recovery-empty-state.test.tsx index fdc4547b8..74e1c5dca 100644 --- a/src/components/insights/__tests__/recovery-empty-state.test.tsx +++ b/src/components/insights/__tests__/recovery-empty-state.test.tsx @@ -80,7 +80,7 @@ beforeEach(() => { describe(" empty note", () => { it.each([ - ["oura_permission", "heart-health access"], + ["oura_permission", "stress permission"], ["general", "WHOOP or Polar"], ])("renders the server's %s reason with the one action", (reason, text) => { const html = render(reason); diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 436278879..f02b0d0e7 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -348,6 +348,8 @@ export const INSTANCE_SCOPED: Readonly> = { EncryptionKeyCanary: "one known value per configured encryption key id, checked at boot; belongs to the instance's keys, not to any account", RateLimit: "instance-wide rate-limit buckets, keyed by string, self-expiring", + OAuthCallbackClaim: + "single-use claims on in-flight OAuth callbacks, keyed on a hash of the state, minutes-lived and swept by expiry; no user id", HostMetric: "instance-wide host telemetry, not attributable to an account", OffhostBackupKeyUse: "per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account", diff --git a/src/lib/insights/recovery-empty.ts b/src/lib/insights/recovery-empty.ts index c824d300e..40d48065d 100644 --- a/src/lib/insights/recovery-empty.ts +++ b/src/lib/insights/recovery-empty.ts @@ -9,9 +9,10 @@ * - `oura_permission`: Oura is connected and the stored grant leaves out * the scope its resilience level sits behind; one reconnect fixes it. * - `oura_no_resilience`: Oura is connected and no level has arrived. The - * server cannot tell "the grant covers it" from "not recorded yet" (the - * scope note is cleared on every reconnect and written only by a sync - * that reached Oura), so this case never claims coverage. + * server cannot always tell "the grant covers it" from "not recorded yet" + * (the scope note is rewritten on every reconnect from the scopes Oura + * reports, when it reports them, and otherwise only by a sync that + * reached Oura), so this case never claims coverage. * - `general`: anyone else. * * `null` means the page has something to chart. diff --git a/src/lib/integrations/oauth-refresh.ts b/src/lib/integrations/oauth-refresh.ts index 89c424051..ca0da45e3 100644 --- a/src/lib/integrations/oauth-refresh.ts +++ b/src/lib/integrations/oauth-refresh.ts @@ -61,7 +61,7 @@ export async function persistRotatedToken( return opts.readPeerAccessToken(); } -export type SerializedRefreshProvider = "whoop" | "withings"; +export type SerializedRefreshProvider = "whoop" | "withings" | "oura"; /** * Provider refreshes hold a database transaction open around one bounded diff --git a/src/lib/oauth/callback-claim.ts b/src/lib/oauth/callback-claim.ts new file mode 100644 index 000000000..dd88588d0 --- /dev/null +++ b/src/lib/oauth/callback-claim.ts @@ -0,0 +1,207 @@ +/** + * Single-use OAuth callbacks for every provider connect flow. + * + * A browser can deliver the same callback twice, a few hundred milliseconds + * apart, with the same `code` and `state`. An authorization code works once + * (RFC 6749 4.1.2), so the second exchange is refused, and the same section + * tells the authorization server to revoke what the first exchange issued. + * Oura does: the connection then fails its first sync with 401 everywhere and + * asks for a reconnect. + * + * The callback therefore claims its `state` before it touches the code. The + * claim is an insert that skips on conflict, so of any number of concurrent + * requests exactly one wins and exchanges the code. Every other request is a + * duplicate: it waits briefly for the winner to finish and sends the browser to + * the same place, success page or error page, without reading or writing any + * token. When the winner has not finished in time, the duplicate lands on the + * integrations page, which shows the connection's real state. + * + * Where the claim sits differs by flow: + * - signed-state flows (Oura, Polar, Strava) claim right after the signature + * check, so only a state this server minted can create a row; + * - ledger flows (WHOOP, Withings, Fitbit, Google Health) already consume + * their state row with an atomic delete. The claim rides the same + * transaction (`consumeLedgerState`), so a duplicate's delete waits on the + * row lock, finds the row gone, and then sees the committed claim. + * + * A failed exchange settles the claim with its error page. Trying again means + * starting a new connect, which mints a new state and so a new claim. + */ +import { createHash } from "node:crypto"; +import { NextResponse } from "next/server"; +import { prisma } from "@/lib/db"; +import type { Prisma } from "@/generated/prisma/client"; +import { annotate, getEvent } from "@/lib/logging/context"; + +/** Providers whose callbacks claim their state here. */ +export type OAuthCallbackProvider = + | "oura" + | "polar" + | "strava" + | "whoop" + | "withings" + | "fitbit" + | "google-health"; + +/** + * How long a claim is kept. Longer than any state lives (10 minutes), so a + * state that still verifies can never find its claim already swept. + */ +const OAUTH_CALLBACK_CLAIM_TTL_MS = 15 * 60 * 1000; + +/** How long a duplicate waits for the winning request to finish. */ +const DUPLICATE_WAIT_MS = 10_000; +const DUPLICATE_POLL_MS = 150; + +/** Result URLs are short app paths; anything longer is not stored. */ +const MAX_LOCATION_LENGTH = 2048; + +type ClaimDb = Pick; + +function claimKey(provider: OAuthCallbackProvider, state: string): string { + return createHash("sha256").update(`${provider}\0${state}`).digest("hex"); +} + +/** + * Claim one callback. True for the one request that may exchange the code, + * false for every duplicate. `db` lets a ledger flow claim inside the + * transaction that consumes its state row. + */ +export async function claimOAuthCallback( + provider: OAuthCallbackProvider, + state: string, + db: ClaimDb = prisma, +): Promise { + const { count } = await db.oAuthCallbackClaim.createMany({ + data: [ + { + key: claimKey(provider, state), + provider, + expiresAt: new Date(Date.now() + OAUTH_CALLBACK_CLAIM_TTL_MS), + }, + ], + skipDuplicates: true, + }); + return count === 1; +} + +/** + * Consume a ledger flow's state row and claim the callback in one + * transaction. `consume` is the flow's own atomic delete; it throws P2025 for + * a row already consumed, which aborts the claim with it. + */ +export async function consumeLedgerState( + provider: OAuthCallbackProvider, + state: string, + consume: (tx: Prisma.TransactionClient) => Promise, +): Promise { + return prisma.$transaction(async (tx) => { + const row = await consume(tx); + await claimOAuthCallback(provider, state, tx); + return row; + }); +} + +/** + * Run the claimed callback and record where it sent the browser, so a + * duplicate can follow. A throw is recorded as settled without a result, and + * re-thrown for the route's own error handling. + */ +export async function settleOAuthCallback( + provider: OAuthCallbackProvider, + state: string, + run: () => Promise, +): Promise { + let location: string | null = null; + try { + const response = await run(); + location = response.headers.get("location"); + return response; + } finally { + const now = new Date(); + try { + await prisma.oAuthCallbackClaim.updateMany({ + where: { key: claimKey(provider, state) }, + data: { + settledAt: now, + location: + location && location.length <= MAX_LOCATION_LENGTH + ? location + : null, + }, + }); + await prisma.oAuthCallbackClaim.deleteMany({ + where: { expiresAt: { lt: now } }, + }); + } catch (err) { + getEvent()?.addWarning(`oauth callback claim settle failed: ${err}`); + } + } +} + +/** + * Answer a duplicate callback: wait for the claiming request to finish and + * redirect to the same place, or to `fallbackUrl` when it does not finish in + * time or finished without a result. Returns null when no claim exists for + * this state at all, which a ledger flow reads as a genuine replay. + * + * Never touches tokens. Clears the state cookie like the winning request does, + * since the browser may render this response instead of that one. + */ +export async function duplicateOAuthCallback( + provider: OAuthCallbackProvider, + state: string, + opts: { + fallbackUrl: string; + stateCookie: string; + waitMs?: number; + }, +): Promise { + const key = claimKey(provider, state); + const deadline = Date.now() + (opts.waitMs ?? DUPLICATE_WAIT_MS); + let row = await prisma.oAuthCallbackClaim.findUnique({ + where: { key }, + select: { settledAt: true, location: true }, + }); + if (!row) return null; + while (!row?.settledAt && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, DUPLICATE_POLL_MS)); + row = await prisma.oAuthCallbackClaim.findUnique({ + where: { key }, + select: { settledAt: true, location: true }, + }); + } + const settled = Boolean(row?.settledAt); + annotate({ + meta: { + reason: "duplicate_callback", + duplicate_callback_settled: settled, + }, + }); + const response = NextResponse.redirect( + settled && row?.location ? row.location : opts.fallbackUrl, + ); + response.cookies.delete(opts.stateCookie); + return response; +} + +/** + * The whole single-use step for a flow whose state carries no ledger row: + * claim, then either run the callback and record its result, or answer as the + * duplicate. + */ +export async function runOAuthCallbackOnce( + provider: OAuthCallbackProvider, + state: string, + opts: { fallbackUrl: string; stateCookie: string }, + run: () => Promise, +): Promise { + if (await claimOAuthCallback(provider, state)) { + return settleOAuthCallback(provider, state, run); + } + const duplicate = await duplicateOAuthCallback(provider, state, opts); + if (duplicate) return duplicate; + const response = NextResponse.redirect(opts.fallbackUrl); + response.cookies.delete(opts.stateCookie); + return response; +} diff --git a/src/lib/oauth/signed-state.ts b/src/lib/oauth/signed-state.ts index 0abc5f87b..fdfc4c317 100644 --- a/src/lib/oauth/signed-state.ts +++ b/src/lib/oauth/signed-state.ts @@ -139,7 +139,7 @@ export function verifySignedState( export function stateMatchesCookie( urlState: string | null | undefined, cookieState: string | null | undefined, -): boolean { +): urlState is string { if (!urlState || !cookieState) return false; if (urlState.length !== cookieState.length) return false; return timingSafeEqual(Buffer.from(urlState), Buffer.from(cookieState)); diff --git a/src/lib/openapi/routes/integrations.ts b/src/lib/openapi/routes/integrations.ts index 9e36dfcb6..d0e0500b2 100644 --- a/src/lib/openapi/routes/integrations.ts +++ b/src/lib/openapi/routes/integrations.ts @@ -1146,7 +1146,7 @@ const integrationEnvelopeEntry = z .optional() .meta( legacy( - "Oura only. Collections the stored grant does not cover: they answered 401 while sibling collections succeeded with the same token. Not a failure and not counted toward the reconnect threshold; one reconnect asks for the missing scopes. Empty when the grant covers everything HealthLog reads.", + "Oura only. Collections the stored grant does not cover: Oura refused them for a missing scope (named in its 401, or inferred when sibling collections succeeded with the same token), or the scopes Oura reported at connect leave theirs out. Not a failure and not counted toward the reconnect threshold; one reconnect asks for the missing scopes. Empty when the grant covers everything HealthLog reads.", ), ), hasToken: z.boolean().optional().describe("Nightscout only."), diff --git a/src/lib/oura/__tests__/client.test.ts b/src/lib/oura/__tests__/client.test.ts index e4c6148df..13535d03c 100644 --- a/src/lib/oura/__tests__/client.test.ts +++ b/src/lib/oura/__tests__/client.test.ts @@ -2,7 +2,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { OURA_OAUTH_LEGACY_TOKEN_URL, - OURA_OAUTH_SCOPE, OURA_OAUTH_TOKEN_URL, RESILIENCE_LEVELS, derivePeriodDaysFromCyclePhases, @@ -78,7 +77,7 @@ describe("getAuthorizationUrl", () => { it("builds the Oura authorize URL", () => { const url = getAuthorizationUrl("signed", CREDS); expect(url).toContain("cloud.ouraring.com/oauth/authorize"); - expect(url).toContain("scope=daily+personal"); + expect(url).toContain("scope=daily+spo2+heart_health+stress+personal"); expect(url).toContain("state=signed"); }); }); @@ -704,14 +703,44 @@ describe("mapResilience", () => { }); }); -describe("OURA_OAUTH_SCOPE", () => { - it("requests daily, personal and the scopes behind SpO2 and heart health", () => { - expect(OURA_OAUTH_SCOPE.split(" ")).toEqual([ - "daily", - "personal", - "spo2", - "heart_health", +describe("collection refusals", () => { + const query = { startDate: "2026-06-01", endDate: "2026-06-10" }; + + it("reads a 401 that names a scope as a missing permission, not a dead token", async () => { + installFetchMock([ + { + status: 401, + body: { detail: "Token is not authorized access stress scope." }, + }, ]); + const err = await fetchResilience("tok", query).catch((e) => e); + expect(err).toBeInstanceOf(OuraApiError); + expect(err.httpStatus).toBe(401); + expect(err.missingScope).toBe("stress"); + expect(err.reason).toBe("scope_missing"); + }); + + it("keeps a dead token's 401 a token refusal", async () => { + installFetchMock([ + { + status: 401, + body: { + detail: + "The access token provided is expired, revoked, malformed, or invalid for other reasons.", + }, + }, + ]); + const err = await fetchReadiness("tok", query).catch((e) => e); + expect(err.httpStatus).toBe(401); + expect(err.missingScope).toBeUndefined(); + expect(err.classification).toBe("reauth_required"); + }); + + it("classifies a 403 (lapsed membership) as persistent rather than a reconnect", async () => { + installFetchMock([{ status: 403, body: { detail: "forbidden" } }]); + const err = await fetchReadiness("tok", query).catch((e) => e); + expect(err.httpStatus).toBe(403); + expect(err.classification).toBe("persistent"); }); }); diff --git a/src/lib/oura/__tests__/scope.test.ts b/src/lib/oura/__tests__/scope.test.ts new file mode 100644 index 000000000..caf2bdfab --- /dev/null +++ b/src/lib/oura/__tests__/scope.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; + +import { + OURA_COLLECTIONS, + OURA_OAUTH_SCOPE, + OURA_SCOPE_GATED_COLLECTIONS, + ouraCollectionsMissingFromGrant, + ouraScopeReconnectList, + parseOuraGrantedScopes, + parseOuraMissingScope, +} from "../scope"; + +describe("Oura scope map", () => { + it("asks for the scope of every collection the sync reads", () => { + const requested = new Set(OURA_OAUTH_SCOPE.split(" ")); + for (const c of OURA_COLLECTIONS) expect(requested).toContain(c.scope); + expect(requested).toContain("personal"); + }); + + it("puts resilience behind stress, not heart_health", () => { + const scopeOf = Object.fromEntries( + OURA_COLLECTIONS.map((c) => [c.name, c.scope]), + ); + expect(scopeOf.resilience).toBe("stress"); + expect(scopeOf.spo2).toBe("spo2"); + expect(scopeOf.vo2max).toBe("heart_health"); + expect(scopeOf.cardio_age).toBe("heart_health"); + expect(OURA_OAUTH_SCOPE.split(" ")).toEqual([ + "daily", + "spo2", + "heart_health", + "stress", + "personal", + ]); + }); + + it("derives the gated collections from the map, in display order", () => { + expect([...OURA_SCOPE_GATED_COLLECTIONS]).toEqual([ + "spo2", + "vo2max", + "cardio_age", + "resilience", + ]); + expect(ouraScopeReconnectList(["resilience", "readiness", "spo2"])).toEqual( + ["spo2", "resilience"], + ); + }); +}); + +describe("granted scopes", () => { + it("reads Oura's extapi-prefixed grant", () => { + expect( + parseOuraGrantedScopes("extapi:daily extapi:personal extapi:spo2"), + ).toEqual(new Set(["daily", "personal", "spo2"])); + }); + + it("knows nothing when the response names no scopes", () => { + expect(parseOuraGrantedScopes(undefined)).toBeNull(); + expect(parseOuraGrantedScopes(" ")).toBeNull(); + expect(ouraCollectionsMissingFromGrant(undefined)).toBeNull(); + }); + + it("lists the gated collections a grant leaves out", () => { + expect( + ouraCollectionsMissingFromGrant( + "extapi:daily extapi:personal extapi:spo2 extapi:heart_health", + ), + ).toEqual(["resilience"]); + expect( + ouraCollectionsMissingFromGrant( + "daily personal spo2 heart_health stress", + ), + ).toEqual([]); + }); +}); + +describe("parseOuraMissingScope", () => { + it("reads the scope a refusal names", () => { + expect( + parseOuraMissingScope({ + detail: "Token is not authorized access stress scope.", + }), + ).toBe("stress"); + expect( + parseOuraMissingScope({ + detail: "Token is not authorized access extapi:heart_health scope.", + }), + ).toBe("heart_health"); + }); + + it("returns null for a dead token or any other body", () => { + expect( + parseOuraMissingScope({ + detail: + "The access token provided is expired, revoked, malformed, or invalid for other reasons.", + }), + ).toBeNull(); + expect(parseOuraMissingScope(null)).toBeNull(); + expect(parseOuraMissingScope({ detail: 42 })).toBeNull(); + }); +}); diff --git a/src/lib/oura/__tests__/sync.test.ts b/src/lib/oura/__tests__/sync.test.ts index 1f2d22728..451084b43 100644 --- a/src/lib/oura/__tests__/sync.test.ts +++ b/src/lib/oura/__tests__/sync.test.ts @@ -64,9 +64,25 @@ const { scopeDeniedMock: vi.fn(), })); +// The serialised refresh itself is covered against Postgres in +// `tests/integration/oura-refresh-serialization.test.ts`; here it stands in as +// "refresh with the client, then persist", so the sync's own branching is what +// these cases pin. vi.mock("../credentials", () => ({ getOuraConnection: getConnMock, - storeOuraTokens: storeTokensMock, + refreshOuraAccessToken: async ( + userId: string, + spentCiphertext: string, + creds: unknown, + ) => { + const rotated = await refreshMock("ref", creds); + return storeTokensMock( + userId, + rotated.access_token, + rotated.refresh_token, + spentCiphertext, + ); + }, getOuraClientCredentials: getCredsMock, syncOuraScopeDenied: scopeDeniedMock, })); @@ -145,7 +161,6 @@ const DAY_MS = 24 * 60 * 60 * 1000; const CONN = { accessToken: "acc", - refreshToken: "ref", refreshTokenCiphertext: "enc(ref)", }; @@ -230,6 +245,26 @@ function err401() { }); } +/** The 401 Oura sends when the token is fine but the grant lacks `scope`. */ +function errScope401(scope: string) { + return new OuraApiError({ + verb: "fetchResilience", + classification: "reauth_required", + httpStatus: 401, + reason: "scope_missing", + missingScope: scope, + }); +} + +function err403() { + return new OuraApiError({ + verb: "fetchReadiness", + classification: "persistent", + httpStatus: 403, + reason: "http_403", + }); +} + function createdRows() { return reconcileMock.mock.calls.map((call) => call[1]); } @@ -1128,3 +1163,127 @@ describe("syncUserOura — a grant without the spo2 / heart_health scopes", () = expect(scopeDeniedMock).not.toHaveBeenCalled(); }); }); + +describe("syncUserOura — Oura names the missing scope", () => { + it("records resilience as a permission gap without a refresh, even when nothing else succeeded", async () => { + getConnMock.mockResolvedValue(CONN); + // Every core collection is merely empty: the token is never proven by a + // sibling, only by Oura naming the scope it lacks. + fetchReadinessMock.mockRejectedValue( + new OuraApiError({ + verb: "fetchReadiness", + classification: "transient", + httpStatus: 503, + reason: "http_503", + }), + ); + fetchSleepMock.mockRejectedValue( + new OuraApiError({ + verb: "fetchSleep", + classification: "transient", + httpStatus: 503, + reason: "http_503", + }), + ); + fetchActivityMock.mockRejectedValue( + new OuraApiError({ + verb: "fetchDailyActivity", + classification: "transient", + httpStatus: 503, + reason: "http_503", + }), + ); + fetchDailySleepMock.mockRejectedValue( + new OuraApiError({ + verb: "fetchDailySleep", + classification: "transient", + httpStatus: 503, + reason: "http_503", + }), + ); + fetchSpo2Mock.mockRejectedValue(errScope401("spo2")); + fetchVo2MaxMock.mockRejectedValue(errScope401("heart_health")); + fetchCardioAgeMock.mockRejectedValue(errScope401("heart_health")); + fetchResilienceMock.mockRejectedValue(errScope401("stress")); + + await syncUserOura("u1"); + + expect(refreshMock).not.toHaveBeenCalled(); + expect(scopeDeniedMock).toHaveBeenCalledWith("u1", [ + "spo2", + "vo2max", + "cardio_age", + "resilience", + ]); + // The outage stays a failure; the permission gap never joins it. + const failure = recordFailureMock.mock.calls[0]![0] as { message: string }; + expect(failure.message).not.toContain("resilience"); + }); + + it("spends no refresh when the only 401 names a scope", async () => { + getConnMock.mockResolvedValue(CONN); + fetchReadinessMock.mockResolvedValue([ + { id: "x", day: "2026-06-10", score: 70 }, + ]); + fetchResilienceMock.mockRejectedValue(errScope401("stress")); + + const result = await syncUserOura("u1"); + + expect(refreshMock).not.toHaveBeenCalled(); + expect(result).toEqual({ imported: 1, failed: false }); + expect(scopeDeniedMock).toHaveBeenCalledWith("u1", ["resilience"]); + expect(recordSuccessMock).toHaveBeenCalledWith("u1", "oura"); + }); + + it("does not retry a named-scope refusal after a refresh another collection needed", async () => { + getConnMock.mockResolvedValue(CONN); + fetchReadinessMock + .mockRejectedValueOnce(err401()) + .mockResolvedValueOnce([{ id: "x", day: "2026-06-10", score: 70 }]); + fetchResilienceMock.mockRejectedValue(errScope401("stress")); + refreshMock.mockResolvedValue({ + access_token: "newAcc", + refresh_token: "newRef", + expires_in: 86400, + }); + + const result = await syncUserOura("u1"); + + expect(refreshMock).toHaveBeenCalledTimes(1); + expect(fetchResilienceMock).toHaveBeenCalledTimes(1); + expect(result.failed).toBe(false); + expect(scopeDeniedMock).toHaveBeenCalledWith("u1", ["resilience"]); + }); + + it("keeps a named-scope 401 on a core collection a failure", async () => { + getConnMock.mockResolvedValue(CONN); + fetchReadinessMock.mockRejectedValue(errScope401("daily")); + fetchSleepMock.mockRejectedValue(errScope401("daily")); + fetchActivityMock.mockRejectedValue(errScope401("daily")); + fetchDailySleepMock.mockRejectedValue(errScope401("daily")); + + const result = await syncUserOura("u1"); + + expect(refreshMock).not.toHaveBeenCalled(); + expect(result.failed).toBe(true); + expect(recordFailureMock).toHaveBeenCalledWith( + expect.objectContaining({ integration: "oura", kind: "reauth_required" }), + ); + }); + + it("does not refresh over a 403 and does not count it toward a reconnect", async () => { + getConnMock.mockResolvedValue(CONN); + fetchReadinessMock.mockResolvedValue([ + { id: "x", day: "2026-06-10", score: 70 }, + ]); + fetchSleepMock.mockRejectedValue(err403()); + + const result = await syncUserOura("u1"); + + expect(refreshMock).not.toHaveBeenCalled(); + expect(result.failed).toBe(true); + expect(recordFailureMock).toHaveBeenCalledWith( + expect.objectContaining({ integration: "oura", kind: "persistent" }), + ); + }); +}); diff --git a/src/lib/oura/client.ts b/src/lib/oura/client.ts index cf21d02bd..ed99a0505 100644 --- a/src/lib/oura/client.ts +++ b/src/lib/oura/client.ts @@ -23,6 +23,7 @@ import { OuraApiError, classifyOuraResponse } from "./response-classifier"; import { envOr, envValue } from "@/lib/env"; import { importValue } from "@/lib/integrations/import-value"; import { walkPages } from "@/lib/integrations/page-walk"; +import { OURA_OAUTH_SCOPE, parseOuraMissingScope } from "./scope"; const OURA_API_BASE = "https://api.ouraring.com"; const OURA_OAUTH_AUTH_URL = "https://cloud.ouraring.com/oauth/authorize"; @@ -112,24 +113,11 @@ export function getOuraRedirectUri(): string { } /** - * The scopes HealthLog asks Oura for. - * - * - `daily` : readiness, sleep, daily sleep, daily activity - * - `personal` : profile info - * - `spo2` : `daily_spo2` - * - `heart_health` : `vO2_max`, `daily_cardiovascular_age`, - * `daily_resilience` - * - * Oura's current authorization server enforces these per collection: a grant - * without `spo2` or `heart_health` answers 401 on those collections while every - * other one succeeds. The published OpenAPI security scheme (openapi-1.41) - * lists `spo2` and `heart_health` but maps no endpoint to them; the mapping - * above is what the live API enforces, as recorded by integrations that probed - * it. Grants issued before these were requested keep the old set until the - * user reconnects once — see `syncOuraScopeDenied`. + * The consent-screen URL. The scopes come from `./scope`, which maps every + * collection HealthLog reads to the scope Oura enforces for it. A grant issued + * before a scope was added keeps the old set until the person reconnects once; + * the sync records what it lacks (`syncOuraScopeDenied`). */ -export const OURA_OAUTH_SCOPE = "daily personal spo2 heart_health" as const; - export function getAuthorizationUrl( state: string, creds: OuraCredentials, @@ -471,6 +459,51 @@ interface DateRangeQuery { maxPages?: number; } +/** + * The error for a refused collection read. Oura's status alone does not say + * what went wrong; its body does (see `./scope`): + * + * - 401 naming a scope: the token is fine, the grant does not cover this + * collection. `missingScope` carries the name, and the sync neither + * refreshes over it nor counts it toward the reconnect threshold. + * - any other 401: the token is dead or expired (refresh, then reconnect). + * - 403: a lapsed Oura membership, per the API reference. A reconnect does + * not fix that, so it is a persistent failure rather than a reauth. + */ +function collectionRefusal( + verb: string, + status: number, + verdict: ReturnType, + body: unknown, +): OuraApiError { + if (status === 401) { + const missingScope = parseOuraMissingScope(body); + if (missingScope) { + return new OuraApiError({ + verb, + classification: "reauth_required", + httpStatus: status, + reason: "scope_missing", + missingScope, + }); + } + } + if (status === 403) { + return new OuraApiError({ + verb, + classification: "persistent", + httpStatus: status, + reason: "http_403", + }); + } + return new OuraApiError({ + verb, + classification: verdict.classification, + httpStatus: verdict.httpStatus, + reason: verdict.reason, + }); +} + async function fetchCollection( path: string, accessToken: string, @@ -508,12 +541,7 @@ async function fetchCollection( verdict.classification === "success" ? undefined : verdict.reason, }); if (verdict.classification !== "success") { - throw new OuraApiError({ - verb, - classification: verdict.classification, - httpStatus: verdict.httpStatus, - reason: verdict.reason, - }); + throw collectionRefusal(verb, res.status, verdict, json); } return { items: json?.data ?? [], next: json?.next_token ?? null }; }, diff --git a/src/lib/oura/credentials.ts b/src/lib/oura/credentials.ts index c7088eb70..4804cfeb7 100644 --- a/src/lib/oura/credentials.ts +++ b/src/lib/oura/credentials.ts @@ -13,8 +13,15 @@ import { prisma } from "@/lib/db"; import { Prisma } from "@/generated/prisma/client"; import { decrypt, encrypt } from "@/lib/crypto"; -import { persistRotatedToken } from "@/lib/integrations/oauth-refresh"; -import { getOuraCredentials, type OuraCredentials } from "./client"; +import { + acquireProviderTokenRefreshLock, + PROVIDER_REFRESH_TRANSACTION_OPTIONS, +} from "@/lib/integrations/oauth-refresh"; +import { + getOuraCredentials, + refreshAccessToken, + type OuraCredentials, +} from "./client"; /** * Resolve the user's Oura OAuth client id/secret, DB-first then env. @@ -78,11 +85,10 @@ export async function clearOuraClientCredentials( export interface OuraConnection { accessToken: string; - refreshToken: string; /** - * The exact stored refresh-token ciphertext (NOT the decrypted value). The - * rotating-refresh compare-and-swap in `storeOuraTokens` matches on this so a - * concurrent sync that already rotated the token is not clobbered. + * The exact stored refresh-token ciphertext (NOT the decrypted value). + * `refreshOuraAccessToken` compares against it under the refresh lock to + * tell whether a peer already rotated the token this sync started with. */ refreshTokenCiphertext: string; /** Collections the grant was last seen not to cover (`ouraScopeDenied`). */ @@ -105,53 +111,79 @@ export async function getOuraConnection( } return { accessToken: decrypt(user.ouraAccessTokenEncrypted), - refreshToken: decrypt(user.ouraRefreshTokenEncrypted), refreshTokenCiphertext: user.ouraRefreshTokenEncrypted, scopeDenied: readOuraScopeDenied(user.ouraScopeDenied), }; } /** - * Persist a rotated Oura token pair with compare-and-swap on the stored refresh - * ciphertext (the inverse of Fitbit / WHOOP, which key by connection id). Oura's - * tokens live on `User` and the refresh is REACTIVE on a 401, so two overlapping - * syncs can race the same one-time-use refresh token. The CAS guard writes only - * when the stored ciphertext still equals the one this caller spent; on a lost - * race (0 rows) it re-reads and returns the peer's freshly rotated access token. + * Refresh the user's Oura access token once, however many syncs ask at the + * same moment. * - * Returns the access token the caller should use for the current sync (its own - * on a win, the peer's on a loss), or null if the connection vanished. + * Oura's refresh tokens are single-use, and spending one twice is what an + * authorization server treats as theft: the second request is refused and the + * grant behind it may be revoked, parking a healthy connection at reconnect. + * The refresh is reactive on a 401, so an hourly tick, a manual sync and the + * first sync after a reconnect can all reach it together. A per-user advisory + * lock lets one of them refresh; each later one re-reads the stored refresh + * ciphertext under the lock and, finding it changed from the one its own sync + * started with, takes the access token stored beside it instead of refreshing + * again. The write is still conditional on the ciphertext the refresh spent, + * because a reconnect stores a new grant without taking the lock: a refresh of + * the old grant finishing after it must not put the old grant back. + * + * Returns the access token to retry with, or null when the connection is gone. + * A refused refresh throws for the caller to classify. */ -export async function storeOuraTokens( +export async function refreshOuraAccessToken( userId: string, - accessToken: string, - refreshToken: string, - expectedRefreshCiphertext: string, + spentRefreshCiphertext: string, + creds: OuraCredentials, ): Promise { - return persistRotatedToken(accessToken, { - conditionalUpdate: async () => { - const { count } = await prisma.user.updateMany({ - where: { - id: userId, - ouraRefreshTokenEncrypted: expectedRefreshCiphertext, - }, - data: { - ouraAccessTokenEncrypted: encrypt(accessToken), - ouraRefreshTokenEncrypted: encrypt(refreshToken), - }, - }); - return count; - }, - readPeerAccessToken: async () => { - const fresh = await prisma.user.findUnique({ - where: { id: userId }, - select: { ouraAccessTokenEncrypted: true }, - }); - return fresh?.ouraAccessTokenEncrypted - ? decrypt(fresh.ouraAccessTokenEncrypted) - : null; - }, - }); + return prisma.$transaction(async (tx) => { + await acquireProviderTokenRefreshLock(tx, "oura", userId); + const current = await tx.user.findUnique({ + where: { id: userId }, + select: { + ouraAccessTokenEncrypted: true, + ouraRefreshTokenEncrypted: true, + }, + }); + if ( + !current?.ouraAccessTokenEncrypted || + !current.ouraRefreshTokenEncrypted + ) { + return null; + } + if (current.ouraRefreshTokenEncrypted !== spentRefreshCiphertext) { + // Rotated by a peer, or replaced by a reconnect, while this sync ran. + return decrypt(current.ouraAccessTokenEncrypted); + } + + const rotated = await refreshAccessToken( + decrypt(current.ouraRefreshTokenEncrypted), + creds, + ); + const { count } = await tx.user.updateMany({ + where: { + id: userId, + ouraRefreshTokenEncrypted: spentRefreshCiphertext, + }, + data: { + ouraAccessTokenEncrypted: encrypt(rotated.access_token), + ouraRefreshTokenEncrypted: encrypt(rotated.refresh_token), + }, + }); + if (count > 0) return rotated.access_token; + // A reconnect stored a new grant while Oura answered; that one wins. + const fresh = await tx.user.findUnique({ + where: { id: userId }, + select: { ouraAccessTokenEncrypted: true }, + }); + return fresh?.ouraAccessTokenEncrypted + ? decrypt(fresh.ouraAccessTokenEncrypted) + : null; + }, PROVIDER_REFRESH_TRANSACTION_OPTIONS); } /** diff --git a/src/lib/oura/response-classifier.ts b/src/lib/oura/response-classifier.ts index 9dbb18f0a..46a748ba0 100644 --- a/src/lib/oura/response-classifier.ts +++ b/src/lib/oura/response-classifier.ts @@ -8,6 +8,10 @@ * - `transient` → 429 (rate-limited), 5xx, 3xx, unknown. * - `reauth_required` → 401 / 403 — token rejected / grant revoked. * - `persistent` → any other 4xx — malformed request / contract drift. + * + * A collection read refines this from the body (`collectionRefusal` in + * `./client`): a 401 that names a scope is a missing permission, a 403 a lapsed + * membership. */ import { @@ -26,15 +30,31 @@ export function classifyOuraResponse( } export class OuraApiError extends IntegrationApiError { + /** + * The scope Oura named when it refused a collection read for a missing + * permission (`401 {"detail":"Token is not authorized access stress + * scope."}`). Undefined for every other failure, a dead token included. + */ + readonly missingScope: string | undefined; + constructor(opts: { verb: string; classification: IntegrationClassification; httpStatus: number | undefined; reason: string; upstreamError?: string; + missingScope?: string; }) { - super({ vendor: "Oura", ...opts }); + const { missingScope, ...rest } = opts; + super({ + vendor: "Oura", + ...rest, + upstreamError: + rest.upstreamError ?? + (missingScope ? `scope_missing:${missingScope}` : undefined), + }); this.name = "OuraApiError"; + this.missingScope = missingScope; } } diff --git a/src/lib/oura/scope.ts b/src/lib/oura/scope.ts index e1c74f11b..8356c6246 100644 --- a/src/lib/oura/scope.ts +++ b/src/lib/oura/scope.ts @@ -1,19 +1,106 @@ /** - * Oura collections that sit behind a scope beyond `daily` (`spo2`, - * `heart_health`). A grant issued before HealthLog requested those scopes - * answers 401 on exactly these, so they are the only names the settings card - * can be asked to explain. Each has a label under - * `settings.integrations.oura.scope.items`. + * Which Oura OAuth scope each collection HealthLog reads sits behind. The one + * source for the authorize request (`OURA_OAUTH_SCOPE`), the per-collection + * missing-permission detection in the sync, the "reconnect once to grant the + * new permissions" hint and the settings card. + * + * Oura's published material does not carry this map. The OpenAPI document + * (https://cloud.ouraring.com/v2/static/json/openapi-1.42.json, the spec behind + * https://cloud.ouraring.com/v2/docs) declares nine scopes and binds no + * endpoint to any of them, and its list lacks `stress` altogether; the + * authentication page (https://cloud.ouraring.com/docs/authentication) still + * names `spo2Daily`, which the authorization server ignores. The map below is + * what the live API enforces. It names the scope in its own refusal: + * + * 401 {"detail":"Token is not authorized access stress scope."} + * + * and integrations that read those refusals record the same assignments + * (tidepool-org/platform `oura/oura.go`, `ScopesForDataType`). The developer + * portal lists `stress`, `heart_health` and `spo2` among an app's permissions. + * A dead token reads differently: + * + * 401 {"detail":"The access token provided is expired, revoked, malformed, + * or invalid for other reasons."} + * + * so the body, not the status, tells a missing permission from a dead token + * (`parseOuraMissingScope`). A 403 is a lapsed Oura membership per the spec. + * + * collection endpoint (/v2/usercollection/…) scope + * readiness daily_readiness daily + * sleep sleep daily + * activity daily_activity daily + * daily_sleep daily_sleep daily + * spo2 daily_spo2 spo2 + * vo2max vO2_max heart_health + * cardio_age daily_cardiovascular_age heart_health + * resilience daily_resilience stress + * (profile) personal_info personal + * (cycle) daily_cycle_phases reproductive_cycle + * + * `reproductive_cycle` is not requested: the developer portal does not list it + * for a self-registered app, and asking for a scope the app does not hold + * would risk the whole authorization request. Cycle phases stay a best-effort read (`./cycle-sync`) + * that never touches the connection's health, and no reconnect is suggested + * for them because none would help. */ -export const OURA_SCOPE_GATED_COLLECTIONS = [ - "spo2", - "vo2max", - "cardio_age", - "resilience", + +export const OURA_COLLECTIONS = [ + { name: "readiness", scope: "daily" }, + { name: "sleep", scope: "daily" }, + { name: "activity", scope: "daily" }, + { name: "daily_sleep", scope: "daily" }, + { name: "spo2", scope: "spo2" }, + { name: "vo2max", scope: "heart_health" }, + { name: "cardio_age", scope: "heart_health" }, + { name: "resilience", scope: "stress" }, ] as const; -export type OuraScopeGatedCollection = - (typeof OURA_SCOPE_GATED_COLLECTIONS)[number]; +type OuraCollectionEntry = (typeof OURA_COLLECTIONS)[number]; + +/** The scope the profile probe (`personal_info`) needs. */ +const OURA_PROFILE_SCOPE = "personal"; + +/** + * Every scope HealthLog asks for, in a stable order: the scopes of the + * collections it reads, plus the profile scope. + */ +const OURA_REQUESTED_SCOPES: readonly string[] = [ + ...new Set([ + ...OURA_COLLECTIONS.map((c) => c.scope), + OURA_PROFILE_SCOPE, + ]), +]; + +/** The authorize request's `scope` parameter. */ +export const OURA_OAUTH_SCOPE = OURA_REQUESTED_SCOPES.join(" "); + +/** + * Collections behind a scope beyond `daily`. A grant issued before HealthLog + * requested that scope, or one where the person unticked it on the consent + * screen, answers 401 on exactly these while the rest succeed, so they are the + * only names the settings card is asked to explain. Each has a label under + * `settings.integrations.oura.scope.items`. A collection behind `daily` is not + * here: without `daily` nothing syncs, and that is a reconnect, not a hint. + */ +export type OuraScopeGatedCollection = Exclude< + OuraCollectionEntry, + { scope: "daily" } +>["name"]; + +export const OURA_SCOPE_GATED_COLLECTIONS = OURA_COLLECTIONS.filter( + (c) => c.scope !== "daily", +).map((c) => c.name) as unknown as readonly [ + OuraScopeGatedCollection, + ...OuraScopeGatedCollection[], +]; + +const GATED: ReadonlySet = new Set(OURA_SCOPE_GATED_COLLECTIONS); + +export function isOuraScopeGatedCollection( + name: string, +): name is OuraScopeGatedCollection { + return GATED.has(name); +} /** Narrow a stored list to the known names, in display order. */ export function ouraScopeReconnectList( @@ -21,3 +108,57 @@ export function ouraScopeReconnectList( ): OuraScopeGatedCollection[] { return OURA_SCOPE_GATED_COLLECTIONS.filter((c) => denied.includes(c)); } + +/** + * The scope a granted-scope string or a refusal names, without the `extapi:` + * prefix Oura's authorization server puts on the scopes it grants (a request + * for `daily` comes back as `extapi:daily`). + */ +function bareScope(s: string): string { + return s.startsWith("extapi:") ? s.slice("extapi:".length) : s; +} + +/** + * The scopes a token response says were granted, or null when it carried no + * `scope` (older apps; the field is optional). Space-separated per RFC 6749. + */ +export function parseOuraGrantedScopes( + scope: string | null | undefined, +): Set | null { + if (typeof scope !== "string" || scope.trim() === "") return null; + return new Set(scope.trim().split(/\s+/).map(bareScope)); +} + +/** + * The gated collections a granted-scope string leaves out, in display order. + * Null when the response named no scopes: then nothing is known until the + * first sync reads the collections. + */ +export function ouraCollectionsMissingFromGrant( + scope: string | null | undefined, +): OuraScopeGatedCollection[] | null { + const granted = parseOuraGrantedScopes(scope); + if (!granted) return null; + return OURA_COLLECTIONS.filter( + ( + c, + ): c is Extract => + c.scope !== "daily" && !granted.has(c.scope), + ).map((c) => c.name); +} + +const MISSING_SCOPE_DETAIL = + /not authori[sz]ed (?:to )?access(?: the)? ([a-z0-9_:]{1,60}) scope/i; + +/** + * The scope a 401 body names when Oura refused the request for a missing + * permission (`Token is not authorized access stress scope.`), or null for any + * other body, a dead token's included. + */ +export function parseOuraMissingScope(body: unknown): string | null { + if (body === null || typeof body !== "object") return null; + const detail = (body as { detail?: unknown }).detail; + if (typeof detail !== "string") return null; + const m = MISSING_SCOPE_DETAIL.exec(detail); + return m ? bareScope(m[1]!.toLowerCase()) : null; +} diff --git a/src/lib/oura/sync.ts b/src/lib/oura/sync.ts index 0a5c9b384..b6e6e035c 100644 --- a/src/lib/oura/sync.ts +++ b/src/lib/oura/sync.ts @@ -20,9 +20,11 @@ * unknown / missing level mints no row. See `RESILIENCE_LEVELS` in `./client`. * * Token model: Oura uses refresh tokens. The merged schema has no expiry - * column, so the sync refreshes REACTIVELY — the first read that 401s triggers - * one refresh (persisting BOTH rotated tokens) and a single retry. A failed - * refresh (`invalid_grant`) records `reauth_required` on the `oura` ledger. + * column, so the sync refreshes REACTIVELY — the first read refused for its + * token triggers one refresh (persisting BOTH rotated tokens, serialised per + * user) and a single retry. A 401 that names a missing scope is not a token + * refusal and spends no refresh. A failed refresh (`invalid_grant`) records + * `reauth_required` on the `oura` ledger. * * Idempotency: `externalId = ::` for the day-keyed * collections; sleep rows carry a record-scoped `sleep::` @@ -82,18 +84,17 @@ import { mapResilience, mapSleep, mapVo2Max, - refreshAccessToken, type MappedMeasurement, } from "./client"; import { getOuraClientCredentials, getOuraConnection, - storeOuraTokens, + refreshOuraAccessToken, syncOuraScopeDenied, } from "./credentials"; import { OuraApiError, classifyOuraError } from "./response-classifier"; import { syncUserOuraCyclePhases } from "./cycle-sync"; -import { OURA_SCOPE_GATED_COLLECTIONS } from "./scope"; +import { isOuraScopeGatedCollection } from "./scope"; import { shiftDateKey, userDayKey } from "@/lib/tz/format"; import { resolveUserTimezone } from "@/lib/tz/resolver"; @@ -254,53 +255,81 @@ export interface OuraFetchResult { */ sleepRetirements: string[]; /** - * Collections refused with 401 while a sibling succeeded with the same - * token: the token is fine, the grant does not cover them. Not failures; the - * card asks for one reconnect instead. `null` when the run never proved the - * token (no collection succeeded, or the refresh was abandoned): it learned - * nothing about the grant, so the note from the last run that did stands. + * Collections the grant does not cover: refused with a 401 that names the + * missing scope, or with a bare 401 while a sibling succeeded with the same + * token. Not failures; the card asks for one reconnect instead. `null` when + * the run never proved the token (no collection succeeded, none was refused + * by name, or the refresh was abandoned): it learned nothing about the + * grant, so the note from the last run that did stands. */ scopeDenied: string[] | null; } /** - * A 401 from Oura: the token, or the grant's scope, was refused. A 403 stays a - * plain failure: Oura also answers 403 for a lapsed membership, which a - * reconnect does not fix. + * A 401 whose body names the missing scope (see `./scope`): Oura checked the + * token and found it valid, the grant just does not cover this collection. A + * refresh cannot widen a grant, so none is spent on it. */ -function isAuthRefusal(err: unknown): boolean { - return err instanceof OuraApiError && err.httpStatus === 401; -} - -const SCOPE_GATED: ReadonlySet = new Set(OURA_SCOPE_GATED_COLLECTIONS); - -/** - * A 401 that can be the grant's scope rather than the token: only the - * collections behind `spo2` / `heart_health` sit outside the `daily` scope - * every grant carries. A 401 on sleep, readiness or activity is a real auth - * failure whatever its siblings did. - */ -function isScopeRefusal(f: OuraCollectionFailure): boolean { - return SCOPE_GATED.has(f.name) && isAuthRefusal(f.err); +function isNamedScopeRefusal(err: unknown): boolean { + return ( + err instanceof OuraApiError && + err.httpStatus === 401 && + err.missingScope !== undefined + ); } /** - * Split the scope refusals out of a run's failures once a sibling collection - * has proven the token. Oura's authorization server enforces scopes per - * collection, so a grant issued without `spo2` / `heart_health` answers 401 on - * exactly the collections behind them and 200 everywhere else. Counting those - * toward the reconnect ladder painted a working connection as broken on every - * sync, and refreshing the token over them changed nothing. Any other failure, - * a 401 on a core collection included, stays a failure. + * Any other 401: the token was refused, or an older authorization server sent + * no body to tell. This is what the one refresh is for. A 403 is neither: Oura + * answers it for a lapsed membership, which no refresh or reconnect fixes. + */ +function isTokenRefusal(err: unknown): boolean { + return ( + err instanceof OuraApiError && + err.httpStatus === 401 && + err.missingScope === undefined + ); +} + +/** + * A failure that is the grant's scope rather than the token. Only the + * collections behind a scope beyond `daily` qualify: without `daily` nothing + * syncs, and that is a reconnect, not a hint. On those, a 401 naming its scope + * always qualifies; a bare 401 qualifies only once a sibling collection has + * proven the token, the inference that predates Oura naming the scope. + */ +function isScopeRefusal( + f: OuraCollectionFailure, + tokenProven: boolean, +): boolean { + if (!isOuraScopeGatedCollection(f.name)) return false; + return isNamedScopeRefusal(f.err) || (tokenProven && isTokenRefusal(f.err)); +} + +/** + * Split the scope refusals out of a run's failures. Oura's authorization + * server enforces scopes per collection, so a grant issued without `spo2`, + * `heart_health` or `stress` answers 401 on exactly the collections behind + * them and 200 everywhere else. Counting those toward the reconnect ladder + * painted a working connection as broken on every sync, and refreshing the + * token over them changed nothing. Any other failure, a 401 on a core + * collection included, stays a failure. + * + * A 401 that names its scope proves the token by itself: Oura validated it in + * order to say which permission it lacks. */ function splitScopeDenied( failures: OuraCollectionFailure[], - tokenProven: boolean, + siblingSucceeded: boolean, ): { failures: OuraCollectionFailure[]; scopeDenied: string[] | null } { + const tokenProven = + siblingSucceeded || failures.some((f) => isNamedScopeRefusal(f.err)); if (!tokenProven) return { failures, scopeDenied: null }; return { - failures: failures.filter((f) => !isScopeRefusal(f)), - scopeDenied: failures.filter(isScopeRefusal).map((f) => f.name), + failures: failures.filter((f) => !isScopeRefusal(f, true)), + scopeDenied: failures + .filter((f) => isScopeRefusal(f, true)) + .map((f) => f.name), }; } @@ -323,7 +352,6 @@ function splitScopeDenied( async function fetchAll( userId: string, accessToken: string, - refreshToken: string, refreshTokenCiphertext: string, lookbackDays: number, knownScopeDenied: readonly string[] = [], @@ -448,18 +476,14 @@ async function fetchAll( specs: OuraCollectionSpec[], ): Promise<{ readings: OuraMeasurementUpsert[]; - failed: OuraCollectionSpec[]; failures: OuraCollectionFailure[]; - auth401: boolean; succeeded: number; }> => { const settled = await Promise.allSettled( specs.map((spec) => spec.collect(token)), ); const readings: OuraMeasurementUpsert[] = []; - const failed: OuraCollectionSpec[] = []; const failures: OuraCollectionFailure[] = []; - let auth401 = false; let succeeded = 0; settled.forEach((res, i) => { const spec = specs[i]!; @@ -467,28 +491,26 @@ async function fetchAll( succeeded++; readings.push(...res.value); } else { - const err = res.reason; - if (err instanceof OuraApiError && err.httpStatus === 401) - auth401 = true; - failed.push(spec); - failures.push({ name: spec.name, err }); + failures.push({ name: spec.name, err: res.reason }); } }); - return { readings, failed, failures, auth401, succeeded }; + return { readings, failures, succeeded }; }; const first = await attempt(accessToken, collections); - // A 401 limited to collections already known to sit outside the grant, next - // to collections that succeeded, is the grant's scope again: a refresh cannot - // widen it, so none is spent. Any other 401 still takes the one refresh, and - // whatever 401s after it beside a success is recorded as scope. - const knownDenied = new Set( - knownScopeDenied.filter((name) => SCOPE_GATED.has(name)), + // The refresh is for a refused token. A 401 that names its scope is not one, + // and neither is a bare 401 on a collection already known to sit outside the + // grant while a sibling succeeded with the same token: a refresh cannot + // widen a grant, so none is spent on either. + const knownDenied = new Set( + knownScopeDenied.filter((name) => isOuraScopeGatedCollection(name)), ); - const onlyKnownScope401 = first.failures - .filter((f) => isAuthRefusal(f.err)) - .every((f) => knownDenied.has(f.name)); - if (!first.auth401 || (first.succeeded > 0 && onlyKnownScope401)) { + const needsRefresh = first.failures.some( + (f) => + isTokenRefusal(f.err) && + !(first.succeeded > 0 && knownDenied.has(f.name)), + ); + if (!needsRefresh) { return { readings: first.readings, ...splitScopeDenied(first.failures, first.succeeded > 0), @@ -498,43 +520,56 @@ async function fetchAll( } // Reactive refresh: the access token expired (a 401 on the shared token). - // Refresh once (rotating both tokens) and retry ONLY the collections that - // failed. A failed refresh throws so the caller parks reauth_required. + // Refresh once, serialised per user so an overlapping sync never spends the + // same one-time refresh token twice, and retry the collections that failed + // for any reason but a named scope. A failed refresh throws so the caller + // parks reauth_required. const creds = await getOuraClientCredentials(userId); if (!creds) return { readings: first.readings, - failures: first.failures, + // The refresh never happened, so no sibling's success can be trusted + // over it; a refusal that names its scope still says what it says. + ...splitScopeDenied(first.failures, false), sleepSweeps, sleepRetirements, - scopeDenied: null, }; - const rotated = await refreshAccessToken(refreshToken, creds); - // Compare-and-swap persist: on a lost race against a concurrent sync this - // returns the peer's freshly rotated access token rather than the (now - // invalidated) one we just minted, so neither sync parks the connection. - const usableToken = await storeOuraTokens( + const usableToken = await refreshOuraAccessToken( userId, - rotated.access_token, - rotated.refresh_token, refreshTokenCiphertext, + creds, ); if (!usableToken) return { readings: first.readings, - failures: first.failures, + // The refresh never happened, so no sibling's success can be trusted + // over it; a refusal that names its scope still says what it says. + ...splitScopeDenied(first.failures, false), sleepSweeps, sleepRetirements, - scopeDenied: null, }; - const retry = await attempt(usableToken, first.failed); + const settledByScope = first.failures.filter((f) => + isNamedScopeRefusal(f.err), + ); + const retryNames = new Set( + first.failures + .filter((f) => !isNamedScopeRefusal(f.err)) + .map((f) => f.name), + ); + const retry = await attempt( + usableToken, + collections.filter((c) => retryNames.has(c.name)), + ); return { readings: [...first.readings, ...retry.readings], // Proven by either pass: a collection that succeeded before the refresh // proves the grant as much as one that succeeded after it. - ...splitScopeDenied(retry.failures, first.succeeded + retry.succeeded > 0), + ...splitScopeDenied( + [...settledByScope, ...retry.failures], + first.succeeded + retry.succeeded > 0, + ), sleepSweeps, sleepRetirements, }; @@ -615,7 +650,6 @@ export async function syncUserOura( result = await fetchAll( userId, conn.accessToken, - conn.refreshToken, conn.refreshTokenCiphertext, lookbackDays, conn.scopeDenied, diff --git a/tests/integration/oauth-callback-single-use.test.ts b/tests/integration/oauth-callback-single-use.test.ts new file mode 100644 index 000000000..ec4e181e9 --- /dev/null +++ b/tests/integration/oauth-callback-single-use.test.ts @@ -0,0 +1,283 @@ +/** + * A browser that delivers the same OAuth callback twice must cause one code + * exchange, against real Postgres. + * + * Chrome was seen sending `/api/oura/callback` twice with the same code about + * 150 ms apart. Both requests exchanged the code; the second was refused, and + * Oura answered the reuse by revoking the token the first exchange had issued, + * so the first sync failed with 401 everywhere. The callback now claims its + * state before the exchange: one request exchanges, the other follows it to + * the same page and touches nothing. + * + * Covered for a signed-state flow (Oura) and a ledger flow (Withings), the two + * shapes every connect callback takes. + */ +import { NextRequest } from "next/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +process.env.API_TOKEN_HMAC_KEY ??= + "integration-hmac-key-0123456789abcdef0123456789"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const USER = "user-oauth-single-use"; +const OURA_TOKEN_URL = "https://moi.ouraring.com/oauth/v2/ext/oauth-token"; +const OURA_ISS = "https://moi.ouraring.com/oauth/v2/ext/oauth-anonymous"; +const WITHINGS_TOKEN_URL = "https://wbsapi.withings.net/v2/oauth2"; +const WITHINGS_NOTIFY_URL = "https://wbsapi.withings.net/notify"; + +/** Long enough that two callbacks fired together overlap at the provider. */ +const PROVIDER_LATENCY_MS = 150; + +function urlOf(input: unknown): string { + return typeof input === "string" ? input : (input as Request).url; +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + process.env.NEXT_PUBLIC_APP_URL = "http://localhost:3000"; + delete process.env.OURA_REDIRECT_URI; + + const { encrypt } = await import("@/lib/crypto"); + await getPrismaClient().user.create({ + data: { + id: USER, + username: "oauth-single-use", + email: "oauth-single-use@example.test", + role: "USER", + ouraClientIdEncrypted: encrypt("oura-client"), + ouraClientSecretEncrypted: encrypt("oura-secret"), + withingsClientIdEncrypted: encrypt("withings-client"), + withingsClientSecretEncrypted: encrypt("withings-secret"), + }, + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +async function ouraCallback(state: string, code = "oura-code") { + const { GET } = await import("@/app/api/oura/callback/route"); + const params = new URLSearchParams({ code, state, iss: OURA_ISS }); + return GET( + new NextRequest(`http://localhost/api/oura/callback?${params}`, { + method: "GET", + headers: { cookie: `oura_oauth_state=${state}` }, + }), + ); +} + +async function mintOuraState(): Promise { + const { mintSignedState } = await import("@/lib/oauth/signed-state"); + return mintSignedState("oura", USER); +} + +describe("Oura callback delivered twice", () => { + it("exchanges the code once and sends both requests to the connected page", async () => { + const tokenCalls: string[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const url = urlOf(input); + if (url === OURA_TOKEN_URL) { + tokenCalls.push(String(init?.body ?? "")); + await new Promise((r) => setTimeout(r, PROVIDER_LATENCY_MS)); + // A reused code is refused, as Oura does. + if (tokenCalls.length > 1) { + return new Response(JSON.stringify({ error: "invalid_grant" }), { + status: 400, + headers: { "content-type": "application/json" }, + }); + } + return new Response( + JSON.stringify({ + access_token: "oura-access", + refresh_token: "oura-refresh", + expires_in: 86400, + scope: + "extapi:daily extapi:personal extapi:spo2 extapi:heart_health extapi:stress", + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + } + throw new Error(`Unexpected fetch: ${url}`); + }); + + const state = await mintOuraState(); + const [a, b] = await Promise.all([ + ouraCallback(state), + ouraCallback(state), + ]); + + expect(tokenCalls).toHaveLength(1); + expect(a.headers.get("location")).toContain("oura=connected"); + expect(b.headers.get("location")).toBe(a.headers.get("location")); + + const { decrypt } = await import("@/lib/crypto"); + const user = await getPrismaClient().user.findUniqueOrThrow({ + where: { id: USER }, + select: { + ouraAccessTokenEncrypted: true, + ouraRefreshTokenEncrypted: true, + ouraScopeDenied: true, + }, + }); + expect(decrypt(user.ouraAccessTokenEncrypted!)).toBe("oura-access"); + expect(decrypt(user.ouraRefreshTokenEncrypted!)).toBe("oura-refresh"); + // Every requested scope was granted, so nothing awaits a reconnect. + expect(user.ouraScopeDenied).toBeNull(); + }); + + it("notes the collections a grant leaves out, straight from the token response", async () => { + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + if (urlOf(input) !== OURA_TOKEN_URL) throw new Error("unexpected"); + return new Response( + JSON.stringify({ + access_token: "oura-access", + refresh_token: "oura-refresh", + expires_in: 86400, + scope: "extapi:daily extapi:personal extapi:spo2 extapi:heart_health", + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + + const res = await ouraCallback(await mintOuraState()); + + expect(res.headers.get("location")).toContain("oura=connected"); + const user = await getPrismaClient().user.findUniqueOrThrow({ + where: { id: USER }, + select: { ouraScopeDenied: true }, + }); + expect(user.ouraScopeDenied).toEqual(["resilience"]); + }); + + it("lets both requests see a failed exchange, and a new connect still works", async () => { + let tokenCalls = 0; + let refuse = true; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + if (urlOf(input) !== OURA_TOKEN_URL) throw new Error("unexpected"); + tokenCalls++; + await new Promise((r) => setTimeout(r, PROVIDER_LATENCY_MS)); + if (refuse) { + return new Response(JSON.stringify({ error: "invalid_request" }), { + status: 400, + headers: { "content-type": "application/json" }, + }); + } + return new Response( + JSON.stringify({ + access_token: "second-access", + refresh_token: "second-refresh", + expires_in: 86400, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + + const first = await mintOuraState(); + const [a, b] = await Promise.all([ + ouraCallback(first), + ouraCallback(first), + ]); + expect(tokenCalls).toBe(1); + expect(a.headers.get("location")).toContain("oura=error&reason=token"); + expect(b.headers.get("location")).toBe(a.headers.get("location")); + + // Starting over mints a new state, which is a new claim. + refuse = false; + const retry = await ouraCallback(await mintOuraState(), "fresh-code"); + expect(tokenCalls).toBe(2); + expect(retry.headers.get("location")).toContain("oura=connected"); + }); +}); + +describe("Withings callback delivered twice", () => { + it("exchanges the code once and sends the duplicate where the first went", async () => { + const prisma = getPrismaClient(); + const session = await prisma.session.create({ + data: { userId: USER, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + + let tokenCalls = 0; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + const url = urlOf(input); + if (url === WITHINGS_TOKEN_URL) { + tokenCalls++; + await new Promise((r) => setTimeout(r, PROVIDER_LATENCY_MS)); + return new Response( + JSON.stringify({ + status: 0, + body: { + userid: "withings-single-use", + access_token: "w-access", + refresh_token: "w-refresh", + expires_in: 3600, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + } + if (url === WITHINGS_NOTIFY_URL) { + return new Response(JSON.stringify({ status: 0 }), { status: 200 }); + } + throw new Error(`Unexpected fetch: ${url}`); + }); + + const nonce = "single-use-nonce-22chr"; + await prisma.withingsOAuthState.create({ + data: { + nonce, + userId: USER, + expiresAt: new Date(Date.now() + 600_000), + }, + }); + + const { GET } = await import("@/app/api/withings/callback/route"); + const request = () => + new NextRequest( + `http://localhost/api/withings/callback?code=w-code&state=${nonce}`, + { + method: "GET", + headers: { + cookie: `healthlog_session=${session.id}; withings_state=${nonce}`, + }, + }, + ); + const [a, b] = await Promise.all([GET(request()), GET(request())]); + + expect(tokenCalls).toBe(1); + expect(a.headers.get("location")).toContain("withings=connected"); + expect(b.headers.get("location")).toBe(a.headers.get("location")); + }); +}); diff --git a/tests/integration/oura-refresh-serialization.test.ts b/tests/integration/oura-refresh-serialization.test.ts new file mode 100644 index 000000000..488ae9a1b --- /dev/null +++ b/tests/integration/oura-refresh-serialization.test.ts @@ -0,0 +1,156 @@ +/** + * Oura's refresh tokens are single-use. Overlapping syncs that all meet an + * expired access token must refresh it once between them, against real + * Postgres: spending the same refresh token twice is refused, and the + * authorization server may revoke the grant behind it. + * + * Also pinned: a reconnect that stores a new grant while a refresh of the old + * one is in flight keeps the new grant. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type * as OuraClientModule from "@/lib/oura/client"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +process.env.ENCRYPTION_KEY = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +delete process.env.ENCRYPTION_KEYS; +delete process.env.ENCRYPTION_ACTIVE_KEY_ID; + +const { ouraRefresh, collection } = vi.hoisted(() => ({ + ouraRefresh: vi.fn(), + collection: vi.fn(), +})); + +vi.mock("@/lib/oura/client", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + refreshAccessToken: ouraRefresh, + fetchReadiness: collection, + fetchSleep: collection, + fetchDailyActivity: collection, + fetchDailySleep: collection, + fetchDailySpo2: collection, + fetchVo2Max: collection, + fetchCardiovascularAge: collection, + fetchResilience: collection, + fetchDailyCyclePhases: collection, + }; +}); + +import { decrypt, encrypt } from "@/lib/crypto"; +import { OuraApiError } from "@/lib/oura/response-classifier"; +import { refreshOuraAccessToken } from "@/lib/oura/credentials"; +import { syncUserOura } from "@/lib/oura/sync"; + +const USER = "user-oura-refresh"; +const CONTENDERS = 12; + +function expired() { + return new OuraApiError({ + verb: "fetchReadiness", + classification: "reauth_required", + httpStatus: 401, + reason: "http_401", + }); +} + +async function seed() { + await getPrismaClient().user.create({ + data: { + id: USER, + username: "oura-refresh", + email: "oura-refresh@example.test", + ouraClientIdEncrypted: encrypt("oura-client"), + ouraClientSecretEncrypted: encrypt("oura-secret"), + ouraAccessTokenEncrypted: encrypt("stale-access"), + ouraRefreshTokenEncrypted: encrypt("stale-refresh"), + }, + }); +} + +async function storedTokens() { + const row = await getPrismaClient().user.findUniqueOrThrow({ + where: { id: USER }, + select: { + ouraAccessTokenEncrypted: true, + ouraRefreshTokenEncrypted: true, + }, + }); + return { + access: decrypt(row.ouraAccessTokenEncrypted!), + refresh: decrypt(row.ouraRefreshTokenEncrypted!), + }; +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + vi.clearAllMocks(); + await seed(); + // Every collection refuses the stale token and serves the fresh one. + collection.mockImplementation(async (token: string) => { + if (token === "stale-access") throw expired(); + return []; + }); +}); + +describe("Oura token refresh (real Postgres)", () => { + it("refreshes once for any number of overlapping syncs", async () => { + ouraRefresh.mockImplementation(async (refreshToken: string) => { + expect(refreshToken).toBe("stale-refresh"); + await new Promise((r) => setTimeout(r, 100)); + return { + access_token: "fresh-access", + refresh_token: "fresh-refresh", + expires_in: 86400, + }; + }); + + const results = await Promise.all( + Array.from({ length: CONTENDERS }, () => syncUserOura(USER)), + ); + + expect(ouraRefresh).toHaveBeenCalledTimes(1); + expect(results.every((r) => r.failed === false)).toBe(true); + expect(await storedTokens()).toEqual({ + access: "fresh-access", + refresh: "fresh-refresh", + }); + }); + + it("keeps a grant a reconnect stored while a refresh of the old one ran", async () => { + const prisma = getPrismaClient(); + const spent = (await prisma.user.findUniqueOrThrow({ + where: { id: USER }, + select: { ouraRefreshTokenEncrypted: true }, + }))!.ouraRefreshTokenEncrypted!; + + ouraRefresh.mockImplementation(async () => { + // The callback writes a new grant without taking the refresh lock. + await prisma.user.update({ + where: { id: USER }, + data: { + ouraAccessTokenEncrypted: encrypt("reconnect-access"), + ouraRefreshTokenEncrypted: encrypt("reconnect-refresh"), + }, + }); + return { + access_token: "old-grant-access", + refresh_token: "old-grant-refresh", + expires_in: 86400, + }; + }); + + const token = await refreshOuraAccessToken(USER, spent, { + clientId: "oura-client", + clientSecret: "oura-secret", + }); + + expect(token).toBe("reconnect-access"); + expect(await storedTokens()).toEqual({ + access: "reconnect-access", + refresh: "reconnect-refresh", + }); + }); +}); diff --git a/tests/integration/setup.ts b/tests/integration/setup.ts index 44d3d9996..fca0c34da 100644 --- a/tests/integration/setup.ts +++ b/tests/integration/setup.ts @@ -66,6 +66,9 @@ export async function truncateAllTables(client: PrismaClient): Promise { "mood_entries", "notification_channels", "notification_preferences", + // Instance-wide OAuth callback claims, keyed on a hash of the state: not + // reached by the users cascade. + "oauth_callback_claims", // Host-wide off-host backup state: not keyed to a user, so the users // cascade does not reach it, and a file that reads it whole (the purge // job, the encryption view) saw what earlier files left behind. diff --git a/tests/integration/withings-oauth-flow.test.ts b/tests/integration/withings-oauth-flow.test.ts index a1fb8fa40..1ba5e6197 100644 --- a/tests/integration/withings-oauth-flow.test.ts +++ b/tests/integration/withings-oauth-flow.test.ts @@ -10,9 +10,10 @@ * 1. `withings/connect` mints a row and stamps the cookie. * 2. `withings/callback` resolves the user via the row's `userId`, * verifies it matches the session, and deletes the row. - * 3. A second callback with the same nonce hits the "row not - * found" branch and redirects to the error page (replay - * protection). + * 3. A second callback with the same nonce never reaches the token + * endpoint: while the first callback's claim stands it follows + * that callback to the same page, and without one it hits the + * "row not found" branch and the error page (replay protection). * 4. A callback whose row has expired hits the same branch. * 5. The connect handler refuses to start when the user has no * Withings credentials — no ledger row is minted. @@ -157,7 +158,7 @@ describe("Withings OAuth nonce ledger (real Postgres)", () => { expect(rows).toHaveLength(0); }); - it("connect → callback consumes the row (single-use); replay of the same nonce hits the error page", async () => { + it("connect → callback consumes the row (single-use); a repeat of the same nonce never exchanges again", async () => { vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { const url = typeof input === "string" ? input : (input as Request).url; if (url === WITHINGS_TOKEN_URL) { @@ -214,22 +215,30 @@ describe("Withings OAuth nonce ledger (real Postgres)", () => { }); expect(afterCallback).toBeNull(); - // Leg 3 — replay the same nonce. The cookie compare still passes - // (matching strings) but the row lookup misses, so the handler - // bounces to the error page without hitting the token endpoint - // again. + // Leg 3 — the browser delivers the same callback again. The cookie + // compare still passes (matching strings) and the row is gone, but the + // first callback's claim stands: the repeat follows it to the same page + // without hitting the token endpoint again. const fetchSpy = vi.mocked(globalThis.fetch); fetchSpy.mockClear(); - const replayReq = new NextRequest( - `http://localhost/api/withings/callback?code=auth-code&state=${encodeURIComponent(nonce)}`, - { - method: "GET", - headers: { - cookie: `healthlog_session=${cookieJar.get("healthlog_session")}; withings_state=${nonce}`, + const replayRequest = () => + new NextRequest( + `http://localhost/api/withings/callback?code=auth-code&state=${encodeURIComponent(nonce)}`, + { + method: "GET", + headers: { + cookie: `healthlog_session=${cookieJar.get("healthlog_session")}; withings_state=${nonce}`, + }, }, - }, - ); - const replayRes = await callback(replayReq); + ); + const repeatRes = await callback(replayRequest()); + expect(repeatRes.status).toBe(307); + expect(repeatRes.headers.get("location")).toContain("withings=connected"); + + // Leg 4 — once the claim has been swept, the same nonce is a plain + // replay and lands on the error page. + await prisma.oAuthCallbackClaim.deleteMany(); + const replayRes = await callback(replayRequest()); expect(replayRes.status).toBe(307); expect(replayRes.headers.get("location")).toContain( "withings=error&reason=replay", From 1be2c27b2738c4a28a53c2da6c1723b52dbbb309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 00:06:21 +0200 Subject: [PATCH 19/23] fix(ai): serialize Codex refreshes so a lost race never parks a live sign-in Two callers refreshing one Codex token raced: the loser heard refresh_token_reused, read it as a dead grant and parked the connection before the winner wrote its rotated tokens, leaving fresh tokens beside an expired status. Refreshes now share one flight per process and queue on the per-credential advisory lock across processes, re-read the row inside it, use a token a peer already rotated, write the rotated tokens compare-and-set on the spent one together with connected, and re-read after a refusal before parking. refresh_token_reused no longer counts as a dead grant, and neither does a bare 403 without an OAuth error code: an edge challenge answers a valid grant so, and it keeps the ledger's timed cooldown instead of parking until the next sign-in. --- src/lib/ai/__tests__/codex-oauth.test.ts | 17 + src/lib/ai/codex-oauth.ts | 17 +- src/lib/ai/provider.ts | 453 +++++++++++------- src/lib/integrations/oauth-refresh.ts | 3 +- .../integration/codex-credential-park.test.ts | 58 +++ 5 files changed, 362 insertions(+), 186 deletions(-) diff --git a/src/lib/ai/__tests__/codex-oauth.test.ts b/src/lib/ai/__tests__/codex-oauth.test.ts index 1092e26af..dbbc695cf 100644 --- a/src/lib/ai/__tests__/codex-oauth.test.ts +++ b/src/lib/ai/__tests__/codex-oauth.test.ts @@ -225,6 +225,23 @@ describe("codex-oauth", () => { expect(err.oauthError).toBeNull(); }); + it("does not read a bare 403 as a dead sign-in", async () => { + // An edge challenge in front of the issuer answers a valid grant so. + failWith(403, "challenge"); + const err = await refreshError(); + expect(err.credentialExpired).toBe(false); + expect(err.httpStatus).toBe(403); + }); + + it("does not read refresh_token_reused as a dead sign-in", async () => { + // The loser of two refreshes of one token hears this; the winner's + // rotated token is alive. + failWith(400, JSON.stringify({ error: "refresh_token_reused" })); + const err = await refreshError(); + expect(err.credentialExpired).toBe(false); + expect(err.oauthError).toBe("refresh_token_reused"); + }); + it("never quotes the response body in the message", async () => { failWith( 401, diff --git a/src/lib/ai/codex-oauth.ts b/src/lib/ai/codex-oauth.ts index 6d389160c..bcfd2c630 100644 --- a/src/lib/ai/codex-oauth.ts +++ b/src/lib/ai/codex-oauth.ts @@ -296,14 +296,18 @@ export async function refreshDeviceTokens( /** * OAuth error codes on the refresh grant that mean the stored refresh token - * is dead for good: revoked, expired, already rotated, or issued to another - * client. Retrying cannot help; only signing in again can. + * is dead for good: revoked, expired, or issued to another client. Retrying + * cannot help; only signing in again can. + * + * `refresh_token_reused` is deliberately not among them. It is what the loser + * of two refreshes of one token hears while the winner's rotated token is + * perfectly alive, so it says nothing about the sign-in: the refresh re-reads + * the row and carries on with the newer token (`refreshCodexCredential`). */ const DEAD_REFRESH_TOKEN_CODES: ReadonlySet = new Set([ "invalid_grant", "invalid_refresh_token", "refresh_token_expired", - "refresh_token_reused", "refresh_token_invalidated", "invalid_token", ]); @@ -333,9 +337,12 @@ export class CodexRefreshError extends Error { readonly oauthError: string | null; constructor(status: number, oauthError: string | null) { + // A bare 403 is not proof: an edge challenge in front of the issuer + // answers 403 to a perfectly valid grant. It keeps its own status and + // with it the ledger's timed auth cooldown; only a 401 or the issuer's + // own dead-grant code parks the sign-in until the person reconnects. const credentialExpired = status === 401 || - status === 403 || (oauthError !== null && DEAD_REFRESH_TOKEN_CODES.has(oauthError)); super( credentialExpired @@ -345,7 +352,7 @@ export class CodexRefreshError extends Error { this.name = "CodexRefreshError"; this.credentialExpired = credentialExpired; this.oauthError = oauthError; - this.httpStatus = credentialExpired && status !== 403 ? 401 : status; + this.httpStatus = credentialExpired ? 401 : status; } } diff --git a/src/lib/ai/provider.ts b/src/lib/ai/provider.ts index e05758bed..2a5e24b1e 100644 --- a/src/lib/ai/provider.ts +++ b/src/lib/ai/provider.ts @@ -14,6 +14,7 @@ import { decryptCodexCreds, encryptAdminCodexCreds, decryptAdminCodexCreds, + type CodexCreds, } from "./codex-oauth"; import { isPublicUrl } from "@/lib/validations/notifications"; import { isLocalAiHostAllowed } from "./local-host-allowlist"; @@ -31,6 +32,11 @@ import { } from "@/lib/sharing/provider-work-authority"; import { memoizePerRequest } from "@/lib/request-cache"; import { emitSignal } from "@/lib/logging/signal"; +import { + acquireProviderTokenRefreshLock, + PROVIDER_REFRESH_TRANSACTION_OPTIONS, +} from "@/lib/integrations/oauth-refresh"; +import type { Prisma } from "@/generated/prisma/client"; import type { AiModality } from "@/lib/ai/capabilities/types"; import type { ProviderEntryPresence, @@ -62,72 +68,253 @@ function oauthErrorOf(error: unknown): string | null { return typeof code === "string" ? code : null; } +type CodexCredentialOwner = "codex" | "admin-codex"; + /** - * Park a Codex connection whose sign-in the issuer has refused for good. - * - * Retrying a dead refresh token cannot succeed; until this existed every - * generation of every card of every run asked the issuer again, failed with - * the same 401, and logged it. Parking flips the connection to the existing - * `expired` state — the one Settings already renders as "reconnect ChatGPT" — - * so the chain, the presence probe and with it every AI capability stop - * counting it, and nothing is sent until the person signs in again (the - * device-code flow writes `connected`). - * - * Compare-and-set on the refresh token that was refused: a reconnect that - * landed in between wrote a different one and is left alone, and of several - * workers failing at once exactly one flips the row, so the transition is - * signalled once and never again per run. + * One Codex credential row as a refresh sees it: the user's own connection or + * the operator's central one. `writeRotated` and `park` are compare-and-set on + * the refresh token this refresh spent, so a sign-in that landed meanwhile (the + * device-code flow writes without the refresh lock) is never overwritten. */ -async function parkUserCodex( - userId: string, - refusedRefreshEncrypted: string, - error: unknown, -): Promise { - try { - const parked = await prisma.user.updateMany({ - where: { - id: userId, - codexConnectionStatus: "connected", - codexRefreshTokenEncrypted: refusedRefreshEncrypted, - }, - data: { codexConnectionStatus: "expired" }, - }); - if (parked.count === 1) { - emitSignal({ - action: "ai.credential.expired", - level: "warn", - meta: { provider: "codex", oauth_error: oauthErrorOf(error) }, - }); - } - } catch { - // Best effort: an unparked connection is retried exactly as before. - } +interface CodexCredentialStore { + owner: CodexCredentialOwner; + lockId: string; + read(tx: Prisma.TransactionClient): Promise<{ + status: string | null; + refreshEncrypted: string | null; + creds: CodexCreds | null; + } | null>; + writeRotated( + tx: Prisma.TransactionClient, + spentRefreshEncrypted: string, + fresh: CodexCreds, + ): Promise; + park( + tx: Prisma.TransactionClient, + spentRefreshEncrypted: string, + ): Promise; } -/** The operator's central Codex, parked the same way. */ -async function parkCentralCodex( - refusedRefreshEncrypted: string, - error: unknown, -): Promise { - try { - const parked = await prisma.appSettings.updateMany({ +function userCodexStore(userId: string): CodexCredentialStore { + return { + owner: "codex", + lockId: userId, + async read(tx) { + const row = await tx.user.findUnique({ + where: { id: userId }, + select: { + codexConnectionStatus: true, + codexAccessTokenEncrypted: true, + codexRefreshTokenEncrypted: true, + }, + }); + if (!row) return null; + return { + status: row.codexConnectionStatus, + refreshEncrypted: row.codexRefreshTokenEncrypted, + creds: + row.codexAccessTokenEncrypted && row.codexRefreshTokenEncrypted + ? decryptCodexCreds({ + accessEncrypted: row.codexAccessTokenEncrypted, + refreshEncrypted: row.codexRefreshTokenEncrypted, + }) + : null, + }; + }, + async writeRotated(tx, spent, fresh) { + const enc = encryptCodexCreds(fresh); + const written = await tx.user.updateMany({ + where: { id: userId, codexRefreshTokenEncrypted: spent }, + data: { + codexAccessTokenEncrypted: enc.accessEncrypted, + codexRefreshTokenEncrypted: enc.refreshEncrypted, + codexTokenExpiresAt: fresh.expiresAt, + codexConnectionStatus: "connected", + }, + }); + return written.count; + }, + async park(tx, spent) { + const parked = await tx.user.updateMany({ + where: { + id: userId, + codexConnectionStatus: "connected", + codexRefreshTokenEncrypted: spent, + }, + data: { codexConnectionStatus: "expired" }, + }); + return parked.count; + }, + }; +} + +const centralCodexStore: CodexCredentialStore = { + owner: "admin-codex", + lockId: "singleton", + async read(tx) { + const row = await tx.appSettings.findUnique({ + where: { id: "singleton" }, + select: { + adminCodexConnectionStatus: true, + adminCodexAccessTokenEncrypted: true, + adminCodexRefreshTokenEncrypted: true, + adminCodexAccountIdEncrypted: true, + adminCodexTokenExpiresAt: true, + }, + }); + if (!row) return null; + return { + status: row.adminCodexConnectionStatus, + refreshEncrypted: row.adminCodexRefreshTokenEncrypted, + creds: + row.adminCodexAccessTokenEncrypted && + row.adminCodexRefreshTokenEncrypted && + row.adminCodexAccountIdEncrypted + ? decryptAdminCodexCreds({ + accessEncrypted: row.adminCodexAccessTokenEncrypted, + refreshEncrypted: row.adminCodexRefreshTokenEncrypted, + accountIdEncrypted: row.adminCodexAccountIdEncrypted, + expiresAt: row.adminCodexTokenExpiresAt, + }) + : null, + }; + }, + async writeRotated(tx, spent, fresh) { + const enc = encryptAdminCodexCreds(fresh); + const written = await tx.appSettings.updateMany({ + where: { id: "singleton", adminCodexRefreshTokenEncrypted: spent }, + data: { + adminCodexAccessTokenEncrypted: enc.accessEncrypted, + adminCodexRefreshTokenEncrypted: enc.refreshEncrypted, + adminCodexAccountIdEncrypted: enc.accountIdEncrypted, + adminCodexTokenExpiresAt: enc.expiresAt, + adminCodexConnectionStatus: "connected", + }, + }); + return written.count; + }, + async park(tx, spent) { + const parked = await tx.appSettings.updateMany({ where: { id: "singleton", adminCodexConnectionStatus: "connected", - adminCodexRefreshTokenEncrypted: refusedRefreshEncrypted, + adminCodexRefreshTokenEncrypted: spent, }, data: { adminCodexConnectionStatus: "expired" }, }); - if (parked.count === 1) { - emitSignal({ - action: "ai.credential.expired", - level: "warn", - meta: { provider: "admin-codex", oauth_error: oauthErrorOf(error) }, - }); - } - } catch { - // Best effort, as above. + return parked.count; + }, +}; + +/** Refreshes in flight in this process, keyed on credential and spent token. */ +const codexRefreshesInFlight = new Map>(); + +/** + * Refresh one Codex credential, at most once per spent token. + * + * The issuer rotates the refresh token on every refresh and honours each one + * once. Two callers that refreshed the same token used to race: the loser got + * `refresh_token_reused`, read it as a dead sign-in and parked the connection + * while the winner's fresh tokens landed beside an `expired` status. So: + * + * - callers in one process share one refresh (single flight), and callers + * in different processes queue on the per-credential advisory lock the + * provider syncs use (`acquireProviderTokenRefreshLock`); + * - inside the lock the row is read again, and a token a peer already + * rotated is used as it is, without asking the issuer; + * - the rotated tokens are written compare-and-set on the spent token, and + * the write sets `connected`; + * - after a refusal the row is read again too, and only a refusal of the + * token still stored, with the issuer's dead-grant answer, parks it + * (`expired`, the state Settings renders as "reconnect ChatGPT"). The + * transition is signalled once: only the lock holder can make it. + * + * Parking stops the chain, the presence probe and every AI capability from + * counting the connection until the person signs in again, which is what + * stopped every run from asking the issuer about the same dead token. + */ +function refreshCodexCredential( + store: CodexCredentialStore, + spentRefreshEncrypted: string, +): Promise { + const key = `${store.owner}:${store.lockId}:${spentRefreshEncrypted}`; + const inFlight = codexRefreshesInFlight.get(key); + if (inFlight) return inFlight; + const run = runCodexRefresh(store, spentRefreshEncrypted).finally(() => { + codexRefreshesInFlight.delete(key); + }); + codexRefreshesInFlight.set(key, run); + return run; +} + +async function runCodexRefresh( + store: CodexCredentialStore, + spent: string, +): Promise { + const outcome = await prisma.$transaction( + async ( + tx, + ): Promise< + | { fresh: CodexCreds } + | { refused: unknown; parked: boolean } + | { failed: unknown } + > => { + await acquireProviderTokenRefreshLock(tx, store.owner, store.lockId); + const row = await store.read(tx); + if (!row || row.status !== "connected" || !row.refreshEncrypted) { + return { + refused: codexReconnectRequired("no refresh token"), + parked: false, + }; + } + if (row.refreshEncrypted !== spent && row.creds) { + // A peer rotated while this caller waited: its token is the live one. + return { fresh: row.creds }; + } + if (!row.creds) { + const refused = codexReconnectRequired("token storage corrupt"); + return { + refused, + parked: (await store.park(tx, row.refreshEncrypted)) === 1, + }; + } + let fresh: CodexCreds; + try { + fresh = await refreshDeviceTokens(row.creds.refreshToken); + } catch (error) { + const now = await store.read(tx); + if (now?.refreshEncrypted !== row.refreshEncrypted && now?.creds) { + // A sign-in replaced the token while the issuer refused the old one. + return { fresh: now.creds }; + } + if (!isDeadCodexSignIn(error)) return { failed: error }; + return { + refused: error, + parked: (await store.park(tx, row.refreshEncrypted)) === 1, + }; + } + if ((await store.writeRotated(tx, row.refreshEncrypted, fresh)) === 0) { + // A sign-in replaced the row meanwhile; it wins over this refresh. + const now = await store.read(tx); + if (now?.creds) return { fresh: now.creds }; + } + return { fresh }; + }, + PROVIDER_REFRESH_TRANSACTION_OPTIONS, + ); + if ("fresh" in outcome) return outcome.fresh; + if ("failed" in outcome) throw outcome.failed; + if (outcome.parked) { + emitSignal({ + action: "ai.credential.expired", + level: "warn", + meta: { + provider: store.owner, + oauth_error: oauthErrorOf(outcome.refused), + }, + }); } + throw outcome.refused; } class NoProvider implements AIProvider { @@ -384,27 +571,16 @@ async function resolveCodexProvider( // Proactive refresh if the access token is within 5 min of expiry. if (stored.expiresAt.getTime() < Date.now() + TOKEN_REFRESH_BUFFER_MS) { try { - const fresh = await refreshDeviceTokens(stored.refreshToken); - active = fresh; - - const enc = encryptCodexCreds(fresh); - await prisma.user.update({ - where: { id: userId }, - data: { - codexAccessTokenEncrypted: enc.accessEncrypted, - codexRefreshTokenEncrypted: enc.refreshEncrypted, - codexTokenExpiresAt: fresh.expiresAt, - }, - }); + active = await refreshCodexCredential( + userCodexStore(userId), + user.codexRefreshTokenEncrypted, + ); } catch (error) { - // A refused grant parks the connection and drops it from this chain: - // calling Codex with the expired access token could only 401 into a - // second refusal. Anything else falls through — CodexClient will - // trigger an on-401 refresh and persist there. - if (isDeadCodexSignIn(error)) { - await parkUserCodex(userId, user.codexRefreshTokenEncrypted, error); - return null; - } + // A refused grant drops the connection from this chain (the refresh + // parked it): calling Codex with the expired access token could only + // 401 into a second refusal. Anything else falls through — CodexClient + // will trigger an on-401 refresh. + if (isDeadCodexSignIn(error)) return null; } } @@ -412,54 +588,17 @@ async function resolveCodexProvider( accessToken: active.accessToken, accountId: active.accountId, onTokenRefresh: async () => { - const freshUser = await prisma.user.findUnique({ + const current = await prisma.user.findUnique({ where: { id: userId }, - select: { - codexAccessTokenEncrypted: true, - codexRefreshTokenEncrypted: true, - }, + select: { codexRefreshTokenEncrypted: true }, }); - if ( - !freshUser?.codexAccessTokenEncrypted || - !freshUser.codexRefreshTokenEncrypted - ) { + if (!current?.codexRefreshTokenEncrypted) { throw codexReconnectRequired("no refresh token"); } - const decoded = decryptCodexCreds({ - accessEncrypted: freshUser.codexAccessTokenEncrypted, - refreshEncrypted: freshUser.codexRefreshTokenEncrypted, - }); - if (!decoded) { - const error = codexReconnectRequired("token storage corrupt"); - await parkUserCodex( - userId, - freshUser.codexRefreshTokenEncrypted, - error, - ); - throw error; - } - let fresh; - try { - fresh = await refreshDeviceTokens(decoded.refreshToken); - } catch (error) { - if (isDeadCodexSignIn(error)) { - await parkUserCodex( - userId, - freshUser.codexRefreshTokenEncrypted, - error, - ); - } - throw error; - } - const enc = encryptCodexCreds(fresh); - await prisma.user.update({ - where: { id: userId }, - data: { - codexAccessTokenEncrypted: enc.accessEncrypted, - codexRefreshTokenEncrypted: enc.refreshEncrypted, - codexTokenExpiresAt: fresh.expiresAt, - }, - }); + const fresh = await refreshCodexCredential( + userCodexStore(userId), + current.codexRefreshTokenEncrypted, + ); return { accessToken: fresh.accessToken, accountId: fresh.accountId }; }, }); @@ -521,26 +660,15 @@ async function resolveAdminCodexProvider(): Promise { if (stored.expiresAt.getTime() < Date.now() + TOKEN_REFRESH_BUFFER_MS) { try { - const fresh = await refreshDeviceTokens(stored.refreshToken); - active = fresh; - const enc = encryptAdminCodexCreds(fresh); - await prisma.appSettings.update({ - where: { id: "singleton" }, - data: { - adminCodexAccessTokenEncrypted: enc.accessEncrypted, - adminCodexRefreshTokenEncrypted: enc.refreshEncrypted, - adminCodexAccountIdEncrypted: enc.accountIdEncrypted, - adminCodexTokenExpiresAt: enc.expiresAt, - }, - }); + active = await refreshCodexCredential( + centralCodexStore, + settings.adminCodexRefreshTokenEncrypted, + ); } catch (error) { - // A refused grant parks the central connection (see `parkUserCodex`); - // anything else falls through — CodexClient triggers an on-401 refresh - // and persists via the callback below. - if (isDeadCodexSignIn(error)) { - await parkCentralCodex(settings.adminCodexRefreshTokenEncrypted, error); - return null; - } + // A refused grant parked the central connection (see + // `refreshCodexCredential`); anything else falls through — CodexClient + // triggers an on-401 refresh via the callback below. + if (isDeadCodexSignIn(error)) return null; } } @@ -548,52 +676,17 @@ async function resolveAdminCodexProvider(): Promise { accessToken: active.accessToken, accountId: active.accountId, onTokenRefresh: async () => { - const fresh = await prisma.appSettings.findUnique({ + const current = await prisma.appSettings.findUnique({ where: { id: "singleton" }, - select: { - adminCodexAccessTokenEncrypted: true, - adminCodexRefreshTokenEncrypted: true, - adminCodexAccountIdEncrypted: true, - adminCodexTokenExpiresAt: true, - }, + select: { adminCodexRefreshTokenEncrypted: true }, }); - if ( - !fresh?.adminCodexAccessTokenEncrypted || - !fresh.adminCodexRefreshTokenEncrypted || - !fresh.adminCodexAccountIdEncrypted - ) { + if (!current?.adminCodexRefreshTokenEncrypted) { throw codexReconnectRequired("no central refresh token"); } - const decoded = decryptAdminCodexCreds({ - accessEncrypted: fresh.adminCodexAccessTokenEncrypted, - refreshEncrypted: fresh.adminCodexRefreshTokenEncrypted, - accountIdEncrypted: fresh.adminCodexAccountIdEncrypted, - expiresAt: fresh.adminCodexTokenExpiresAt, - }); - if (!decoded) { - const error = codexReconnectRequired("central token storage corrupt"); - await parkCentralCodex(fresh.adminCodexRefreshTokenEncrypted, error); - throw error; - } - let refreshed; - try { - refreshed = await refreshDeviceTokens(decoded.refreshToken); - } catch (error) { - if (isDeadCodexSignIn(error)) { - await parkCentralCodex(fresh.adminCodexRefreshTokenEncrypted, error); - } - throw error; - } - const enc = encryptAdminCodexCreds(refreshed); - await prisma.appSettings.update({ - where: { id: "singleton" }, - data: { - adminCodexAccessTokenEncrypted: enc.accessEncrypted, - adminCodexRefreshTokenEncrypted: enc.refreshEncrypted, - adminCodexAccountIdEncrypted: enc.accountIdEncrypted, - adminCodexTokenExpiresAt: enc.expiresAt, - }, - }); + const refreshed = await refreshCodexCredential( + centralCodexStore, + current.adminCodexRefreshTokenEncrypted, + ); return { accessToken: refreshed.accessToken, accountId: refreshed.accountId, diff --git a/src/lib/integrations/oauth-refresh.ts b/src/lib/integrations/oauth-refresh.ts index 89c424051..b2ecaec70 100644 --- a/src/lib/integrations/oauth-refresh.ts +++ b/src/lib/integrations/oauth-refresh.ts @@ -61,7 +61,8 @@ export async function persistRotatedToken( return opts.readPeerAccessToken(); } -export type SerializedRefreshProvider = "whoop" | "withings"; +export type SerializedRefreshProvider = + "whoop" | "withings" | "codex" | "admin-codex"; /** * Provider refreshes hold a database transaction open around one bounded diff --git a/tests/integration/codex-credential-park.test.ts b/tests/integration/codex-credential-park.test.ts index 95ea57391..811fefd0c 100644 --- a/tests/integration/codex-credential-park.test.ts +++ b/tests/integration/codex-credential-park.test.ts @@ -261,3 +261,61 @@ describe("a refused Codex sign-in", () => { expect(vi.mocked(refreshDeviceTokens)).toHaveBeenCalledTimes(1); }); }); + +describe("two refreshes of one Codex token", () => { + async function codexStatus(userId: string) { + return ( + await prisma.user.findUniqueOrThrow({ + where: { id: userId }, + select: { codexConnectionStatus: true }, + }) + ).codexConnectionStatus; + } + + it("share one refresh: the slower caller never parks the winner's sign-in", async () => { + const user = await seedCodexUser(); + // The issuer honours each refresh token once. The first caller's refresh + // takes a moment; a second refresh of the same token is refused as reused. + let asked = 0; + vi.mocked(refreshDeviceTokens).mockImplementation(async () => { + asked += 1; + if (asked > 1) throw new CodexRefreshError(400, "refresh_token_reused"); + await new Promise((resolve) => setTimeout(resolve, 200)); + return creds("rotated", 10 * HOUR_MS); + }); + + const [first, second] = await Promise.all([ + resolveProviderChain(user.id), + resolveProviderChain(user.id), + ]); + + expect(first.map((p) => p.providerType)).toContain("codex"); + expect(second.map((p) => p.providerType)).toContain("codex"); + expect(asked).toBe(1); + expect(await codexStatus(user.id)).toBe("connected"); + expect(parkSignals("codex")).toHaveLength(0); + }); + + it("carries on with the token a peer rotated while the issuer refused the old one", async () => { + const user = await seedCodexUser(); + // Another process won the race: by the time the issuer answers this + // caller "reused", the peer's rotated tokens are in the row. + vi.mocked(refreshDeviceTokens).mockImplementation(async () => { + const peer = encryptCodexCreds(creds("peer", 10 * HOUR_MS)); + await prisma.user.update({ + where: { id: user.id }, + data: { + codexAccessTokenEncrypted: peer.accessEncrypted, + codexRefreshTokenEncrypted: peer.refreshEncrypted, + }, + }); + throw new CodexRefreshError(400, "refresh_token_reused"); + }); + + const chain = await resolveProviderChain(user.id); + + expect(chain.map((p) => p.providerType)).toContain("codex"); + expect(await codexStatus(user.id)).toBe("connected"); + expect(parkSignals("codex")).toHaveLength(0); + }); +}); From 769d0334a7d502540b6afda14d0f49250f9373fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 00:06:21 +0200 Subject: [PATCH 20/23] fix(rollups): match the keyed refold's days in UTC whatever the session zone The keyed refold's day filter and delete compared the naive UTC measured_at and bucket_start with a timestamptz[] parameter, which goes through the session time zone; in any zone but UTC neither matched and the refold left its buckets stale. Both now bind timestamp[] literals. Keys are refolded in (type, day) order in chunks of 500, and the rollup-recompute queue describes the runs of buckets a job now covers. --- src/lib/jobs/queue-runtime.ts | 4 +- src/lib/rollups/measurement-rollups.ts | 37 +++++++-- .../rollup-keyed-refold-session-tz.test.ts | 76 +++++++++++++++++++ 3 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 tests/integration/rollup-keyed-refold-session-tz.test.ts diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 3c3d007ff..3bc085fd0 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -190,7 +190,9 @@ export const QUEUE_RUNTIME: Readonly> = { "daily-briefing": short("A light cohort scan every fifteen minutes."), // ── Rollups ──────────────────────────────────────────────────────── - "rollup-recompute": short("One bucket."), + "rollup-recompute": short( + "One type at one granularity over one bucket or a run of adjacent buckets, inside the five-year fold window.", + ), "rollup-full-backfill": { runtime: "long", expireInSeconds: ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS, diff --git a/src/lib/rollups/measurement-rollups.ts b/src/lib/rollups/measurement-rollups.ts index 0a9eae21b..a5277687f 100644 --- a/src/lib/rollups/measurement-rollups.ts +++ b/src/lib/rollups/measurement-rollups.ts @@ -362,8 +362,19 @@ export async function recomputeBucketsForMeasurement( ); } +/** + * UTC wall-clock literals for a `timestamp[]` parameter. `measured_at` and + * `bucket_start` are naive UTC timestamps, so the instants they are compared + * with must be too: a `timestamptz[]` parameter is converted through the + * session time zone, and in any zone but UTC neither the day filter nor the + * keyed delete matched a single day, so a refold left its buckets stale. + */ +function utcStamps(dates: readonly Date[]): string[] { + return dates.map((d) => d.toISOString().replace("Z", "")); +} + /** Keys one set-based DAY refold carries; bounds the statement's arrays. */ -const KEYED_REFOLD_CHUNK = 2_000; +const KEYED_REFOLD_CHUNK = 500; /** * The set-based sibling of {@link recomputeBucketsForMeasurement}, for a write @@ -381,7 +392,17 @@ export async function recomputeBucketsForKeys( userId: string, rows: Array<{ type: MeasurementType; measuredAt: Date }>, ): Promise { - const keys = collapseToTypeDayKeys(rows); + // Sorted on (type, day), the order every upsert of the tier takes its row + // locks in (`persistRollupRows` sorts the same way), and chunked small, so + // a keyed refold racing the full fold or another refold holds a short, + // deterministic slice and waits rather than interleaving lock orders. + const keys = collapseToTypeDayKeys(rows).sort((a, b) => + a.type !== b.type + ? a.type < b.type + ? -1 + : 1 + : a.measuredAt.getTime() - b.measuredAt.getTime(), + ); for (let i = 0; i < keys.length; i += KEYED_REFOLD_CHUNK) { await refoldDayBuckets(userId, keys.slice(i, i + KEYED_REFOLD_CHUNK)); } @@ -441,11 +462,11 @@ async function refoldDayBuckets( AND granularity = 'DAY'::"measurement_rollup_granularity" AND (type, bucket_start) IN ( SELECT k.t::"measurement_type", k.b - FROM unnest($2::text[], $3::timestamptz[]) AS k(t, b) + FROM unnest($2::text[], $3::timestamp[]) AS k(t, b) )`, userId, keys.map((k) => k.type), - keys.map((k) => k.measuredAt.toISOString()), + utcStamps(keys.map((k) => k.measuredAt)), ); for (let i = 0; i < ordered.length; i += 500) { const { sql, params } = buildRollupUpsert( @@ -642,9 +663,9 @@ async function runRollupAggregate( if (input.types && input.types.length === 0) return []; // Bound as the fourth parameter, never spliced. const dayFilter = input.days - ? `AND date_trunc('day', m."measured_at") = ANY($4::timestamptz[])` + ? `AND date_trunc('day', m."measured_at") = ANY($4::timestamp[])` : ""; - const dayParams = input.days ? [input.days.map((d) => d.toISOString())] : []; + const dayParams = input.days ? [utcStamps(input.days)] : []; // Postgres rejects `date_trunc($1::text, m."measured_at")` in a // GROUP BY because the planner sees the parameterised expression @@ -857,7 +878,7 @@ async function applyHourlyMeanDay( AND m."deleted_at" IS NULL ${ input.days - ? `AND date_trunc('day', m."measured_at") = ANY($4::timestamptz[])` + ? `AND date_trunc('day', m."measured_at") = ANY($4::timestamp[])` : "" } GROUP BY m."type", m."source", date_trunc('hour', m."measured_at") @@ -869,7 +890,7 @@ async function applyHourlyMeanDay( input.userId, input.from, input.to, - ...(input.days ? [input.days.map((d) => d.toISOString())] : []), + ...(input.days ? [utcStamps(input.days)] : []), ); const byKey = new Map( means.map((m) => [ diff --git a/tests/integration/rollup-keyed-refold-session-tz.test.ts b/tests/integration/rollup-keyed-refold-session-tz.test.ts new file mode 100644 index 000000000..30e88bb3c --- /dev/null +++ b/tests/integration/rollup-keyed-refold-session-tz.test.ts @@ -0,0 +1,76 @@ +/** + * The keyed DAY refold does not depend on the session time zone. + * + * `measured_at` and `bucket_start` are naive UTC timestamps. The refold's day + * filter and its delete compared them with a `timestamptz[]` parameter, which + * Postgres converts through the session time zone: outside UTC neither matched + * a single day, so the refold silently left every touched bucket stale. The + * app pins its sessions to UTC, so this runs the pool in another zone on + * purpose. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/db-session", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, TIME_ZONE_OPTION: "-c TimeZone=Pacific/Auckland" }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { prisma } from "@/lib/db"; +import { + recomputeBucketsForKeys, + recomputeBucketsForMeasurement, +} from "@/lib/rollups/measurement-rollups"; + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); +}); + +describe("keyed DAY refold in a non-UTC session", () => { + it("folds a new reading into its day", async () => { + const [{ zone }] = await prisma.$queryRaw>` + SELECT current_setting('TimeZone') AS zone + `; + expect(zone).toBe("Pacific/Auckland"); + + const user = await getPrismaClient().user.create({ + data: { username: "tz-refold", email: "tz-refold@example.test" }, + }); + const day = new Date("2026-05-10T00:00:00.000Z"); + await getPrismaClient().measurement.createMany({ + data: [7, 19].map((hour, i) => ({ + userId: user.id, + type: "WEIGHT" as const, + value: 80 + i, + unit: "kg", + measuredAt: new Date(day.getTime() + hour * 3_600_000), + })), + }); + await recomputeBucketsForMeasurement(user.id, "WEIGHT", day); + // A third reading lands; the refold has to fold it in. + await getPrismaClient().measurement.create({ + data: { + userId: user.id, + type: "WEIGHT", + value: 82, + unit: "kg", + measuredAt: new Date(day.getTime() + 21 * 3_600_000), + }, + }); + + await recomputeBucketsForKeys(user.id, [ + { type: "WEIGHT", measuredAt: new Date(day.getTime() + 7 * 3_600_000) }, + ]); + + const rows = await getPrismaClient().measurementRollup.findMany({ + where: { userId: user.id, granularity: "DAY" }, + }); + expect(rows).toHaveLength(1); + expect(rows[0]!.bucketStart.toISOString()).toBe(day.toISOString()); + expect(rows[0]!.count).toBe(3); + }); +}); From 28c840692421b8b356b1d3275425be2a53c9eadf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 01:16:51 +0200 Subject: [PATCH 21/23] fix(ai): park a revoked Codex family and stop at a disconnect mid-refresh Under the refresh lock, with the row still holding the token just spent, no peer rotated it, so refresh_token_reused is the issuer refusing the whole family: it now parks the sign-in instead of being asked again after every cooldown. Outside the lock it stays a lost race. A refresh whose compare-and-set write finds the connection gone (the person disconnected Codex meanwhile) now refuses with reconnect-required instead of using the rotated token once more. --- src/lib/ai/provider.ts | 26 +++++++-- .../integration/codex-credential-park.test.ts | 54 +++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/src/lib/ai/provider.ts b/src/lib/ai/provider.ts index 2a5e24b1e..9b03bfc5d 100644 --- a/src/lib/ai/provider.ts +++ b/src/lib/ai/provider.ts @@ -14,6 +14,7 @@ import { decryptCodexCreds, encryptAdminCodexCreds, decryptAdminCodexCreds, + CodexRefreshError, type CodexCreds, } from "./codex-oauth"; import { isPublicUrl } from "@/lib/validations/notifications"; @@ -287,16 +288,33 @@ async function runCodexRefresh( // A sign-in replaced the token while the issuer refused the old one. return { fresh: now.creds }; } - if (!isDeadCodexSignIn(error)) return { failed: error }; + // `refresh_token_reused` is a dead grant here and only here: under the + // lock, with the row still holding the very token just spent, no peer + // of ours rotated it, so the issuer is refusing the whole family (a + // revoked sign-in answers every refresh so). Outside the lock it is + // the lost race it usually is (see DEAD_REFRESH_TOKEN_CODES). + const refused = + oauthErrorOf(error) === "refresh_token_reused" + ? new CodexRefreshError(401, "refresh_token_reused") + : error; + if (!isDeadCodexSignIn(refused)) return { failed: error }; return { - refused: error, + refused, parked: (await store.park(tx, row.refreshEncrypted)) === 1, }; } if ((await store.writeRotated(tx, row.refreshEncrypted, fresh)) === 0) { - // A sign-in replaced the row meanwhile; it wins over this refresh. + // A sign-in replaced the row meanwhile; it wins over this refresh. A + // row with no usable credential left was disconnected meanwhile: the + // person took Codex away, so nothing may be sent with it any more. const now = await store.read(tx); - if (now?.creds) return { fresh: now.creds }; + if (now?.status === "connected" && now.creds) { + return { fresh: now.creds }; + } + return { + refused: codexReconnectRequired("disconnected during refresh"), + parked: false, + }; } return { fresh }; }, diff --git a/tests/integration/codex-credential-park.test.ts b/tests/integration/codex-credential-park.test.ts index 811fefd0c..e4d283eb4 100644 --- a/tests/integration/codex-credential-park.test.ts +++ b/tests/integration/codex-credential-park.test.ts @@ -319,3 +319,57 @@ describe("two refreshes of one Codex token", () => { expect(parkSignals("codex")).toHaveLength(0); }); }); + +describe("a Codex refresh under the lock", () => { + it("parks a sign-in whose every refresh the issuer answers as reused", async () => { + const user = await seedCodexUser(); + // The issuer revoked the token family: no peer rotated anything, yet each + // refresh comes back reused. + vi.mocked(refreshDeviceTokens).mockRejectedValue( + new CodexRefreshError(400, "refresh_token_reused"), + ); + + const chain = await resolveProviderChain(user.id); + + expect(chain.map((p) => p.providerType)).not.toContain("codex"); + expect( + ( + await prisma.user.findUniqueOrThrow({ + where: { id: user.id }, + select: { codexConnectionStatus: true }, + }) + ).codexConnectionStatus, + ).toBe("expired"); + expect(parkSignals("codex")).toHaveLength(1); + expect(parkSignals("codex")[0][0].meta?.oauth_error).toBe( + "refresh_token_reused", + ); + }); + + it("does not use Codex once more when the person disconnected it meanwhile", async () => { + const user = await seedCodexUser(); + vi.mocked(refreshDeviceTokens).mockImplementation(async () => { + await prisma.user.update({ + where: { id: user.id }, + data: { + codexAccessTokenEncrypted: null, + codexRefreshTokenEncrypted: null, + codexConnectionStatus: "disconnected", + }, + }); + return creds("rotated", 10 * HOUR_MS); + }); + + const chain = await resolveProviderChain(user.id); + + expect(chain.map((p) => p.providerType)).not.toContain("codex"); + expect( + ( + await prisma.user.findUniqueOrThrow({ + where: { id: user.id }, + select: { codexConnectionStatus: true }, + }) + ).codexConnectionStatus, + ).toBe("disconnected"); + }); +}); From 192b6593a4d946b6aaac27992482c5d8094c9261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 01:17:20 +0200 Subject: [PATCH 22/23] fix(oauth): claim a signed callback only once its code can be exchanged The signed-state callbacks (Oura, Polar, Strava) claimed the state before the session, code and issuer checks. A denied consent comes back without a code, so it spent the claim on a `nocode` error page, and allowing on the same authorize URL afterwards never exchanged the valid code: the browser was sent to the stored error page instead. Every check that refuses a callback without touching the code now runs first; the claim still sits strictly before the exchange. Expired claims were swept only when a callback completed, so a quiet instance kept its last rows. The daily OAuth-state cleanup at 03:20 now sweeps them as well. --- prisma/schema.prisma | 4 +- .../api/oura/callback/__tests__/route.test.ts | 39 +++++++++- src/app/api/oura/callback/route.ts | 41 +++++----- .../polar/callback/__tests__/route.test.ts | 21 ++++- src/app/api/polar/callback/route.ts | 44 ++++++----- .../strava/callback/__tests__/route.test.ts | 26 +++++-- src/app/api/strava/callback/route.ts | 44 ++++++----- .../oauth-callback-claim-cleanup.test.ts | 77 +++++++++++++++++++ src/lib/jobs/job-outcome.ts | 2 + src/lib/jobs/oauth-callback-claim-cleanup.ts | 21 +++++ src/lib/jobs/reminder/cleanup-handlers.ts | 10 ++- src/lib/oauth/callback-claim.ts | 5 +- .../oauth-callback-single-use.test.ts | 39 +++++++++- 13 files changed, 296 insertions(+), 77 deletions(-) create mode 100644 src/lib/jobs/__tests__/oauth-callback-claim-cleanup.test.ts create mode 100644 src/lib/jobs/oauth-callback-claim-cleanup.ts diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e972d56af..5509b6aeb 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -4560,8 +4560,8 @@ model GoogleHealthOAuthState { /// sends it to the same place instead of to an error page. Holds no user id, /// token or code: `location` is the app's own result URL (`?oura=connected`, /// `?whoop=error&reason=…`). Rows outlive the state they guard by a few -/// minutes and are swept by expiry on the next settle -/// (`src/lib/oauth/callback-claim.ts`). +/// minutes and are swept by expiry when a callback completes and by the daily +/// OAuth-state cleanup (`src/lib/jobs/oauth-callback-claim-cleanup.ts`). model OAuthCallbackClaim { /// sha256(provider + NUL + state), hex. key String @id diff --git a/src/app/api/oura/callback/__tests__/route.test.ts b/src/app/api/oura/callback/__tests__/route.test.ts index 898390087..2806299e4 100644 --- a/src/app/api/oura/callback/__tests__/route.test.ts +++ b/src/app/api/oura/callback/__tests__/route.test.ts @@ -3,13 +3,17 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; // Single-use claiming is covered against Postgres in // tests/integration/oauth-callback-single-use.test.ts; here every callback // is the first and only one. +const { claimSpy } = vi.hoisted(() => ({ claimSpy: vi.fn() })); vi.mock("@/lib/oauth/callback-claim", () => ({ runOAuthCallbackOnce: ( _p: string, _s: string, _o: unknown, run: () => Promise, - ) => run(), + ) => { + claimSpy(); + return run(); + }, settleOAuthCallback: (_p: string, _s: string, run: () => Promise) => run(), consumeLedgerState: async ( @@ -227,3 +231,36 @@ describe("GET /api/oura/callback — connect-time first sync", () => { expect(res.headers.get("location")).toContain("oura=connected"); }); }); + +// A refused callback must not spend the state: Oura sends a denied consent +// back without a code, and the person may return to the same authorize URL +// and allow it. +describe("GET /api/oura/callback — claim only what can be exchanged", () => { + it("does not claim the state for a callback without a code", async () => { + const res = await GET(makeReq({ state: "S", cookie: "S" })); + expect(res.headers.get("location")).toContain("reason=nocode"); + expect(claimSpy).not.toHaveBeenCalled(); + }); + + it("does not claim the state for a foreign issuer or a cross-user session", async () => { + await GET( + makeReq({ + code: "c", + state: "S", + cookie: "S", + iss: "https://evil.example/oauth", + }), + ); + (getSession as ReturnType).mockResolvedValue({ + user: { id: "attacker" }, + }); + await GET(makeReq({ code: "c", state: "S", cookie: "S" })); + expect(claimSpy).not.toHaveBeenCalled(); + }); + + it("claims the state once before exchanging a valid code", async () => { + await GET(makeReq({ code: "c", state: "S", cookie: "S" })); + expect(claimSpy).toHaveBeenCalledTimes(1); + expect(exchangeMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/app/api/oura/callback/route.ts b/src/app/api/oura/callback/route.ts index 8871b3a12..6e7b7ed20 100644 --- a/src/app/api/oura/callback/route.ts +++ b/src/app/api/oura/callback/route.ts @@ -70,23 +70,12 @@ export const GET = apiHandler(async (request: NextRequest) => { } const userId = verified.userId; - return runOAuthCallbackOnce( - "oura", - state, - { - fallbackUrl: SETTINGS_URL(), - stateCookie: oauthStateCookieName("oura"), - }, - () => completeCallback(request, userId), - ); -}); - -async function completeCallback( - request: NextRequest, - userId: string, -): Promise { - const { searchParams } = new URL(request.url); - const code = searchParams.get("code"); + // Everything that can refuse the callback without the code runs before + // the claim, so a refused callback (a denied consent arrives without a + // code) never spends the state: the same authorize URL still works. The + // claim stays strictly before the exchange. + const params = new URL(request.url).searchParams; + const code = params.get("code"); const evt = getEvent(); if (evt) evt.setAuth({ user_id: userId, auth_method: "session" }); @@ -102,12 +91,28 @@ async function completeCallback( // RFC 9207: an authorization server that names itself must be Oura's. The // issuer also tells which token endpoint the code belongs to. Older apps // send none, which is fine. - const iss = searchParams.get("iss"); + const iss = params.get("iss"); if (!isAcceptedOuraIssuer(iss)) { annotate({ meta: { reason: "issuer" } }); return ERR("issuer"); } + return runOAuthCallbackOnce( + "oura", + state, + { + fallbackUrl: SETTINGS_URL(), + stateCookie: oauthStateCookieName("oura"), + }, + () => completeCallback(userId, code, iss), + ); +}); + +async function completeCallback( + userId: string, + code: string, + iss: string | null, +): Promise { try { const creds = await getOuraClientCredentials(userId); if (!creds) return ERR("nocreds"); diff --git a/src/app/api/polar/callback/__tests__/route.test.ts b/src/app/api/polar/callback/__tests__/route.test.ts index 18e5c6847..9addbc134 100644 --- a/src/app/api/polar/callback/__tests__/route.test.ts +++ b/src/app/api/polar/callback/__tests__/route.test.ts @@ -25,13 +25,17 @@ const { // Single-use claiming is covered against Postgres in // tests/integration/oauth-callback-single-use.test.ts; here every callback // is the first and only one. +const { claimSpy } = vi.hoisted(() => ({ claimSpy: vi.fn() })); vi.mock("@/lib/oauth/callback-claim", () => ({ runOAuthCallbackOnce: ( _p: string, _s: string, _o: unknown, run: () => Promise, - ) => run(), + ) => { + claimSpy(); + return run(); + }, settleOAuthCallback: (_p: string, _s: string, run: () => Promise) => run(), consumeLedgerState: async ( @@ -78,9 +82,9 @@ import { GET } from "../route"; process.env.NEXT_PUBLIC_APP_URL = "https://app.example"; -function makeReq(): Parameters[0] { +function makeReq(query = "code=abc&state=s"): Parameters[0] { return { - url: "https://app.example/api/polar/callback?code=abc&state=s", + url: `https://app.example/api/polar/callback?${query}`, cookies: { get: () => ({ value: "s" }) }, } as unknown as Parameters[0]; } @@ -163,3 +167,14 @@ describe("GET /api/polar/callback — connect-time first sync", () => { expect(res.headers.get("location")).toContain("polar=connected"); }); }); + +describe("GET /api/polar/callback — claim only what can be exchanged", () => { + it("does not claim the state for a callback without a code or for another user", async () => { + const denied = await run(makeReq("state=s&error=access_denied")); + expect(denied.headers.get("location")).toContain("reason=nocode"); + getSessionMock.mockResolvedValue({ user: { id: "other" } }); + await run(makeReq()); + expect(claimSpy).not.toHaveBeenCalled(); + expect(exchangeCodeMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/app/api/polar/callback/route.ts b/src/app/api/polar/callback/route.ts index b18b1be34..cca2ff691 100644 --- a/src/app/api/polar/callback/route.ts +++ b/src/app/api/polar/callback/route.ts @@ -60,26 +60,10 @@ export const GET = apiHandler(async (request: NextRequest) => { } const userId = verified.userId; - // Single-use: a callback delivered twice exchanges its code once, and the - // duplicate follows the first request's result (src/lib/oauth/callback-claim.ts). - return runOAuthCallbackOnce( - "polar", - state, - { - fallbackUrl: new URL( - "/settings/integrations", - process.env.NEXT_PUBLIC_APP_URL!, - ).toString(), - stateCookie: oauthStateCookieName("polar"), - }, - () => completeCallback(request, userId), - ); -}); - -async function completeCallback( - request: NextRequest, - userId: string, -): Promise { + // Everything that can refuse the callback without the code runs before + // the claim, so a refused callback (a denied consent arrives without a + // code) never spends the state: the same authorize URL still works. The + // claim stays strictly before the exchange. const code = new URL(request.url).searchParams.get("code"); const evt = getEvent(); @@ -94,6 +78,26 @@ async function completeCallback( if (!code) return ERR("nocode"); + // Single-use: a callback delivered twice exchanges its code once, and the + // duplicate follows the first request's result (src/lib/oauth/callback-claim.ts). + return runOAuthCallbackOnce( + "polar", + state, + { + fallbackUrl: new URL( + "/settings/integrations", + process.env.NEXT_PUBLIC_APP_URL!, + ).toString(), + stateCookie: oauthStateCookieName("polar"), + }, + () => completeCallback(userId, code), + ); +}); + +async function completeCallback( + userId: string, + code: string, +): Promise { try { const creds = await getPolarClientCredentials(userId); if (!creds) return ERR("nocreds"); diff --git a/src/app/api/strava/callback/__tests__/route.test.ts b/src/app/api/strava/callback/__tests__/route.test.ts index 407cff3b5..c967bce51 100644 --- a/src/app/api/strava/callback/__tests__/route.test.ts +++ b/src/app/api/strava/callback/__tests__/route.test.ts @@ -28,13 +28,17 @@ const { // Single-use claiming is covered against Postgres in // tests/integration/oauth-callback-single-use.test.ts; here every callback // is the first and only one. +const { claimSpy } = vi.hoisted(() => ({ claimSpy: vi.fn() })); vi.mock("@/lib/oauth/callback-claim", () => ({ runOAuthCallbackOnce: ( _p: string, _s: string, _o: unknown, run: () => Promise, - ) => run(), + ) => { + claimSpy(); + return run(); + }, settleOAuthCallback: (_p: string, _s: string, run: () => Promise) => run(), consumeLedgerState: async ( @@ -85,11 +89,12 @@ import { GET } from "../route"; process.env.NEXT_PUBLIC_APP_URL = "https://app.example"; -function callback(): Promise<{ headers: { get(k: string): string | null } }> { - const req = new NextRequest( - "http://localhost/api/strava/callback?code=abc&state=signed-state", - { headers: { cookie: "strava_oauth_state=signed-state" } }, - ); +function callback( + query = "code=abc&state=signed-state", +): Promise<{ headers: { get(k: string): string | null } }> { + const req = new NextRequest(`http://localhost/api/strava/callback?${query}`, { + headers: { cookie: "strava_oauth_state=signed-state" }, + }); return (GET as unknown as (r: NextRequest) => Promise)(req); } @@ -127,3 +132,12 @@ describe("GET /api/strava/callback — S-1 backfill enqueue", () => { expect(bossSend).not.toHaveBeenCalled(); }); }); + +describe("GET /api/strava/callback — claim only what can be exchanged", () => { + it("does not claim the state for a callback without a code", async () => { + const res = await callback("state=signed-state&error=access_denied"); + expect(res.headers.get("location")).toContain("reason=nocode"); + expect(claimSpy).not.toHaveBeenCalled(); + expect(exchangeCodeMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/app/api/strava/callback/route.ts b/src/app/api/strava/callback/route.ts index 807c36d4c..1fc7ddc2b 100644 --- a/src/app/api/strava/callback/route.ts +++ b/src/app/api/strava/callback/route.ts @@ -60,26 +60,10 @@ export const GET = apiHandler(async (request: NextRequest) => { } const userId = verified.userId; - // Single-use: a callback delivered twice exchanges its code once, and the - // duplicate follows the first request's result (src/lib/oauth/callback-claim.ts). - return runOAuthCallbackOnce( - "strava", - state, - { - fallbackUrl: new URL( - "/settings/integrations", - process.env.NEXT_PUBLIC_APP_URL!, - ).toString(), - stateCookie: oauthStateCookieName("strava"), - }, - () => completeCallback(request, userId), - ); -}); - -async function completeCallback( - request: NextRequest, - userId: string, -): Promise { + // Everything that can refuse the callback without the code runs before + // the claim, so a refused callback (a denied consent arrives without a + // code) never spends the state: the same authorize URL still works. The + // claim stays strictly before the exchange. const code = new URL(request.url).searchParams.get("code"); const evt = getEvent(); @@ -93,6 +77,26 @@ async function completeCallback( if (!code) return ERR("nocode"); + // Single-use: a callback delivered twice exchanges its code once, and the + // duplicate follows the first request's result (src/lib/oauth/callback-claim.ts). + return runOAuthCallbackOnce( + "strava", + state, + { + fallbackUrl: new URL( + "/settings/integrations", + process.env.NEXT_PUBLIC_APP_URL!, + ).toString(), + stateCookie: oauthStateCookieName("strava"), + }, + () => completeCallback(userId, code), + ); +}); + +async function completeCallback( + userId: string, + code: string, +): Promise { try { const creds = await getStravaClientCredentials(userId); if (!creds) return ERR("nocreds"); diff --git a/src/lib/jobs/__tests__/oauth-callback-claim-cleanup.test.ts b/src/lib/jobs/__tests__/oauth-callback-claim-cleanup.test.ts new file mode 100644 index 000000000..b226a850d --- /dev/null +++ b/src/lib/jobs/__tests__/oauth-callback-claim-cleanup.test.ts @@ -0,0 +1,77 @@ +/** + * Expired OAuth callback claims are swept by the daily OAuth-state cleanup, + * not only when a callback completes: a quiet instance may not see another + * connect for weeks. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const h = vi.hoisted(() => ({ + withingsCleanup: vi.fn(), + claimDeleteMany: vi.fn(), + meta: {} as Record, +})); + +vi.mock("@/lib/jobs/withings-oauth-state-cleanup", () => ({ + cleanupExpiredWithingsOAuthStates: h.withingsCleanup, +})); +vi.mock("../reminder/shared", async (importOriginal) => ({ + ...(await importOriginal>()), + getWorkerPrisma: () => ({ + oAuthCallbackClaim: { deleteMany: h.claimDeleteMany }, + }), +})); +vi.mock("@/lib/logging/background", () => ({ + withBackgroundEvent: async ( + _name: string, + fn: (evt: { + addMeta: (k: string, v: unknown) => void; + addWarning: (w: string) => void; + }) => Promise, + ) => + fn({ + addMeta: (k, v) => { + h.meta[k] = v; + }, + addWarning: () => {}, + }), +})); + +import { cleanupExpiredOAuthCallbackClaims } from "../oauth-callback-claim-cleanup"; +import { handleWithingsOAuthStateCleanup } from "../reminder/cleanup-handlers"; + +beforeEach(() => { + vi.clearAllMocks(); + for (const key of Object.keys(h.meta)) delete h.meta[key]; + h.withingsCleanup.mockResolvedValue(0); + h.claimDeleteMany.mockResolvedValue({ count: 3 }); +}); + +describe("cleanupExpiredOAuthCallbackClaims", () => { + it("deletes the claims whose expiry has passed and returns the count", async () => { + const now = new Date("2026-10-21T03:20:00Z"); + const deleted = await cleanupExpiredOAuthCallbackClaims( + { oAuthCallbackClaim: { deleteMany: h.claimDeleteMany } } as never, + now, + ); + expect(deleted).toBe(3); + expect(h.claimDeleteMany).toHaveBeenCalledWith({ + where: { expiresAt: { lt: now } }, + }); + }); +}); + +describe("daily OAuth-state cleanup", () => { + it("sweeps expired callback claims alongside the Withings states", async () => { + await handleWithingsOAuthStateCleanup([]); + expect(h.withingsCleanup).toHaveBeenCalledTimes(1); + expect(h.claimDeleteMany).toHaveBeenCalledTimes(1); + expect(h.meta.oauth_callback_claim_cleanup_deleted).toBe(3); + }); + + it("fails the job when the claim sweep fails", async () => { + h.claimDeleteMany.mockRejectedValue(new Error("statement timeout")); + await expect(handleWithingsOAuthStateCleanup([])).resolves.toMatchObject({ + ok: false, + }); + }); +}); diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index d75c1b29f..1460c78e4 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -63,6 +63,8 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "buckets_recomputed", "budget_blocked", "cached", + // The daily Withings OAuth-state cleanup also sweeps the callback claims. + "callback_claims_deleted", "candidates_scanned", // The nightly sign-in ledger cleanup (`auth-ledger-cleanup`). "challenges_deleted", diff --git a/src/lib/jobs/oauth-callback-claim-cleanup.ts b/src/lib/jobs/oauth-callback-claim-cleanup.ts new file mode 100644 index 000000000..cd2ba7a5a --- /dev/null +++ b/src/lib/jobs/oauth-callback-claim-cleanup.ts @@ -0,0 +1,21 @@ +/** + * Daily sweep for `oauth_callback_claims`. + * + * A claim is taken when a connect callback is about to exchange its code + * (`src/lib/oauth/callback-claim.ts`) and is useless once its state has + * expired. The callback itself sweeps expired rows whenever one completes, so + * on a quiet instance the last few claims would stay until the next connect. + * This pass rides the daily Withings OAuth-state cleanup (03:20 Europe/Berlin) + * so the table empties on its own regardless. Idempotent. + */ +import type { PrismaClient } from "@/generated/prisma/client"; + +export async function cleanupExpiredOAuthCallbackClaims( + prisma: Pick, + now: Date = new Date(), +): Promise { + const { count } = await prisma.oAuthCallbackClaim.deleteMany({ + where: { expiresAt: { lt: now } }, + }); + return count; +} diff --git a/src/lib/jobs/reminder/cleanup-handlers.ts b/src/lib/jobs/reminder/cleanup-handlers.ts index e15318865..b5d28bedf 100644 --- a/src/lib/jobs/reminder/cleanup-handlers.ts +++ b/src/lib/jobs/reminder/cleanup-handlers.ts @@ -1,5 +1,5 @@ /** - * Retention and maintenance cleanups: rate-limit rows, idempotency keys, audit logs, OAuth states (Withings / WHOOP), mood-reminder events, push attempts, and measurement tombstones. + * Retention and maintenance cleanups: rate-limit rows, idempotency keys, audit logs, OAuth states (Withings / WHOOP) and OAuth callback claims, mood-reminder events, push attempts, and measurement tombstones. * * Extracted from reminder-worker.ts, which owns the queue names, cron * schedules, and the queue bindings. @@ -33,6 +33,7 @@ import { cleanupOldAuditLogs } from "@/lib/jobs/audit-log-cleanup"; import { cleanupOldCoachMessages } from "@/lib/jobs/coach-message-cleanup"; import { cleanupOldWorkoutInsightClaims } from "@/lib/jobs/workout-insight-claim-cleanup"; import { cleanupExpiredWithingsOAuthStates } from "@/lib/jobs/withings-oauth-state-cleanup"; +import { cleanupExpiredOAuthCallbackClaims } from "@/lib/jobs/oauth-callback-claim-cleanup"; import { cleanupExpiredOidcNativeHandoffs } from "@/lib/jobs/oidc-handoff-cleanup"; import { cleanupExpiredMeasurementTombstones, @@ -462,7 +463,12 @@ export async function handleWithingsOAuthStateCleanup( try { const deleted = await cleanupExpiredWithingsOAuthStates(p); evt.addMeta("withings_oauth_state_cleanup_deleted", deleted); - return jobDone({ deleted }); + // The provider-neutral callback claims ride this daily pass: the + // callbacks sweep them only when one completes, which a quiet + // instance may not see for weeks. + const claimsDeleted = await cleanupExpiredOAuthCallbackClaims(p); + evt.addMeta("oauth_callback_claim_cleanup_deleted", claimsDeleted); + return jobDone({ deleted, callback_claims_deleted: claimsDeleted }); } catch (err) { // The OAuth flow tolerates a stale row for an extra day, which used // to be the argument for swallowing this so the queue would not diff --git a/src/lib/oauth/callback-claim.ts b/src/lib/oauth/callback-claim.ts index dd88588d0..f9b0969a1 100644 --- a/src/lib/oauth/callback-claim.ts +++ b/src/lib/oauth/callback-claim.ts @@ -32,6 +32,7 @@ import { NextResponse } from "next/server"; import { prisma } from "@/lib/db"; import type { Prisma } from "@/generated/prisma/client"; import { annotate, getEvent } from "@/lib/logging/context"; +import { cleanupExpiredOAuthCallbackClaims } from "@/lib/jobs/oauth-callback-claim-cleanup"; /** Providers whose callbacks claim their state here. */ export type OAuthCallbackProvider = @@ -130,9 +131,7 @@ export async function settleOAuthCallback( : null, }, }); - await prisma.oAuthCallbackClaim.deleteMany({ - where: { expiresAt: { lt: now } }, - }); + await cleanupExpiredOAuthCallbackClaims(prisma, now); } catch (err) { getEvent()?.addWarning(`oauth callback claim settle failed: ${err}`); } diff --git a/tests/integration/oauth-callback-single-use.test.ts b/tests/integration/oauth-callback-single-use.test.ts index ec4e181e9..4c5a1b824 100644 --- a/tests/integration/oauth-callback-single-use.test.ts +++ b/tests/integration/oauth-callback-single-use.test.ts @@ -87,9 +87,13 @@ afterEach(() => { vi.restoreAllMocks(); }); -async function ouraCallback(state: string, code = "oura-code") { +async function ouraCallback(state: string, code: string | null = "oura-code") { const { GET } = await import("@/app/api/oura/callback/route"); - const params = new URLSearchParams({ code, state, iss: OURA_ISS }); + const params = new URLSearchParams( + code === null + ? { error: "access_denied", state, iss: OURA_ISS } + : { code, state, iss: OURA_ISS }, + ); return GET( new NextRequest(`http://localhost/api/oura/callback?${params}`, { method: "GET", @@ -221,6 +225,37 @@ describe("Oura callback delivered twice", () => { }); }); +describe("Oura consent denied, then allowed", () => { + it("still exchanges the code when the person returns to the same authorize URL", async () => { + let tokenCalls = 0; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + if (urlOf(input) !== OURA_TOKEN_URL) throw new Error("unexpected"); + tokenCalls++; + return new Response( + JSON.stringify({ + access_token: "oura-access", + refresh_token: "oura-refresh", + expires_in: 86400, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + + const state = await mintOuraState(); + // Deny: Oura returns without a code. Nothing is exchanged or claimed. + const denied = await ouraCallback(state, null); + expect(denied.headers.get("location")).toContain( + "oura=error&reason=nocode", + ); + expect(tokenCalls).toBe(0); + + // Allow on the same authorize URL: same state, now with a code. + const allowed = await ouraCallback(state, "allowed-code"); + expect(tokenCalls).toBe(1); + expect(allowed.headers.get("location")).toContain("oura=connected"); + }); +}); + describe("Withings callback delivered twice", () => { it("exchanges the code once and sends the duplicate where the first went", async () => { const prisma = getPrismaClient(); From bba33f13872b918b863a53e4c2c0fe07dc10b787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 01:23:43 +0200 Subject: [PATCH 23/23] chore(release): v1.43.9 --- CHANGELOG.md | 77 +++++++++++++++++++++++++++++++++++++++++++ docs/api/openapi.yaml | 2 +- package.json | 2 +- public/sw.js | 2 +- 4 files changed, 80 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5ec3a159..63e109287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,82 @@ # Changelog +## [1.43.9] — 2026-10-21 + +### Fixed + +- **Oura's resilience level syncs.** Resilience sits behind Oura's stress + permission, which HealthLog did not ask for, so it was refused even with + every other permission granted. HealthLog now asks for it. When Oura refuses + a single data type for a missing permission, the Oura card names exactly + that type, the refusal no longer triggers a token refresh and never counts + toward the reconnect warning. After updating, enable Stress for your app in + Oura's developer portal and connect Oura once more. (#1204) +- **Connecting an integration survives a repeated return link.** Some + browsers open the provider's return link twice. The second visit spent the + same one-time code again, and Oura answered by revoking the connection the + first visit had just made. Every connect flow (Oura, Polar, Strava, WHOOP, + Withings, Fitbit, Google Health) now claims the return link once; a second + visit waits for the first and lands on the same page. (#1204) +- **Two syncs no longer spend the same Oura sign-in.** Syncs that renewed the + Oura sign-in at the same moment each used the one-time renewal token, which + Oura can treat as reuse. Renewals now take turns. +- **Apple Health rows show the newest reading.** A type HealthLog also writes + back to Apple Health, such as weight entered here, showed the age of the + last sample Apple Health sent while the status above it already counted the + newer entry. The row now shows whichever is newer. (#1247) +- **An expired ChatGPT sign-in waits for you.** When the sign-in is refused as + expired, the connection is marked as needing a reconnect after the first + refusal and stays parked until you sign in again, instead of being retried + on every insight run. Signing in again puts it back in front at once. Two + renewals at the same moment take turns, so a working sign-in is never + parked by mistake. +- The demo instance no longer shows the setup checklist, whose steps cannot be + done in a read-only account. + +### Changed + +- **Faster syncs.** Withings syncs, full syncs included, write a batch in one + pass and skip readings that did not change, so a sync that took half a + minute finishes in about a second. Every connected service refreshes its + daily charts in one pass after a sync instead of one day at a time. +- **Faster dashboard and insights.** The all-time reading counts on the + dashboard and the all-time figures for weight, blood pressure and pulse are + read from the daily summaries instead of every reading, which makes them + many times faster for accounts with a lot of watch data. The all-time + figures fall back to the readings wherever the daily summaries do not match + them. The integrations settings card answers sooner on the first visit. +- **Lighter server.** Each signed-in request loads only the small account + fields it needs, so the profile picture and stored credentials are no longer + read into memory on every call. The server loads each page's code when it + is first used rather than all of it at start, so it starts with less than + half the memory and without a pause in its first minute; a page's first + visit after a restart takes a few milliseconds longer. +- The hourly sign-in location lookup no longer retries addresses it cannot + place on every run; it tries such an address again after a week. +- Requests that probe for server actions, which this app does not use, are + turned away with a 404 and counted in one warning line instead of filling + the error log. +- The periodic memory log line breaks the heap down by area, so a large heap + can be explained from the logs. + +### Security + +- The `hono` dependency floor is raised to 4.13.11 (GHSA-5r4p-p66f-jhc7). + +### Upgrade notes + +- Three additive migrations run at start (0417 to 0419). The first start after + the upgrade refills the daily summaries once in the background; until that + finishes, the all-time figures are read from the readings as before. +- Oura: enable the Stress permission for your app in Oura's developer portal, + then connect Oura once more and leave every permission ticked. + +### Thanks + +@haritowa for tracing the Oura resilience refusal to the missing permission +and for timing the repeated return link, @alondin for staying with #1204 +through every round, and @mills1975 for #1247. + ## [1.43.8] — 2026-10-20 ### Fixed diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 0ce16651c..1cc1af921 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: HealthLog API - version: 1.43.8 + version: 1.43.9 description: >- Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest. diff --git a/package.json b/package.json index f5567108f..6207709fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.43.8", + "version": "1.43.9", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "PolyForm-Noncommercial-1.0.0", "homepage": "https://healthlog.dev", diff --git a/public/sw.js b/public/sw.js index 4edfd1fee..81efa3b64 100644 --- a/public/sw.js +++ b/public/sw.js @@ -36,7 +36,7 @@ try { // v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild. const CACHE_VERSION = (typeof self !== "undefined" && self.__APP_VERSION__) || - /* @sw-version-fallback */ "v1.43.8"; + /* @sw-version-fallback */ "v1.43.9"; const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`; const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`; // v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*`