diff --git a/CHANGELOG.md b/CHANGELOG.md index bd9a1a7ee..6c53b6a09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ ## [Unreleased] +## [1.28.46] — 2026-07-16 — Server performance on large instances + +Performance work for accounts with a lot of history, none of it changing what +any number means: + +- Insight generation computed its 30-day comparison averages by pulling every + matching row of the last 400 days into memory and averaging in code. It now + asks the database for the averages directly — the same figures, without + materialising the rows. (Sleep keeps its per-night reconstruction path.) +- The background scans that run at worker startup used an in-memory + de-duplication over full-table reads. They now de-duplicate in the database + and are backed by partial indexes (migration 0243, additive), so startup on a + large instance no longer does full scans. +- The coach chat re-rendered every message on every streamed token; now only + the message being streamed re-renders. +- The doctor-report mood read no longer pulls fields it doesn't use. + ## [1.28.45] — 2026-07-16 — Doctor report + privacy fixes - Sleep now actually appears in the doctor-report PDF. The sleep section was diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 5b2e5dfaf..54273c02b 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: HealthLog API - version: 1.28.45 + version: 1.28.46 description: >- Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest. diff --git a/package.json b/package.json index 9f9fc92ce..644912f88 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.28.45", + "version": "1.28.46", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "PolyForm-Noncommercial-1.0.0", "homepage": "https://healthlog.dev", diff --git a/prisma/migrations/0243_perf_boot_discovery_partial_indexes/migration.sql b/prisma/migrations/0243_perf_boot_discovery_partial_indexes/migration.sql new file mode 100644 index 000000000..f01cf178c --- /dev/null +++ b/prisma/migrations/0243_perf_boot_discovery_partial_indexes/migration.sql @@ -0,0 +1,54 @@ +-- v1.28.46 perf (H4) — partial indexes for the boot-time converging-backfill +-- discovery scans. Each discovery now runs a DB-level `SELECT DISTINCT user_id` +-- over a growth table filtered by a "still-un-migrated" predicate; without an +-- index that is a full filtered-partition scan at worker boot (the boot-storm +-- class on a heavy tenant). Each index below is PARTIAL on exactly the +-- discovery predicate, so the scan is index-only and shrinks to nothing as the +-- one-time backfill converges (a migrated row drops out of the predicate). +-- +-- Additive-only: every statement is `CREATE INDEX IF NOT EXISTS`, no table +-- rewrite, no data change. NON-CONCURRENT on purpose — Prisma Migrate wraps a +-- migration file in a transaction and `CREATE INDEX CONCURRENTLY` cannot run in +-- one; these build at boot (docker-entrypoint runs `migrate deploy` BEFORE the +-- app serves traffic), so the brief build lock lands in the pre-traffic window. +-- Partial predicates match the write-side note convention (partial indexes are +-- raw-SQL-only here; Prisma cannot express a partial predicate — see the 0087 / +-- 0108 / 0183 precedents documented in schema.prisma). + +-- note-encryption-backfill: measurements (densest table) + mood_entries. +CREATE INDEX IF NOT EXISTS "measurements_note_backfill_idx" + ON "measurements" ("user_id") + WHERE "notes" IS NOT NULL AND "notes_encrypted" IS NULL; + +CREATE INDEX IF NOT EXISTS "mood_entries_note_backfill_idx" + ON "mood_entries" ("user_id") + WHERE "note" IS NOT NULL AND "note_encrypted" IS NULL; + +-- med-notes-encryption-backfill: side-effects, inventory items, dose-changes. +-- Dose-changes carry no user_id; the discovery joins to medications on +-- medication_id, so the partial index leads with that join key. +CREATE INDEX IF NOT EXISTS "medication_side_effects_note_backfill_idx" + ON "medication_side_effects" ("user_id") + WHERE "notes" IS NOT NULL AND "notes_encrypted" IS NULL; + +CREATE INDEX IF NOT EXISTS "medication_inventory_items_note_backfill_idx" + ON "medication_inventory_items" ("user_id") + WHERE "notes" IS NOT NULL AND "notes_encrypted" IS NULL; + +CREATE INDEX IF NOT EXISTS "medication_dose_changes_note_backfill_idx" + ON "medication_dose_changes" ("medication_id") + WHERE "note" IS NOT NULL AND "note_encrypted" IS NULL; + +-- lab-biomarker-backfill: unlinked live lab results. +CREATE INDEX IF NOT EXISTS "lab_results_biomarker_backfill_idx" + ON "lab_results" ("user_id") + WHERE "deleted_at" IS NULL AND "biomarker_id" IS NULL; + +-- document-thumbnail-backfill: live thumbnailable documents. The thumbnail +-- absence lives on the `document_thumbnails` side table (a cross-table +-- anti-join, not indexable here); this partial index bounds the document-side +-- scan on `(user_id, mime_type)` for live rows, and the anti-join rides +-- `document_thumbnails.document_id` (already UNIQUE via the 1:1 relation). +CREATE INDEX IF NOT EXISTS "inbound_documents_thumbnail_backfill_idx" + ON "inbound_documents" ("user_id", "mime_type") + WHERE "deleted_at" IS NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 80848d888..39c642f81 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1707,6 +1707,11 @@ model Measurement { // and series reads for every type NOT covered by the two partials above // and removes the `deleted_at` heap re-check. Same Prisma-can't-express-a- // partial-predicate reason it is omitted here. + // `measurements_note_backfill_idx` is a partial index over `(user_id) + // WHERE notes IS NOT NULL AND notes_encrypted IS NULL` — see migration 0243 + // (v1.28.46 H4). It makes the note-encryption boot-discovery `SELECT DISTINCT + // user_id` an index-only scan that converges to empty. Same partial-predicate + // reason it is omitted here. @@map("measurements") } @@ -2719,6 +2724,11 @@ model MedicationDoseChange { medication Medication @relation(fields: [medicationId], references: [id], onDelete: Cascade) @@index([medicationId, effectiveFrom]) + // `medication_dose_changes_note_backfill_idx` is a partial index over + // `(medication_id) WHERE note IS NOT NULL AND note_encrypted IS NULL` — see + // migration 0243 (v1.28.46 H4). The med-notes boot-discovery joins dose- + // changes to medications on medication_id to reach the owner; the index + // leads with that join key. Raw SQL only (partial predicate). @@map("medication_dose_changes") } @@ -2817,6 +2827,10 @@ model MedicationInventoryItem { @@index([userId, medicationId, state]) @@index([userId, expiresAt]) + // `medication_inventory_items_note_backfill_idx` is a partial index over + // `(user_id) WHERE notes IS NOT NULL AND notes_encrypted IS NULL` — see + // migration 0243 (v1.28.46 H4) for the med-notes boot-discovery scan. Raw + // SQL only (partial predicate not expressible in Prisma). @@map("medication_inventory_items") } @@ -2876,6 +2890,10 @@ model MedicationSideEffect { @@index([userId, medicationId, occurredAt]) @@index([userId, occurredAt]) + // `medication_side_effects_note_backfill_idx` is a partial index over + // `(user_id) WHERE notes IS NOT NULL AND notes_encrypted IS NULL` — see + // migration 0243 (v1.28.46 H4) for the med-notes boot-discovery scan. Raw + // SQL only (partial predicate not expressible in Prisma). @@map("medication_side_effects") } @@ -3539,6 +3557,10 @@ model MoodEntry { // v1.7.0 sync — keyset support for the `/api/sync/changes` mood feed // (`where userId` ordered by `(updatedAt asc, id asc)`). @@index([userId, updatedAt, id]) + // `mood_entries_note_backfill_idx` is a partial index over `(user_id) + // WHERE note IS NOT NULL AND note_encrypted IS NULL` — see migration 0243 + // (v1.28.46 H4), serving the note-encryption boot-discovery scan. Raw SQL + // only (partial predicate not expressible in Prisma). @@map("mood_entries") } @@ -4373,6 +4395,10 @@ model LabResult { @@index([userId, analyte, takenAt]) @@index([userId, deletedAt]) @@index([biomarkerId]) + // `lab_results_biomarker_backfill_idx` is a partial index over `(user_id) + // WHERE deleted_at IS NULL AND biomarker_id IS NULL` — see migration 0243 + // (v1.28.46 H4), serving the lab-biomarker boot-discovery `SELECT DISTINCT + // user_id`. Raw SQL only (partial predicate not expressible in Prisma). @@map("lab_results") } @@ -6055,6 +6081,11 @@ model InboundDocument { // a superset of the plain `(userId, deletedAt)` index, which is therefore // redundant for read planning and dropped. @@index([userId, deletedAt, documentDate]) + // `inbound_documents_thumbnail_backfill_idx` is a partial index over + // `(user_id, mime_type) WHERE deleted_at IS NULL` — see migration 0243 + // (v1.28.46 H4). It bounds the document-side scan of the thumbnail + // boot-discovery (which anti-joins document_thumbnails for the missing + // preview). Raw SQL only (partial predicate not expressible in Prisma). @@map("inbound_documents") } diff --git a/public/sw.js b/public/sw.js index f0792acaf..0870cf826 100644 --- a/public/sw.js +++ b/public/sw.js @@ -36,7 +36,7 @@ try { // v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild. const CACHE_VERSION = (typeof self !== "undefined" && self.__APP_VERSION__) || - /* @sw-version-fallback */ "v1.28.45"; + /* @sw-version-fallback */ "v1.28.46"; const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`; const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`; // v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*` diff --git a/src/components/insights/coach-panel/__tests__/chat-bubble-memo.test.tsx b/src/components/insights/coach-panel/__tests__/chat-bubble-memo.test.tsx new file mode 100644 index 000000000..da9e67e2e --- /dev/null +++ b/src/components/insights/coach-panel/__tests__/chat-bubble-memo.test.tsx @@ -0,0 +1,89 @@ +/** + * v1.28.46 perf (M3) — ChatBubble is wrapped in React.memo so a streamed + * token (which grows the live turn's content and re-runs the thread's + * messages.map) does not re-render every settled bubble. These tests pin the + * memo comparator contract: + * - a per-render `onRegenerate` closure identity change alone does NOT force + * a re-render (the thread rebuilds that closure every render), + * - a `content` or `streaming` change DOES, + * - every other render-affecting prop is compared, + * and that the exported component is actually memoized. + */ +import { describe, it, expect } from "vitest"; + +// Importing the module pulls the bubble's hook deps; stub them so the import +// is side-effect free (mirrors the sibling coach-charts harness). +import { vi } from "vitest"; +vi.mock("@/hooks/use-auth", () => ({ + useAuth: () => ({ user: null, isAuthenticated: true, isLoading: false }), +})); +vi.mock("@tanstack/react-query", () => ({ + useQuery: () => ({ data: undefined }), + useMutation: () => ({ mutate: vi.fn(), isPending: false, isError: false }), +})); + +import { ChatBubble, areChatBubblePropsEqual } from "../chat-bubble"; + +type Props = Parameters[0]; + +function baseProps(overrides: Partial = {}): Props { + return { + role: "assistant", + content: "hello", + metricSource: null, + providerType: "openai", + messageId: "m1", + createdAt: "2026-01-01T00:00:00.000Z", + ...overrides, + }; +} + +describe("areChatBubblePropsEqual", () => { + it("treats a fresh onRegenerate closure as equal (skips re-render)", () => { + const prev = baseProps({ onRegenerate: () => {} }); + const next = baseProps({ onRegenerate: () => {} }); + expect(areChatBubblePropsEqual(prev, next)).toBe(true); + }); + + it("re-renders when the streamed content changes", () => { + const prev = baseProps({ content: "hel", streaming: true }); + const next = baseProps({ content: "hell", streaming: true }); + expect(areChatBubblePropsEqual(prev, next)).toBe(false); + }); + + it("re-renders when the streaming flag flips", () => { + const prev = baseProps({ content: "done", streaming: true }); + const next = baseProps({ content: "done", streaming: false }); + expect(areChatBubblePropsEqual(prev, next)).toBe(false); + }); + + it("re-renders when onRegenerate presence changes (undefined -> defined)", () => { + const prev = baseProps({ onRegenerate: undefined }); + const next = baseProps({ onRegenerate: () => {} }); + expect(areChatBubblePropsEqual(prev, next)).toBe(false); + }); + + it("re-renders when a grounding / identity prop changes by reference", () => { + const prev = baseProps({ metricSource: null }); + const next = baseProps({ metricSource: { metrics: ["weight"] } as never }); + expect(areChatBubblePropsEqual(prev, next)).toBe(false); + + expect( + areChatBubblePropsEqual( + baseProps({ messageId: "m1" }), + baseProps({ messageId: "m2" }), + ), + ).toBe(false); + }); + + it("is stable for identical settled props", () => { + expect(areChatBubblePropsEqual(baseProps(), baseProps())).toBe(true); + }); +}); + +describe("ChatBubble memoization", () => { + it("is wrapped in React.memo", () => { + const typeTag = (ChatBubble as unknown as { $$typeof?: symbol }).$$typeof; + expect(String(typeTag)).toContain("memo"); + }); +}); diff --git a/src/components/insights/coach-panel/chat-bubble.tsx b/src/components/insights/coach-panel/chat-bubble.tsx index 49f3d7dd7..9d821ceae 100644 --- a/src/components/insights/coach-panel/chat-bubble.tsx +++ b/src/components/insights/coach-panel/chat-bubble.tsx @@ -1,6 +1,6 @@ "use client"; -import { useCallback, useEffect, useId, useRef, useState } from "react"; +import { memo, useCallback, useEffect, useId, useRef, useState } from "react"; import Link from "next/link"; import { useMutation } from "@tanstack/react-query"; import { toast } from "sonner"; @@ -409,7 +409,49 @@ interface ChatBubbleProps { onRegenerate?: () => void; } -export function ChatBubble({ +/** + * v1.28.46 perf (M3) — memo comparator for `ChatBubble`. The Coach thread + * re-renders on every streamed token (the live turn's content grows), which + * re-runs `messages.map` and re-creates every persisted bubble. Without memo + * each settled bubble re-runs `selectCoachChartTokens` + provenance work per + * token; on a long thread that is tokens × messages renders and drops frames. + * + * Every prop that changes the bubble's output is compared by value/reference. + * The ONE prop that is not stable across renders is `onRegenerate`: the thread + * builds a fresh `() => onRegenerate(precedingUserContent)` closure per render + * for each assistant message (message-thread.tsx), so a naive shallow memo + * would never skip. Its identity does not affect rendering (the closure is only + * invoked on click, and captures the same preceding user text on a settled + * thread), so it is compared by PRESENCE, not identity. Result: only the + * streaming bubble (whose `content`/`streaming` actually change) re-renders. + * + * Exported so the contract is unit-testable without standing up the thread. + */ +export function areChatBubblePropsEqual( + prev: ChatBubbleProps, + next: ChatBubbleProps, +): boolean { + return ( + prev.role === next.role && + prev.content === next.content && + prev.streaming === next.streaming && + prev.inProgress === next.inProgress && + prev.errorCode === next.errorCode && + prev.providerType === next.providerType && + prev.messageId === next.messageId && + prev.tokensUsed === next.tokensUsed && + prev.model === next.model && + prev.createdAt === next.createdAt && + prev.metricSource === next.metricSource && + prev.suggestion === next.suggestion && + prev.suggestedAction === next.suggestedAction && + prev.usage === next.usage && + // onRegenerate is a per-render closure — compare only whether it is present. + (prev.onRegenerate === undefined) === (next.onRegenerate === undefined) + ); +} + +function ChatBubbleImpl({ role, content, metricSource, @@ -809,6 +851,15 @@ export function ChatBubble({ ); } +/** + * v1.28.46 perf (M3) — memoized bubble. Skips re-render for every settled + * bubble while the streaming turn grows token-by-token; the streaming bubble + * (changed `content`/`streaming`) is the only one that re-renders. `displayName` + * is set so the name survives the memo wrapper in React DevTools. + */ +export const ChatBubble = memo(ChatBubbleImpl, areChatBubblePropsEqual); +ChatBubble.displayName = "ChatBubble"; + /** * v1.16.8 — per-message remember control under the user bubble. * diff --git a/src/lib/doctor-report-data.ts b/src/lib/doctor-report-data.ts index 8f7e0d904..2cf72fdfc 100644 --- a/src/lib/doctor-report-data.ts +++ b/src/lib/doctor-report-data.ts @@ -256,6 +256,13 @@ export async function collectDoctorReportData( moodLoggedAt: { gte: start, lte: end }, }, orderBy: { moodLoggedAt: "asc" }, + // v1.28.46 perf (H1) — narrow select. The collector reads exactly + // two fields off each mood row: `score` (the mood summary / + // distribution) and `tags` (GLP-1 side-effect tag counts). The + // full-width read pulled every column — including the encrypted + // `noteEncrypted` Bytes and the plaintext `note` free text — across + // the whole report window, none of which the report ever touches. + select: { score: true, tags: true }, }) : Promise.resolve([]), prisma.user.findUnique({ diff --git a/src/lib/insights/__tests__/comparison-snapshot-window.test.ts b/src/lib/insights/__tests__/comparison-snapshot-window.test.ts index 22d52888b..17411e666 100644 --- a/src/lib/insights/__tests__/comparison-snapshot-window.test.ts +++ b/src/lib/insights/__tests__/comparison-snapshot-window.test.ts @@ -1,23 +1,28 @@ /** - * v1.20.1 perf — the comparison-snapshot builder must bound its - * per-type measurement read to a ~400-day window instead of scanning a - * user's entire history. The snapshot only feeds summarize()'s avg30 / - * avg30LastMonth / avg30LastYear, and the widest of those reaches into - * the [365, 395)-day window, so nothing older than 395 days can change a - * result. These tests pin that: - * - every measurement read carries a `measuredAt.gte` lower bound, - * - the bound sits at ~400 days (5-day floor over the 395-day reach), - * - the three averages are unaffected for in-window data. + * v1.20.1 perf — the comparison-snapshot builder must bound its per-type + * measurement read instead of scanning a user's entire history. v1.28.46 + * perf — it now computes the two means it actually reads (avg30 + one + * baseline mean) as DB-side AVG aggregates over the exact windows, instead + * of pulling every raw row into JS and averaging there. The snapshot only + * feeds summarize()'s avg30 / avg30LastMonth / avg30LastYear, and only two + * of those are read per call. * - * The read fans out over all seven snapshot types on the page-blocking - * POST /api/insights/generate path AND the nightly pregenerate cron, so - * an unbounded scan is tens of thousands of rows per type on multi-year - * accounts. + * These tests pin: + * - non-sleep types compute via `measurement.aggregate({ _avg })`, never a + * raw row read, over the exact half-open windows summarize() uses, + * - SLEEP_DURATION stays on the raw ~400-day findMany path (per-night + * reconstruction cannot be a bare SQL AVG), + * - OUTPUT-EQUIVALENCE: the aggregate path produces byte-identical + * currentAvg / baselineAvg / delta / deltaPercent to the previous + * raw-findMany-then-summarize() implementation, for a fixture dataset. */ import { describe, it, expect, vi, beforeEach } from "vitest"; +import { summarize, type DataPoint } from "@/lib/analytics/trends"; + const findUnique = vi.fn(); const measurementFindMany = vi.fn(); +const measurementAggregate = vi.fn(); vi.mock("@/lib/db", () => ({ prisma: { @@ -26,10 +31,11 @@ vi.mock("@/lib/db", () => ({ }, measurement: { findMany: (...a: unknown[]) => measurementFindMany(...a), + aggregate: (...a: unknown[]) => measurementAggregate(...a), }, }, })); -// SLEEP_DURATION branch helpers — the test never exercises the sleep +// SLEEP_DURATION branch helpers — most tests never exercise the sleep // reconstruction maths, but the builder imports these unconditionally. vi.mock("@/lib/tz/resolver", () => ({ resolveUserTimezone: vi.fn(async () => "Europe/Berlin"), @@ -42,13 +48,31 @@ import { buildComparisonSnapshotForUser } from "../comprehensive-generate"; const DAY = 86_400_000; +/** + * Faithful in-memory stand-in for Postgres `AVG(value)` over a measured-at + * window: same half-open bounds Prisma would translate, same sum/count as the + * DB. Returns null on an empty slice (matching `_avg.value === null`). + */ +function avgWindow( + rows: { measuredAt: Date; value: number }[], + bounds: { gt?: Date; gte?: Date; lte?: Date; lt?: Date }, +): number | null { + const slice = rows.filter((r) => { + const t = r.measuredAt.getTime(); + if (bounds.gt && !(t > bounds.gt.getTime())) return false; + if (bounds.gte && !(t >= bounds.gte.getTime())) return false; + if (bounds.lte && !(t <= bounds.lte.getTime())) return false; + if (bounds.lt && !(t < bounds.lt.getTime())) return false; + return true; + }); + if (slice.length === 0) return null; + return slice.reduce((s, r) => s + r.value, 0) / slice.length; +} + beforeEach(() => { vi.clearAllMocks(); - // Comparison toggle ON → the builder runs its per-type reads. The - // baseline value drives which prior-period mean the snapshot reports. - // `resolveDashboardLayout` only honours `comparisonBaseline` on a blob - // that carries a numeric `version` + a `widgets` array; otherwise it - // clamps to "none". + // Comparison toggle ON → the builder runs its per-type reads. The baseline + // value drives which prior-period mean the snapshot reports. findUnique.mockResolvedValue({ dashboardWidgetsJson: { version: 1, @@ -56,49 +80,14 @@ beforeEach(() => { comparisonBaseline: "lastYear", }, }); - // No measurements by default; individual tests override per call. + // No sleep rows by default; the raw findMany path only serves SLEEP_DURATION. measurementFindMany.mockResolvedValue([]); + // No measurements by default; individual tests override per call. + measurementAggregate.mockResolvedValue({ _avg: { value: null } }); }); -describe("buildComparisonSnapshotForUser — bounded read window", () => { - it("bounds every measurement read with a ~400-day measuredAt.gte floor", async () => { - const before = Date.now(); - await buildComparisonSnapshotForUser("u1"); - const after = Date.now(); - - // One findMany per snapshot type (sleep takes a different select but - // the same where shape). Every call must carry the lower bound. - expect(measurementFindMany).toHaveBeenCalled(); - for (const call of measurementFindMany.mock.calls) { - const where = (call[0] as { where: Record }).where; - expect(where).toMatchObject({ userId: "u1", deletedAt: null }); - const measuredAt = where.measuredAt as { gte: Date }; - expect(measuredAt).toBeDefined(); - expect(measuredAt.gte).toBeInstanceOf(Date); - - // The floor is anchored on Date.now() - 400 days, read inside the call. - // Since before <= snapshotNow <= after, the floor brackets exactly to - // [before - 400d, after - 400d] — no tick-timing slack assumptions. - const gteMs = measuredAt.gte.getTime(); - expect(gteMs).toBeGreaterThanOrEqual(before - 400 * DAY); - expect(gteMs).toBeLessThanOrEqual(after - 400 * DAY); - } - }); - - it("keeps the floor strictly older than the 395-day reach of avg30LastYear", async () => { - const now = Date.now(); - await buildComparisonSnapshotForUser("u1"); - - // avg30LastYear reads points with age < 395 days. The bound must sit - // at least 5 days older so it never clips that window. - for (const call of measurementFindMany.mock.calls) { - const where = (call[0] as { where: { measuredAt: { gte: Date } } }).where; - const ageDays = (now - where.measuredAt.gte.getTime()) / DAY; - expect(ageDays).toBeGreaterThanOrEqual(395); - } - }); - - it("computes avg30 / avg30LastMonth / avg30LastYear unchanged for in-window data", async () => { +describe("buildComparisonSnapshotForUser — aggregate windows", () => { + it("computes non-sleep means via aggregate with the exact half-open windows", async () => { findUnique.mockResolvedValue({ dashboardWidgetsJson: { version: 1, @@ -106,30 +95,159 @@ describe("buildComparisonSnapshotForUser — bounded read window", () => { comparisonBaseline: "lastMonth", }, }); + const before = Date.now(); + await buildComparisonSnapshotForUser("u1"); + const after = Date.now(); + // Every non-sleep type issues exactly two aggregate reads (current + + // baseline). None issues a raw non-sleep findMany. + expect(measurementAggregate).toHaveBeenCalled(); + const currentCalls: { gt: Date }[] = []; + const baselineCalls: { gt: Date; lte: Date }[] = []; + for (const call of measurementAggregate.mock.calls) { + const where = (call[0] as { where: Record }).where; + expect(where).toMatchObject({ userId: "u1", deletedAt: null }); + const measuredAt = where.measuredAt as { + gt: Date; + lte?: Date; + }; + expect(measuredAt.gt).toBeInstanceOf(Date); + if (measuredAt.lte) + baselineCalls.push(measuredAt as { gt: Date; lte: Date }); + else currentCalls.push(measuredAt as { gt: Date }); + } + // current window: measuredAt > now - 30d (no upper bound). + for (const w of currentCalls) { + const gtMs = w.gt.getTime(); + expect(gtMs).toBeGreaterThanOrEqual(before - 30 * DAY); + expect(gtMs).toBeLessThanOrEqual(after - 30 * DAY); + } + // lastMonth baseline window: (now - 60d, now - 30d]. + for (const w of baselineCalls) { + const gtMs = w.gt.getTime(); + const lteMs = w.lte.getTime(); + expect(gtMs).toBeGreaterThanOrEqual(before - 60 * DAY); + expect(gtMs).toBeLessThanOrEqual(after - 60 * DAY); + expect(lteMs).toBeGreaterThanOrEqual(before - 30 * DAY); + expect(lteMs).toBeLessThanOrEqual(after - 30 * DAY); + } + // Six non-sleep types × 2 windows = 12 aggregate reads; no raw non-sleep read. + expect(currentCalls.length).toBe(6); + expect(baselineCalls.length).toBe(6); + }); + + it("reads SLEEP_DURATION through the raw ~400-day findMany, never aggregate", async () => { + await buildComparisonSnapshotForUser("u1"); + // Exactly one findMany — the SLEEP_DURATION branch — and it carries a + // ~400-day measuredAt.gte floor. + expect(measurementFindMany).toHaveBeenCalledTimes(1); const now = Date.now(); - // WEIGHT is the first non-sleep type. Seed three points, one in each - // of the avg30 (current), avg30LastMonth ([30,60)) windows — all well - // inside the 400-day floor. - measurementFindMany.mockImplementation( - async (args: { where: { type: string } }) => { - if (args.where.type !== "WEIGHT") return []; - return [ - { measuredAt: new Date(now - 5 * DAY), value: 80 }, // current - { measuredAt: new Date(now - 45 * DAY), value: 84 }, // lastMonth - ]; + const where = ( + measurementFindMany.mock.calls[0][0] as { + where: { type: string; measuredAt: { gte: Date } }; + } + ).where; + expect(where.type).toBe("SLEEP_DURATION"); + const ageDays = (now - where.measuredAt.gte.getTime()) / DAY; + expect(ageDays).toBeGreaterThanOrEqual(395); + }); + + it("baseline=lastYear uses the [365, 395)-day window (measuredAt > now-395d && <= now-365d)", async () => { + const before = Date.now(); + await buildComparisonSnapshotForUser("u1"); // default baseline = lastYear + const after = Date.now(); + const baselineCalls = measurementAggregate.mock.calls + .map( + (c) => + (c[0] as { where: { measuredAt: { gt: Date; lte?: Date } } }).where + .measuredAt, + ) + .filter((m): m is { gt: Date; lte: Date } => m.lte != null); + expect(baselineCalls.length).toBe(6); + for (const w of baselineCalls) { + const gtMs = w.gt.getTime(); + const lteMs = w.lte.getTime(); + expect(gtMs).toBeGreaterThanOrEqual(before - 395 * DAY); + expect(gtMs).toBeLessThanOrEqual(after - 395 * DAY); + expect(lteMs).toBeGreaterThanOrEqual(before - 365 * DAY); + expect(lteMs).toBeLessThanOrEqual(after - 365 * DAY); + } + }); +}); + +describe("buildComparisonSnapshotForUser — output-equivalence to the raw-average path", () => { + it("emits identical metrics to findMany-then-summarize() for a fixture (lastMonth)", async () => { + findUnique.mockResolvedValue({ + dashboardWidgetsJson: { + version: 1, + widgets: [], + comparisonBaseline: "lastMonth", + }, + }); + const now = Date.now(); + // A WEIGHT fixture with several rows spread across the current [0,30), + // lastMonth [30,60), and out-of-window regions — the kind of skew where a + // wrong window boundary or rounding step would diverge. + const weightRows: { measuredAt: Date; value: number }[] = [ + { measuredAt: new Date(now - 2 * DAY), value: 80.4 }, + { measuredAt: new Date(now - 9 * DAY), value: 81.1 }, + { measuredAt: new Date(now - 21 * DAY), value: 79.9 }, + { measuredAt: new Date(now - 33 * DAY), value: 84.2 }, + { measuredAt: new Date(now - 48 * DAY), value: 83.3 }, + { measuredAt: new Date(now - 59 * DAY), value: 85.0 }, + // out-of-window (older than lastMonth) — must not influence either mean. + { measuredAt: new Date(now - 120 * DAY), value: 99.0 }, + ]; + + // The aggregate mock is the "database": it applies the builder's window + // and averages the fixture rows. + measurementAggregate.mockImplementation( + async (args: { + where: { type: string; measuredAt: { gt?: Date; lte?: Date } }; + }) => { + if (args.where.type !== "WEIGHT") return { _avg: { value: null } }; + return { + _avg: { value: avgWindow(weightRows, args.where.measuredAt) }, + }; }, ); const snapshot = await buildComparisonSnapshotForUser("u1"); - expect(snapshot).not.toBeNull(); const weight = snapshot!.metrics.find((m) => m.type === "weight"); expect(weight).toBeDefined(); - // avg30 over the single current point. - expect(weight!.currentAvg).toBe(80); - // lastMonth baseline over the single [30,60) point. - expect(weight!.baselineAvg).toBe(84); - expect(weight!.delta).toBe(-4); + + // Reference: the PREVIOUS implementation — read the whole 400-day window + // then summarize() in JS. + const points: DataPoint[] = weightRows + .filter((r) => r.measuredAt.getTime() >= now - 400 * DAY) + .map((r) => ({ date: r.measuredAt, value: r.value })); + const summary = summarize(points); + const expectedCurrent = summary.avg30 ?? null; + const expectedBaseline = summary.avg30LastMonth ?? null; + const expectedDelta = + expectedCurrent !== null && expectedBaseline !== null + ? Math.round((expectedCurrent - expectedBaseline) * 100) / 100 + : null; + const expectedDeltaPercent = + expectedDelta !== null && + expectedBaseline !== null && + expectedBaseline !== 0 + ? Math.round((expectedDelta / Math.abs(expectedBaseline)) * 100 * 10) / + 10 + : null; + + expect(weight!.currentAvg).toBe(expectedCurrent); + expect(weight!.baselineAvg).toBe(expectedBaseline); + expect(weight!.delta).toBe(expectedDelta); + expect(weight!.deltaPercent).toBe(expectedDeltaPercent); + }); + + it("drops a metric with no data in either window (parity with the old filter)", async () => { + // Default aggregate mock returns null for every window → every non-sleep + // metric is null/null and filtered out; sleep has no rows too. + const snapshot = await buildComparisonSnapshotForUser("u1"); + expect(snapshot).not.toBeNull(); + expect(snapshot!.metrics).toEqual([]); }); it("returns null without reading measurements when the comparison toggle is off", async () => { @@ -142,6 +260,7 @@ describe("buildComparisonSnapshotForUser — bounded read window", () => { }); const snapshot = await buildComparisonSnapshotForUser("u1"); expect(snapshot).toBeNull(); + expect(measurementAggregate).not.toHaveBeenCalled(); expect(measurementFindMany).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/insights/comprehensive-generate.ts b/src/lib/insights/comprehensive-generate.ts index 238fe1ed0..61e1d33ea 100644 --- a/src/lib/insights/comprehensive-generate.ts +++ b/src/lib/insights/comprehensive-generate.ts @@ -18,6 +18,7 @@ * NO synchronous request lifecycle: the cron path runs entirely on * pg-boss, never inside an HTTP handler. */ +import pLimit from "p-limit"; import { prisma } from "@/lib/db"; import { extractFeatures, @@ -313,91 +314,127 @@ export async function buildComparisonSnapshotForUser( ACTIVITY_STEPS: "", }; const types = Object.keys(typeToSnapshotKey) as MeasurementType[]; - // The snapshot only feeds summarize()'s avg30 / avg30LastMonth / - // avg30LastYear. The widest of those reaches back into the [365, 395)-day - // window (ageMs < 395 * DAY in meanOfWindow), so nothing older than 395 - // days can change a result. Bound the read at 400 days — a 5-day floor over - // the strict-less-than boundary — instead of scanning the full history, - // which on multi-year accounts is tens of thousands of rows per type on the - // page-blocking generate path and the nightly pregenerate cron. - const sinceMeasuredAt = new Date(Date.now() - 400 * 86_400_000); + + // The snapshot only ever reads two of summarize()'s means per call: avg30 + // (the "current" 30-day mean) and exactly ONE baseline mean — avg30LastMonth + // (window [30, 60) days ago) or avg30LastYear (window [365, 395) days ago), + // chosen by `baseline`. Rather than pull every raw row across a 400-day + // window into JS and average there (tens of thousands of rows per dense type + // on the page-blocking generate path AND the nightly pregenerate cron), + // compute each mean as a DB-side AVG over the exact window the JS reducer + // covered. The 2-dp rounding and the half-open window bounds are matched to + // summarize()/meanOfWindow() byte-for-byte, so the emitted metrics are + // numerically identical to the previous raw-scan implementation. + // + // SLEEP_DURATION stays on the raw path: it is stored ONE ROW PER STAGE per + // night, so a bare SQL AVG would blend per-stage rows and double-count a + // bare ASLEEP aggregate against its granular twin. Only the per-night dedup + // reconstruction yields a meaningful nightly TIME-ASLEEP total. + // + // Anchor every window on ONE clock read so the aggregate reads and the sleep + // reconstruction share an identical "now" (summarize() reads the clock once + // too). Bounds mirror summarize(): avg30 keeps `now - measuredAt < 30d` + // (measuredAt > now-30d, no upper bound); meanOfWindow(a,b) keeps + // `a*DAY <= age < b*DAY` (measuredAt > now-b AND measuredAt <= now-a). + const nowMs = Date.now(); + const DAY = 86_400_000; + const currentWhere: { gt: Date } = { gt: new Date(nowMs - 30 * DAY) }; + const baselineWhere: { gt: Date; lte: Date } = + baseline === "lastMonth" + ? { gt: new Date(nowMs - 60 * DAY), lte: new Date(nowMs - 30 * DAY) } + : { gt: new Date(nowMs - 395 * DAY), lte: new Date(nowMs - 365 * DAY) }; + + // Rounded (2 dp) DB-side mean of `value` over one measured-at window, or null + // when the window holds no live rows — matching summarize()'s avg30 / + // meanOfWindow exactly (both round with Math.round(x * 100) / 100 and return + // null on an empty slice). + const avgOverWindow = async ( + type: MeasurementType, + measuredAt: { gt: Date; lte?: Date }, + ): Promise => { + const agg = await prisma.measurement.aggregate({ + where: { userId, type, deletedAt: null, measuredAt }, + _avg: { value: true }, + }); + const avg = agg._avg.value; + return avg === null ? null : Math.round(avg * 100) / 100; + }; + + // Sleep still needs the raw 400-day window for the per-night reconstruction; + // the widest baseline mean reaches into [365, 395) days, so a 5-day floor + // over that boundary is the smallest safe bound. + const sleepSinceMeasuredAt = new Date(nowMs - 400 * DAY); + + // Cap the type fan-out so a multi-metric account cannot open one DB + // connection per type at once (mirrors the pLimit(4) discipline in + // series-batch/route.ts). + const limit = pLimit(4); + const rows = await Promise.all( - types.map(async (type) => { - // SLEEP_DURATION is stored ONE ROW PER STAGE per night; summarising the - // raw stage rows mislabels a per-stage average as a night total (and - // double-counts a bare ASLEEP aggregate against its granular twin). Route - // the comparison through the per-night dedup reconstruction so the - // current/baseline averages are per-night TIME-ASLEEP totals in minutes. - let dataPoints: DataPoint[]; - if (type === "SLEEP_DURATION") { - const [sleepRows, sleepTz, sleepPriority] = await Promise.all([ - prisma.measurement.findMany({ - where: { - userId, - type, - deletedAt: null, - measuredAt: { gte: sinceMeasuredAt }, - }, - orderBy: { measuredAt: "asc" }, - select: { - measuredAt: true, - value: true, - sleepStage: true, - source: true, - // Writer-level collapse: two HealthKit apps behind one - // source (watch stages vs phone in-bed) must not blend. - deviceType: true, - }, - }), - resolveUserTimezone(userId), - loadUserSourcePriority(userId), - ]); - dataPoints = reconstructSleepNights( - sleepRows as SleepStageRow[], - sleepTz, - sleepPriority, - ) - .filter((n) => n.asleepMinutes > 0) - .map((n) => ({ date: n.measuredAt, value: n.asleepMinutes })); - } else { - const measurements = await prisma.measurement.findMany({ - where: { - userId, - type, - deletedAt: null, - measuredAt: { gte: sinceMeasuredAt }, - }, - orderBy: { measuredAt: "asc" }, - select: { measuredAt: true, value: true }, - }); - dataPoints = measurements.map((m): DataPoint => ({ - date: m.measuredAt, - value: m.value, - })); - } - const summary = summarize(dataPoints); - const baselineAvg = - baseline === "lastMonth" - ? (summary.avg30LastMonth ?? null) - : (summary.avg30LastYear ?? null); - const currentAvg = summary.avg30 ?? null; - const delta = - currentAvg !== null && baselineAvg !== null - ? Math.round((currentAvg - baselineAvg) * 100) / 100 - : null; - const deltaPercent = - delta !== null && baselineAvg !== null && baselineAvg !== 0 - ? Math.round((delta / Math.abs(baselineAvg)) * 100 * 10) / 10 - : null; - return { - type: typeToSnapshotKey[type] ?? type, - currentAvg, - baselineAvg, - delta, - deltaPercent, - unit: typeUnits[type] ?? "", - }; - }), + types.map((type) => + limit(async () => { + let currentAvg: number | null; + let baselineAvg: number | null; + if (type === "SLEEP_DURATION") { + const [sleepRows, sleepTz, sleepPriority] = await Promise.all([ + prisma.measurement.findMany({ + where: { + userId, + type, + deletedAt: null, + measuredAt: { gte: sleepSinceMeasuredAt }, + }, + orderBy: { measuredAt: "asc" }, + select: { + measuredAt: true, + value: true, + sleepStage: true, + source: true, + // Writer-level collapse: two HealthKit apps behind one + // source (watch stages vs phone in-bed) must not blend. + deviceType: true, + }, + }), + resolveUserTimezone(userId), + loadUserSourcePriority(userId), + ]); + const dataPoints: DataPoint[] = reconstructSleepNights( + sleepRows as SleepStageRow[], + sleepTz, + sleepPriority, + ) + .filter((n) => n.asleepMinutes > 0) + .map((n) => ({ date: n.measuredAt, value: n.asleepMinutes })); + const summary = summarize(dataPoints); + currentAvg = summary.avg30 ?? null; + baselineAvg = + baseline === "lastMonth" + ? (summary.avg30LastMonth ?? null) + : (summary.avg30LastYear ?? null); + } else { + [currentAvg, baselineAvg] = await Promise.all([ + avgOverWindow(type, currentWhere), + avgOverWindow(type, baselineWhere), + ]); + } + const delta = + currentAvg !== null && baselineAvg !== null + ? Math.round((currentAvg - baselineAvg) * 100) / 100 + : null; + const deltaPercent = + delta !== null && baselineAvg !== null && baselineAvg !== 0 + ? Math.round((delta / Math.abs(baselineAvg)) * 100 * 10) / 10 + : null; + return { + type: typeToSnapshotKey[type] ?? type, + currentAvg, + baselineAvg, + delta, + deltaPercent, + unit: typeUnits[type] ?? "", + }; + }), + ), ); const metrics = rows.filter( diff --git a/src/lib/jobs/document-thumbnail-backfill.ts b/src/lib/jobs/document-thumbnail-backfill.ts index 0b80f25ef..cc30f88b2 100644 --- a/src/lib/jobs/document-thumbnail-backfill.ts +++ b/src/lib/jobs/document-thumbnail-backfill.ts @@ -18,6 +18,7 @@ * The queue MUST be registered in the maintenance registrar * (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it. */ +import { Prisma } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import { nativeCanvasSupported } from "@/lib/documents/native-canvas-support"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; @@ -112,19 +113,22 @@ export async function enqueueBootTimeThumbnailBackfill(): Promise<{ // would only churn the queue — every generation no-ops behind the gate. if (!nativeCanvasSupported()) return { enqueued: 0 }; - // Distinct users with at least one thumbnailable, not-yet-thumbnailed doc. - const rows = await prisma.inboundDocument.findMany({ - where: { - deletedAt: null, - mimeType: { in: [...THUMBNAILABLE_MIMES] }, - thumbnail: { is: null }, - }, - select: { userId: true }, - distinct: ["userId"], - }); + // v1.28.46 perf (H4) — DB-level SELECT DISTINCT with an anti-join to the + // thumbnail side table, not Prisma `distinct` (which fetches every matching + // row then de-dupes in JS at boot). The migration-0243 partial index over + // `(user_id, mime_type) WHERE deleted_at IS NULL` bounds the document-side + // scan; `document_thumbnails.document_id` is already UNIQUE (the 1:1 + // relation) so the `t.id IS NULL` anti-join is index-driven. + const rows = await prisma.$queryRaw<{ user_id: string }[]>` + SELECT DISTINCT d.user_id AS user_id + FROM inbound_documents d + LEFT JOIN document_thumbnails t ON t.document_id = d.id + WHERE d.deleted_at IS NULL + AND d.mime_type IN (${Prisma.join([...THUMBNAILABLE_MIMES])}) + AND t.id IS NULL`; let enqueued = 0; - for (const { userId } of rows) { + for (const { user_id: userId } of rows) { const payload: ThumbnailBackfillPayload = { userId, enqueuedAt: new Date().toISOString(), diff --git a/src/lib/jobs/lab-biomarker-backfill.ts b/src/lib/jobs/lab-biomarker-backfill.ts index a9cd8d6bb..6c8ca5c7b 100644 --- a/src/lib/jobs/lab-biomarker-backfill.ts +++ b/src/lib/jobs/lab-biomarker-backfill.ts @@ -180,11 +180,14 @@ export async function enqueueBootTimeLabBiomarkerBackfill(): Promise<{ } try { - const rows = await prisma.labResult.findMany({ - where: { deletedAt: null, biomarkerId: null }, - select: { userId: true }, - distinct: ["userId"], - }); + // v1.28.46 perf (H4) — DB-level SELECT DISTINCT, not Prisma `distinct` + // (which fetches every matching row then de-dupes in JS at boot). The + // migration-0243 partial index over `(user_id) WHERE deleted_at IS NULL + // AND biomarker_id IS NULL` makes this an index-only scan that converges + // to empty as rows are linked. + const rows = await prisma.$queryRaw<{ user_id: string }[]>` + SELECT DISTINCT user_id FROM lab_results + WHERE deleted_at IS NULL AND biomarker_id IS NULL`; if (rows.length === 0) { return { enqueued: 0, skipped: 0, error: null }; @@ -192,7 +195,7 @@ export async function enqueueBootTimeLabBiomarkerBackfill(): Promise<{ let enqueued = 0; let skipped = 0; - for (const { userId } of rows) { + for (const { user_id: userId } of rows) { const payload: LabBiomarkerBackfillPayload = { userId, enqueuedAt: new Date().toISOString(), diff --git a/src/lib/jobs/med-notes-encryption-backfill.ts b/src/lib/jobs/med-notes-encryption-backfill.ts index 8e8bdb047..a6cd35e29 100644 --- a/src/lib/jobs/med-notes-encryption-backfill.ts +++ b/src/lib/jobs/med-notes-encryption-backfill.ts @@ -241,31 +241,30 @@ export async function enqueueBootTimeMedNotesEncryptionBackfill(): Promise<{ } try { - const [sideEffectUsers, doseChangeRows, inventoryUsers] = await Promise.all( - [ - prisma.medicationSideEffect.findMany({ - where: { notes: { not: null }, notesEncrypted: null }, - select: { userId: true }, - distinct: ["userId"], - }), - // Dose-changes carry no userId — surface the parent medication's owner - // and dedupe in JS (distinct cannot span a relation field). - prisma.medicationDoseChange.findMany({ - where: { note: { not: null }, noteEncrypted: null }, - select: { medication: { select: { userId: true } } }, - }), - prisma.medicationInventoryItem.findMany({ - where: { notes: { not: null }, notesEncrypted: null }, - select: { userId: true }, - distinct: ["userId"], - }), - ], - ); + // v1.28.46 perf (H4) — DB-level SELECT DISTINCT, not Prisma `distinct` + // (which fetches every matching row then de-dupes in JS at boot). Dose- + // changes carry no userId, so their owner comes through a join on the + // parent medication. The migration-0243 partial indexes match each + // predicate so the scans stay index-only and converge to empty. + const [sideEffectUsers, doseChangeUsers, inventoryUsers] = + await Promise.all([ + prisma.$queryRaw<{ user_id: string }[]>` + SELECT DISTINCT user_id FROM medication_side_effects + WHERE notes IS NOT NULL AND notes_encrypted IS NULL`, + prisma.$queryRaw<{ user_id: string }[]>` + SELECT DISTINCT m.user_id AS user_id + FROM medication_dose_changes dc + JOIN medications m ON m.id = dc.medication_id + WHERE dc.note IS NOT NULL AND dc.note_encrypted IS NULL`, + prisma.$queryRaw<{ user_id: string }[]>` + SELECT DISTINCT user_id FROM medication_inventory_items + WHERE notes IS NOT NULL AND notes_encrypted IS NULL`, + ]); const userIds = new Set(); - for (const { userId } of sideEffectUsers) userIds.add(userId); - for (const { medication } of doseChangeRows) userIds.add(medication.userId); - for (const { userId } of inventoryUsers) userIds.add(userId); + for (const { user_id } of sideEffectUsers) userIds.add(user_id); + for (const { user_id } of doseChangeUsers) userIds.add(user_id); + for (const { user_id } of inventoryUsers) userIds.add(user_id); if (userIds.size === 0) { return { enqueued: 0, skipped: 0, error: null }; diff --git a/src/lib/jobs/note-encryption-backfill.ts b/src/lib/jobs/note-encryption-backfill.ts index 04171bfbe..2b018862f 100644 --- a/src/lib/jobs/note-encryption-backfill.ts +++ b/src/lib/jobs/note-encryption-backfill.ts @@ -192,22 +192,25 @@ export async function enqueueBootTimeNoteEncryptionBackfill( } try { + // v1.28.46 perf (H4) — DB-level SELECT DISTINCT, not Prisma `distinct`. + // Prisma `distinct` fetches EVERY matching row then de-dupes in JS; on the + // densest table (`measurements`) that is a full un-migrated-partition scan + // materialised into the worker at boot. `SELECT DISTINCT user_id` de-dupes + // in Postgres and, with the migration-0243 partial indexes matching each + // predicate, becomes an index-only scan that shrinks to nothing as the + // backfill converges. const [measurementUsers, moodUsers] = await Promise.all([ - prisma.measurement.findMany({ - where: { notes: { not: null }, notesEncrypted: null }, - select: { userId: true }, - distinct: ["userId"], - }), - prisma.moodEntry.findMany({ - where: { note: { not: null }, noteEncrypted: null }, - select: { userId: true }, - distinct: ["userId"], - }), + prisma.$queryRaw<{ user_id: string }[]>` + SELECT DISTINCT user_id FROM measurements + WHERE notes IS NOT NULL AND notes_encrypted IS NULL`, + prisma.$queryRaw<{ user_id: string }[]>` + SELECT DISTINCT user_id FROM mood_entries + WHERE note IS NOT NULL AND note_encrypted IS NULL`, ]); const userIds = new Set(); - for (const { userId } of measurementUsers) userIds.add(userId); - for (const { userId } of moodUsers) userIds.add(userId); + for (const { user_id } of measurementUsers) userIds.add(user_id); + for (const { user_id } of moodUsers) userIds.add(user_id); if (userIds.size === 0) { return { enqueued: 0, skipped: 0, error: null }; diff --git a/tests/integration/boot-discovery-scans.test.ts b/tests/integration/boot-discovery-scans.test.ts new file mode 100644 index 000000000..5e05d967d --- /dev/null +++ b/tests/integration/boot-discovery-scans.test.ts @@ -0,0 +1,182 @@ +/** + * v1.28.46 perf (H4) — the boot-time converging-backfill discovery scans now + * run a DB-level `SELECT DISTINCT user_id` (with a join / anti-join where the + * owner is not a direct column) instead of Prisma's in-memory `distinct`, and + * ride the migration-0243 partial indexes. This suite runs the REAL discovery + * functions against real Postgres to prove the raw SQL is correct (table + + * column names, the medication join, the thumbnail anti-join, the mime IN + * filter) and that each returns EXACTLY the set of users still holding + * un-migrated rows — no more, no fewer. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { setGlobalBoss } from "@/lib/jobs/boss-instance"; +import { nativeCanvasSupported } from "@/lib/documents/native-canvas-support"; +import { enqueueBootTimeNoteEncryptionBackfill } from "@/lib/jobs/note-encryption-backfill"; +import { enqueueBootTimeMedNotesEncryptionBackfill } from "@/lib/jobs/med-notes-encryption-backfill"; +import { enqueueBootTimeLabBiomarkerBackfill } from "@/lib/jobs/lab-biomarker-backfill"; +import { enqueueBootTimeThumbnailBackfill } from "@/lib/jobs/document-thumbnail-backfill"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +/** + * Minimal fake pg-boss that records which (queue, userId) pairs were sent. + * The discovery functions call `boss.send(queue, payload, opts)` and expect a + * truthy job id; capturing the payload's userId lets us assert the discovered + * set without a real queue. + */ +function makeCapturingBoss() { + const sent: { queue: string; userId: string }[] = []; + const boss = { + send: async ( + queue: string, + payload: { userId?: string }, + _opts?: unknown, + ) => { + if (payload.userId) sent.push({ queue, userId: payload.userId }); + return `job-${sent.length}`; + }, + }; + return { boss, sent }; +} + +function usersFor(sent: { userId: string }[]): string[] { + return [...new Set(sent.map((s) => s.userId))].sort(); +} + +async function seedUser(id: string) { + await getPrismaClient().user.create({ + data: { + id, + username: id, + email: `${id}@example.test`, + timezone: "Europe/Berlin", + }, + }); +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); +}); + +describe("boot-discovery scans (real Postgres)", () => { + it("note-encryption discovery returns exactly the users with un-migrated notes", async () => { + const prisma = getPrismaClient(); + await Promise.all([ + seedUser("disc-a"), + seedUser("disc-b"), + seedUser("disc-c"), + ]); + // disc-a: a measurement with a legacy plaintext note (discoverable). + await prisma.measurement.create({ + data: { + userId: "disc-a", + type: "WEIGHT", + value: 80, + unit: "kg", + measuredAt: new Date(), + notes: "legacy plaintext", + }, + }); + // disc-b: a mood entry with a legacy plaintext note (discoverable). + await prisma.moodEntry.create({ + data: { + userId: "disc-b", + date: "2026-01-01", + mood: "GUT", + score: 4, + moodLoggedAt: new Date(), + note: "legacy plaintext mood note", + }, + }); + // disc-c: a measurement with NO note (must NOT be discovered). + await prisma.measurement.create({ + data: { + userId: "disc-c", + type: "WEIGHT", + value: 70, + unit: "kg", + measuredAt: new Date(), + }, + }); + + const { boss, sent } = makeCapturingBoss(); + setGlobalBoss(boss as never); + const res = await enqueueBootTimeNoteEncryptionBackfill(); + expect(res.error).toBeNull(); + expect(usersFor(sent)).toEqual(["disc-a", "disc-b"]); + }); + + it("lab-biomarker discovery returns only users with unlinked live results", async () => { + const prisma = getPrismaClient(); + await Promise.all([seedUser("lab-a"), seedUser("lab-b")]); + // lab-a: unlinked (biomarkerId null), live → discoverable. + await prisma.labResult.create({ + data: { + userId: "lab-a", + analyte: "LDL", + unit: "mg/dL", + takenAt: new Date(), + }, + }); + // lab-b: unlinked but soft-deleted → excluded by `deleted_at IS NULL`. + await prisma.labResult.create({ + data: { + userId: "lab-b", + analyte: "HbA1c", + unit: "%", + takenAt: new Date(), + deletedAt: new Date(), + }, + }); + + const { boss, sent } = makeCapturingBoss(); + setGlobalBoss(boss as never); + const res = await enqueueBootTimeLabBiomarkerBackfill(); + expect(res.error).toBeNull(); + expect(usersFor(sent)).toEqual(["lab-a"]); + }); + + it("med-notes discovery runs its join SQL cleanly (empty tables → no enqueues)", async () => { + // Exercises the three raw statements incl. the dose-changes → medications + // JOIN. Empty tables prove the SQL parses + executes against the real + // schema (a wrong column/join name would throw here). + const { boss, sent } = makeCapturingBoss(); + setGlobalBoss(boss as never); + const res = await enqueueBootTimeMedNotesEncryptionBackfill(); + expect(res.error).toBeNull(); + expect(sent).toEqual([]); + }); + + it("thumbnail discovery applies the mime filter + thumbnail anti-join", async () => { + if (!nativeCanvasSupported()) { + // The gate returns before the query on a non-AVX2 host; the SQL is still + // covered by the other cases' schema, so skip the data assertion here. + return; + } + const prisma = getPrismaClient(); + await Promise.all([seedUser("doc-a"), seedUser("doc-c")]); + // doc-a: a thumbnailable PDF with no thumbnail row → discoverable. + await prisma.inboundDocument.create({ + data: { + userId: "doc-a", + mimeType: "application/pdf", + byteSize: 10, + contentEncrypted: Buffer.from("x"), + }, + }); + // doc-c: a non-thumbnailable text document → excluded by the mime IN filter. + await prisma.inboundDocument.create({ + data: { + userId: "doc-c", + mimeType: "text/plain", + byteSize: 10, + contentEncrypted: Buffer.from("x"), + }, + }); + + const { boss, sent } = makeCapturingBoss(); + setGlobalBoss(boss as never); + await enqueueBootTimeThumbnailBackfill(); + expect(usersFor(sent)).toEqual(["doc-a"]); + }); +});