diff --git a/src/app/api/insights/generate/route.ts b/src/app/api/insights/generate/route.ts index 5e6e35c0c..228fe4849 100644 --- a/src/app/api/insights/generate/route.ts +++ b/src/app/api/insights/generate/route.ts @@ -1,7 +1,10 @@ import { prisma } from "@/lib/db"; import { auditLog } from "@/lib/auth/audit"; import { apiSuccess, apiError, getClientIp } from "@/lib/api-response"; -import { extractFeatures } from "@/lib/insights/features"; +import { + extractFeatures, + FeaturesPayloadTooLargeError, +} from "@/lib/insights/features"; import { detectGlp1Plateau, buildGlp1PlateauPrompt, @@ -266,8 +269,37 @@ export const POST = apiHandler(async (request: NextRequest) => { } const includeRaw = dbUser?.insightsPrivacyMode === "raw"; - const features = await extractFeatures(userId, includeRaw); + // v1.4.36 W3 T1 — `extractFeatures` enforces a 5 MB ceiling on the + // serialised payload. If the raw (bucketed) shape ever blows past + // it (regression watch — the v1.4.35 rawMeasurements shape hit + // 25.9 MB on Marc's account), the helper throws + // `FeaturesPayloadTooLargeError`. We downgrade to the aggregated + // shape rather than 500-ing so the user still gets an insight; the + // annotate event surfaces the regression to ops via the Logflare + // pipeline. + let features: Awaited>; + let payloadDowngraded = false; + let payloadOversizeBytes: number | null = null; + try { + features = await extractFeatures(userId, includeRaw); + } catch (err) { + if (err instanceof FeaturesPayloadTooLargeError) { + payloadDowngraded = true; + payloadOversizeBytes = err.sizeBytes; + features = await extractFeatures(userId, false); + } else { + throw err; + } + } const featuresJson = JSON.stringify(features, null, 2); + if (payloadDowngraded) { + annotate({ + meta: { + insights_features_downgraded: true, + insights_features_oversize_bytes: payloadOversizeBytes, + }, + }); + } // v1.4.16 phase B8 — pick up the user's persisted comparison toggle // and, when active, build a compact prior-period snapshot from the diff --git a/src/lib/insights/__tests__/features.test.ts b/src/lib/insights/__tests__/features.test.ts new file mode 100644 index 000000000..87abb3fb0 --- /dev/null +++ b/src/lib/insights/__tests__/features.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; + +vi.mock("@/lib/db", () => ({ + prisma: { + user: { findUnique: vi.fn() }, + measurement: { findMany: vi.fn() }, + measurementRollup: { findMany: vi.fn() }, + moodEntry: { findMany: vi.fn() }, + medication: { findMany: vi.fn() }, + medicationIntakeEvent: { findMany: vi.fn() }, + }, +})); + +vi.mock("@/lib/medication-category", () => ({ + getMedicationCategories: vi.fn(async () => ({})), +})); + +import { prisma } from "@/lib/db"; +import { + extractFeatures, + FeaturesPayloadTooLargeError, + FEATURES_MAX_BYTES, + type RawFeatures, +} from "../features"; + +const prismaMock = prisma as unknown as { + user: { findUnique: ReturnType }; + measurement: { findMany: ReturnType }; + measurementRollup: { findMany: ReturnType }; + moodEntry: { findMany: ReturnType }; + medication: { findMany: ReturnType }; + medicationIntakeEvent: { findMany: ReturnType }; +}; + +const dayMs = 24 * 60 * 60 * 1000; + +function rollupRow(daysAgo: number, mean: number, count: number) { + return { + bucketStart: new Date(Date.now() - daysAgo * dayMs), + count, + mean, + minValue: mean, + maxValue: mean, + sd: 0, + slope: 0, + r2: 0, + computedAt: new Date(), + }; +} + +beforeEach(() => { + vi.resetAllMocks(); + prismaMock.user.findUnique.mockResolvedValue({ + heightCm: 180, + dateOfBirth: new Date("1980-01-01"), + gender: "MALE", + }); + prismaMock.measurement.findMany.mockResolvedValue([]); + prismaMock.measurementRollup.findMany.mockResolvedValue([]); + prismaMock.moodEntry.findMany.mockResolvedValue([]); + prismaMock.medication.findMany.mockResolvedValue([]); + prismaMock.medicationIntakeEvent.findMany.mockResolvedValue([]); +}); + +describe("extractFeatures — v1.4.36 W3 bucketed payload", () => { + it("does not attach bucketedMeasurements when includeRaw=false", async () => { + const features = await extractFeatures("user-1", false); + const asRecord = features as unknown as Record; + expect(asRecord.bucketedMeasurements).toBeUndefined(); + expect(asRecord.rawMeasurements).toBeUndefined(); + }); + + it("attaches DAY / WEEK / MONTH buckets from measurement_rollups when includeRaw=true", async () => { + // Two WEIGHT DAY buckets in the 0-90d window and one MONTH bucket + // in the 365-1825d window. The reader is called once per + // (type, granularity) combination — return data only for the two + // we care about, empty for the rest. + prismaMock.measurementRollup.findMany.mockImplementation( + async (args: { where: { type: string; granularity: string } }) => { + if (args.where.type === "WEIGHT" && args.where.granularity === "DAY") { + return [rollupRow(10, 82.5, 2), rollupRow(20, 82.7, 1)]; + } + if ( + args.where.type === "WEIGHT" && + args.where.granularity === "MONTH" + ) { + return [rollupRow(400, 85.1, 28)]; + } + return []; + }, + ); + + const features = (await extractFeatures("user-1", true)) as RawFeatures; + + expect(features.bucketedMeasurements).toBeDefined(); + const weightDay = features.bucketedMeasurements.find( + (s) => s.type === "WEIGHT" && s.granularity === "DAY", + ); + expect(weightDay).toBeDefined(); + expect(weightDay?.buckets).toHaveLength(2); + expect(weightDay?.buckets[0]).toMatchObject({ + mean: 82.5, + count: 2, + }); + expect(weightDay?.buckets[0].bucketStart).toMatch( + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/, + ); + + const weightMonth = features.bucketedMeasurements.find( + (s) => s.type === "WEIGHT" && s.granularity === "MONTH", + ); + expect(weightMonth?.buckets).toHaveLength(1); + expect(weightMonth?.buckets[0].count).toBe(28); + }); + + it("drops empty (type, granularity) series so the payload never carries labelled-but-empty arrays", async () => { + prismaMock.measurementRollup.findMany.mockResolvedValue([]); + const features = (await extractFeatures("user-1", true)) as RawFeatures; + expect(features.bucketedMeasurements).toEqual([]); + }); + + it("keeps anthropometrics (heightCm / ageYears / gender) on context", async () => { + const features = await extractFeatures("user-1", false); + expect(features.context.heightCm).toBe(180); + expect(features.context.gender).toBe("MALE"); + expect(features.context.ageYears).toBeGreaterThan(40); + }); + + it("throws FeaturesPayloadTooLargeError when the serialised payload exceeds the 5 MB cap", async () => { + // Fabricate an absurdly long bucket list so the JSON dump crosses + // the ceiling. ~200 KB per series × 28 series ≈ ~5.6 MB. + const giant = new Array(200_000).fill(null).map((_, i) => ({ + bucketStart: new Date(Date.now() - i * dayMs), + count: i, + mean: i, + minValue: i, + maxValue: i, + sd: 0, + slope: 0, + r2: 0, + computedAt: new Date(), + })); + prismaMock.measurementRollup.findMany.mockResolvedValue(giant); + + await expect(extractFeatures("user-1", true)).rejects.toThrow( + FeaturesPayloadTooLargeError, + ); + }); + + it("exposes the size cap as a public constant", () => { + expect(FEATURES_MAX_BYTES).toBe(5 * 1024 * 1024); + }); +}); diff --git a/src/lib/insights/features.ts b/src/lib/insights/features.ts index c802a7972..f7158e475 100644 --- a/src/lib/insights/features.ts +++ b/src/lib/insights/features.ts @@ -15,6 +15,11 @@ import { type CorrelationResult, } from "@/lib/analytics/correlations"; import { getMedicationCategories } from "@/lib/medication-category"; +import { readRollupBuckets } from "@/lib/measurements/rollups"; +import type { + MeasurementType, + RollupGranularity, +} from "@/generated/prisma/client"; interface DataCoverage { count: number; @@ -166,14 +171,100 @@ export interface AggregatedFeatures { }; } -export interface RawFeatures extends AggregatedFeatures { - rawMeasurements: Array<{ - type: string; - value: number; - dayOffset: number; // days ago (anonymized — no exact date) +/** + * One bucketed series. `granularity` decides the bucket width (DAY for + * the trailing 90-day window, WEEK for 90→365 days, MONTH for the + * 365→1825-day deep history). `bucketStart` is an ISO-8601 UTC + * timestamp anchored to the start of the bucket — Postgres + * `date_trunc()` semantics, same as the rollup populator. + * + * Replaces v1.4.35.x `rawMeasurements` (which appended every + * measurement row and blew the 10 MB Codex prompt ceiling on power + * users with multi-year imports — 25.9 MB observed on Marc's account + * with 311 775 rows). Reading from `measurement_rollups` instead caps + * the payload at O(types × buckets) regardless of underlying row + * count. + */ +export interface BucketedSeries { + type: string; + granularity: RollupGranularity; + buckets: Array<{ + bucketStart: string; + mean: number; + count: number; }>; } +export interface RawFeatures extends AggregatedFeatures { + bucketedMeasurements: BucketedSeries[]; +} + +/** + * v1.4.36 W3 T1 — payload size ceiling. The Codex provider rejects + * any prompt-string above 10 MB; the features payload is roughly half + * the eventual prompt (the JSON dump plus the system + comparison + * blocks), so a 5 MB feature payload sits comfortably below the + * ceiling even after locale-prefix expansion. Throwing here lets the + * route handler downgrade to the non-raw payload + annotate so we + * catch the regression instead of blowing the upstream call. + */ +export const FEATURES_MAX_BYTES = 5 * 1024 * 1024; + +export class FeaturesPayloadTooLargeError extends Error { + readonly code = "FEATURES_PAYLOAD_TOO_LARGE"; + readonly sizeBytes: number; + readonly limitBytes: number; + + constructor(sizeBytes: number, limitBytes: number) { + super( + `Features payload too large: ${sizeBytes} bytes exceeds ${limitBytes} byte cap`, + ); + this.name = "FeaturesPayloadTooLargeError"; + this.sizeBytes = sizeBytes; + this.limitBytes = limitBytes; + } +} + +/** + * v1.4.36 W3 T1 — bucket-window definitions for the + * `bucketedMeasurements` payload. Mirrors the rollup populator's + * granularity ladder so the read-side picks up whatever the persistent + * table holds without a recompute round-trip. + * + * The 90 / 365 / 1825-day windows are non-overlapping: each row of + * `measurement_rollups` lives at exactly one granularity, and the + * downstream model reads the union of the three series per type. + * Total volume per type for a Marc-scale power user (5 years of daily + * data): 90 DAY + 39 WEEK + 50 MONTH = 179 buckets × ~40 bytes JSON + * each ≈ 7 KB. Eight metric types lands at ~56 KB — well under the + * 5 MB cap above, vs 25.9 MB for the v1.4.35 rawMeasurements shape. + */ +const BUCKET_WINDOWS: Array<{ + granularity: RollupGranularity; + fromDays: number; + toDays: number; +}> = [ + { granularity: "DAY", fromDays: 0, toDays: 90 }, + { granularity: "WEEK", fromDays: 90, toDays: 365 }, + { granularity: "MONTH", fromDays: 365, toDays: 1825 }, +]; + +/** + * Types the bucketed payload covers. Mirrors the aggregate branches + * above so the model never sees a bucket for a metric whose aggregate + * block was suppressed. New `MeasurementType` enum values flow in by + * adding one row; the rollup populator already covers every type. + */ +const BUCKETED_TYPES: MeasurementType[] = [ + "WEIGHT", + "BLOOD_PRESSURE_SYS", + "BLOOD_PRESSURE_DIA", + "PULSE", + "BODY_FAT", + "SLEEP_DURATION", + "ACTIVITY_STEPS", +]; + function stdDev(values: number[]): number | null { if (values.length < 2) return null; const mean = values.reduce((s, v) => s + v, 0) / values.length; @@ -764,20 +855,75 @@ export async function extractFeatures( }); } - // Raw mode: add anonymized raw data points + // v1.4.36 W3 T1 — "raw" mode no longer dumps every measurement row + // into the prompt. Instead we attach DAY / WEEK / MONTH bucket means + // from `measurement_rollups` so the model gets the same temporal + // shape (granular near-term, coarser deep history) at O(buckets) + // instead of O(rows). The v1.4.35 shape blew past Codex's 10 MB + // string limit on power users (~25.9 MB observed); the bucketed + // shape lands at ~50–500 KB per account. if (includeRaw) { + const bucketedMeasurements = await readBucketedSeries(userId, now); const rawFeatures: RawFeatures = { ...features, - rawMeasurements: measurements.map((m) => ({ - type: m.type, - value: m.value, - dayOffset: Math.round( - (now - m.measuredAt.getTime()) / (24 * 60 * 60 * 1000), - ), - })), + bucketedMeasurements, }; + enforceSizeGuard(rawFeatures); return rawFeatures; } + enforceSizeGuard(features); return features; } + +/** + * Read every BUCKETED_TYPES × BUCKET_WINDOWS combination from the + * persistent rollup table and project to the wire shape. Empty + * (type, granularity) combinations are dropped so the payload never + * carries a labelled-but-empty series. + */ +async function readBucketedSeries( + userId: string, + now: number, +): Promise { + const series: BucketedSeries[] = []; + for (const type of BUCKETED_TYPES) { + for (const window of BUCKET_WINDOWS) { + const from = new Date(now - window.toDays * 24 * 60 * 60 * 1000); + const to = new Date(now - window.fromDays * 24 * 60 * 60 * 1000); + const rows = await readRollupBuckets( + userId, + type, + window.granularity, + from, + to, + ); + if (rows.length === 0) continue; + series.push({ + type, + granularity: window.granularity, + buckets: rows.map((r) => ({ + bucketStart: r.bucketStart.toISOString(), + mean: Math.round(r.mean * 100) / 100, + count: r.count, + })), + }); + } + } + return series; +} + +/** + * Hard ceiling on the serialised payload. Throws + * `FeaturesPayloadTooLargeError` (tagged with `code` so the route + * handler can pattern-match) when the JSON dump crosses + * `FEATURES_MAX_BYTES`. The route handler downgrades to the non-raw + * shape and annotates so the regression is observable; the model + * call still succeeds. + */ +function enforceSizeGuard(payload: AggregatedFeatures | RawFeatures): void { + const sizeBytes = Buffer.byteLength(JSON.stringify(payload), "utf8"); + if (sizeBytes > FEATURES_MAX_BYTES) { + throw new FeaturesPayloadTooLargeError(sizeBytes, FEATURES_MAX_BYTES); + } +}