feat(insights): bucketed measurement payload for raw mode

Replace the per-row rawMeasurements dump with DAY/WEEK/MONTH bucket
means read from measurement_rollups. The legacy shape blew past
Codex's 10 MB string limit on power-user accounts; the bucketed shape
stays O(types × buckets) regardless of row count.

Enforce a 5 MB ceiling on the serialised payload; the generate route
downgrades to the aggregated shape and annotates when the cap trips so
future regressions surface in logs.

src/app/api/insights/generate/route.ts

@@ -1,7 +1,10 @@
 import { prisma } from "@/lib/db";
 import { auditLog } from "@/lib/auth/audit";
 import { apiSuccess, apiError, getClientIp } from "@/lib/api-response";
-import { extractFeatures } from "@/lib/insights/features";
+import {
+  extractFeatures,
+  FeaturesPayloadTooLargeError,
+} from "@/lib/insights/features";
 import {
   detectGlp1Plateau,
   buildGlp1PlateauPrompt,
@@ -266,8 +269,37 @@ export const POST = apiHandler(async (request: NextRequest) => {
   }
 
   const includeRaw = dbUser?.insightsPrivacyMode === "raw";
-  const features = await extractFeatures(userId, includeRaw);
+  // v1.4.36 W3 T1 — `extractFeatures` enforces a 5 MB ceiling on the
+  // serialised payload. If the raw (bucketed) shape ever blows past
+  // it (regression watch — the v1.4.35 rawMeasurements shape hit
+  // 25.9 MB on Marc's account), the helper throws
+  // `FeaturesPayloadTooLargeError`. We downgrade to the aggregated
+  // shape rather than 500-ing so the user still gets an insight; the
+  // annotate event surfaces the regression to ops via the Logflare
+  // pipeline.
+  let features: Awaited<ReturnType<typeof extractFeatures>>;
+  let payloadDowngraded = false;
+  let payloadOversizeBytes: number | null = null;
+  try {
+    features = await extractFeatures(userId, includeRaw);
+  } catch (err) {
+    if (err instanceof FeaturesPayloadTooLargeError) {
+      payloadDowngraded = true;
+      payloadOversizeBytes = err.sizeBytes;
+      features = await extractFeatures(userId, false);
+    } else {
+      throw err;
+    }
+  }
   const featuresJson = JSON.stringify(features, null, 2);
+  if (payloadDowngraded) {
+    annotate({
+      meta: {
+        insights_features_downgraded: true,
+        insights_features_oversize_bytes: payloadOversizeBytes,
+      },
+    });
+  }
 
   // v1.4.16 phase B8 — pick up the user's persisted comparison toggle
   // and, when active, build a compact prior-period snapshot from the

src/lib/insights/__tests__/features.test.ts

@@ -0,0 +1,153 @@
+import { describe, expect, it, vi, beforeEach } from "vitest";
+
+vi.mock("@/lib/db", () => ({
+  prisma: {
+    user: { findUnique: vi.fn() },
+    measurement: { findMany: vi.fn() },
+    measurementRollup: { findMany: vi.fn() },
+    moodEntry: { findMany: vi.fn() },
+    medication: { findMany: vi.fn() },
+    medicationIntakeEvent: { findMany: vi.fn() },
+  },
+}));
+
+vi.mock("@/lib/medication-category", () => ({
+  getMedicationCategories: vi.fn(async () => ({})),
+}));
+
+import { prisma } from "@/lib/db";
+import {
+  extractFeatures,
+  FeaturesPayloadTooLargeError,
+  FEATURES_MAX_BYTES,
+  type RawFeatures,
+} from "../features";
+
+const prismaMock = prisma as unknown as {
+  user: { findUnique: ReturnType<typeof vi.fn> };
+  measurement: { findMany: ReturnType<typeof vi.fn> };
+  measurementRollup: { findMany: ReturnType<typeof vi.fn> };
+  moodEntry: { findMany: ReturnType<typeof vi.fn> };
+  medication: { findMany: ReturnType<typeof vi.fn> };
+  medicationIntakeEvent: { findMany: ReturnType<typeof vi.fn> };
+};
+
+const dayMs = 24 * 60 * 60 * 1000;
+
+function rollupRow(daysAgo: number, mean: number, count: number) {
+  return {
+    bucketStart: new Date(Date.now() - daysAgo * dayMs),
+    count,
+    mean,
+    minValue: mean,
+    maxValue: mean,
+    sd: 0,
+    slope: 0,
+    r2: 0,
+    computedAt: new Date(),
+  };
+}
+
+beforeEach(() => {
+  vi.resetAllMocks();
+  prismaMock.user.findUnique.mockResolvedValue({
+    heightCm: 180,
+    dateOfBirth: new Date("1980-01-01"),
+    gender: "MALE",
+  });
+  prismaMock.measurement.findMany.mockResolvedValue([]);
+  prismaMock.measurementRollup.findMany.mockResolvedValue([]);
+  prismaMock.moodEntry.findMany.mockResolvedValue([]);
+  prismaMock.medication.findMany.mockResolvedValue([]);
+  prismaMock.medicationIntakeEvent.findMany.mockResolvedValue([]);
+});
+
+describe("extractFeatures — v1.4.36 W3 bucketed payload", () => {
+  it("does not attach bucketedMeasurements when includeRaw=false", async () => {
+    const features = await extractFeatures("user-1", false);
+    const asRecord = features as unknown as Record<string, unknown>;
+    expect(asRecord.bucketedMeasurements).toBeUndefined();
+    expect(asRecord.rawMeasurements).toBeUndefined();
+  });
+
+  it("attaches DAY / WEEK / MONTH buckets from measurement_rollups when includeRaw=true", async () => {
+    // Two WEIGHT DAY buckets in the 0-90d window and one MONTH bucket
+    // in the 365-1825d window. The reader is called once per
+    // (type, granularity) combination — return data only for the two
+    // we care about, empty for the rest.
+    prismaMock.measurementRollup.findMany.mockImplementation(
+      async (args: { where: { type: string; granularity: string } }) => {
+        if (args.where.type === "WEIGHT" && args.where.granularity === "DAY") {
+          return [rollupRow(10, 82.5, 2), rollupRow(20, 82.7, 1)];
+        }
+        if (
+          args.where.type === "WEIGHT" &&
+          args.where.granularity === "MONTH"
+        ) {
+          return [rollupRow(400, 85.1, 28)];
+        }
+        return [];
+      },
+    );
+
+    const features = (await extractFeatures("user-1", true)) as RawFeatures;
+
+    expect(features.bucketedMeasurements).toBeDefined();
+    const weightDay = features.bucketedMeasurements.find(
+      (s) => s.type === "WEIGHT" && s.granularity === "DAY",
+    );
+    expect(weightDay).toBeDefined();
+    expect(weightDay?.buckets).toHaveLength(2);
+    expect(weightDay?.buckets[0]).toMatchObject({
+      mean: 82.5,
+      count: 2,
+    });
+    expect(weightDay?.buckets[0].bucketStart).toMatch(
+      /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/,
+    );
+
+    const weightMonth = features.bucketedMeasurements.find(
+      (s) => s.type === "WEIGHT" && s.granularity === "MONTH",
+    );
+    expect(weightMonth?.buckets).toHaveLength(1);
+    expect(weightMonth?.buckets[0].count).toBe(28);
+  });
+
+  it("drops empty (type, granularity) series so the payload never carries labelled-but-empty arrays", async () => {
+    prismaMock.measurementRollup.findMany.mockResolvedValue([]);
+    const features = (await extractFeatures("user-1", true)) as RawFeatures;
+    expect(features.bucketedMeasurements).toEqual([]);
+  });
+
+  it("keeps anthropometrics (heightCm / ageYears / gender) on context", async () => {
+    const features = await extractFeatures("user-1", false);
+    expect(features.context.heightCm).toBe(180);
+    expect(features.context.gender).toBe("MALE");
+    expect(features.context.ageYears).toBeGreaterThan(40);
+  });
+
+  it("throws FeaturesPayloadTooLargeError when the serialised payload exceeds the 5 MB cap", async () => {
+    // Fabricate an absurdly long bucket list so the JSON dump crosses
+    // the ceiling. ~200 KB per series × 28 series ≈ ~5.6 MB.
+    const giant = new Array(200_000).fill(null).map((_, i) => ({
+      bucketStart: new Date(Date.now() - i * dayMs),
+      count: i,
+      mean: i,
+      minValue: i,
+      maxValue: i,
+      sd: 0,
+      slope: 0,
+      r2: 0,
+      computedAt: new Date(),
+    }));
+    prismaMock.measurementRollup.findMany.mockResolvedValue(giant);
+
+    await expect(extractFeatures("user-1", true)).rejects.toThrow(
+      FeaturesPayloadTooLargeError,
+    );
+  });
+
+  it("exposes the size cap as a public constant", () => {
+    expect(FEATURES_MAX_BYTES).toBe(5 * 1024 * 1024);
+  });
+});

src/lib/insights/features.ts

@@ -15,6 +15,11 @@ import {
   type CorrelationResult,
 } from "@/lib/analytics/correlations";
 import { getMedicationCategories } from "@/lib/medication-category";
+import { readRollupBuckets } from "@/lib/measurements/rollups";
+import type {
+  MeasurementType,
+  RollupGranularity,
+} from "@/generated/prisma/client";
 
 interface DataCoverage {
   count: number;
@@ -166,14 +171,100 @@ export interface AggregatedFeatures {
   };
 }
 
-export interface RawFeatures extends AggregatedFeatures {
-  rawMeasurements: Array<{
-    type: string;
-    value: number;
-    dayOffset: number; // days ago (anonymized — no exact date)
+/**
+ * One bucketed series. `granularity` decides the bucket width (DAY for
+ * the trailing 90-day window, WEEK for 90→365 days, MONTH for the
+ * 365→1825-day deep history). `bucketStart` is an ISO-8601 UTC
+ * timestamp anchored to the start of the bucket — Postgres
+ * `date_trunc()` semantics, same as the rollup populator.
+ *
+ * Replaces v1.4.35.x `rawMeasurements` (which appended every
+ * measurement row and blew the 10 MB Codex prompt ceiling on power
+ * users with multi-year imports — 25.9 MB observed on Marc's account
+ * with 311 775 rows). Reading from `measurement_rollups` instead caps
+ * the payload at O(types × buckets) regardless of underlying row
+ * count.
+ */
+export interface BucketedSeries {
+  type: string;
+  granularity: RollupGranularity;
+  buckets: Array<{
+    bucketStart: string;
+    mean: number;
+    count: number;
   }>;
 }
 
+export interface RawFeatures extends AggregatedFeatures {
+  bucketedMeasurements: BucketedSeries[];
+}
+
+/**
+ * v1.4.36 W3 T1 — payload size ceiling. The Codex provider rejects
+ * any prompt-string above 10 MB; the features payload is roughly half
+ * the eventual prompt (the JSON dump plus the system + comparison
+ * blocks), so a 5 MB feature payload sits comfortably below the
+ * ceiling even after locale-prefix expansion. Throwing here lets the
+ * route handler downgrade to the non-raw payload + annotate so we
+ * catch the regression instead of blowing the upstream call.
+ */
+export const FEATURES_MAX_BYTES = 5 * 1024 * 1024;
+
+export class FeaturesPayloadTooLargeError extends Error {
+  readonly code = "FEATURES_PAYLOAD_TOO_LARGE";
+  readonly sizeBytes: number;
+  readonly limitBytes: number;
+
+  constructor(sizeBytes: number, limitBytes: number) {
+    super(
+      `Features payload too large: ${sizeBytes} bytes exceeds ${limitBytes} byte cap`,
+    );
+    this.name = "FeaturesPayloadTooLargeError";
+    this.sizeBytes = sizeBytes;
+    this.limitBytes = limitBytes;
+  }
+}
+
+/**
+ * v1.4.36 W3 T1 — bucket-window definitions for the
+ * `bucketedMeasurements` payload. Mirrors the rollup populator's
+ * granularity ladder so the read-side picks up whatever the persistent
+ * table holds without a recompute round-trip.
+ *
+ * The 90 / 365 / 1825-day windows are non-overlapping: each row of
+ * `measurement_rollups` lives at exactly one granularity, and the
+ * downstream model reads the union of the three series per type.
+ * Total volume per type for a Marc-scale power user (5 years of daily
+ * data): 90 DAY + 39 WEEK + 50 MONTH = 179 buckets × ~40 bytes JSON
+ * each ≈ 7 KB. Eight metric types lands at ~56 KB — well under the
+ * 5 MB cap above, vs 25.9 MB for the v1.4.35 rawMeasurements shape.
+ */
+const BUCKET_WINDOWS: Array<{
+  granularity: RollupGranularity;
+  fromDays: number;
+  toDays: number;
+}> = [
+  { granularity: "DAY", fromDays: 0, toDays: 90 },
+  { granularity: "WEEK", fromDays: 90, toDays: 365 },
+  { granularity: "MONTH", fromDays: 365, toDays: 1825 },
+];
+
+/**
+ * Types the bucketed payload covers. Mirrors the aggregate branches
+ * above so the model never sees a bucket for a metric whose aggregate
+ * block was suppressed. New `MeasurementType` enum values flow in by
+ * adding one row; the rollup populator already covers every type.
+ */
+const BUCKETED_TYPES: MeasurementType[] = [
+  "WEIGHT",
+  "BLOOD_PRESSURE_SYS",
+  "BLOOD_PRESSURE_DIA",
+  "PULSE",
+  "BODY_FAT",
+  "SLEEP_DURATION",
+  "ACTIVITY_STEPS",
+];
+
 function stdDev(values: number[]): number | null {
   if (values.length < 2) return null;
   const mean = values.reduce((s, v) => s + v, 0) / values.length;
@@ -764,20 +855,75 @@ export async function extractFeatures(
     });
   }
 
-  // Raw mode: add anonymized raw data points
+  // v1.4.36 W3 T1 — "raw" mode no longer dumps every measurement row
+  // into the prompt. Instead we attach DAY / WEEK / MONTH bucket means
+  // from `measurement_rollups` so the model gets the same temporal
+  // shape (granular near-term, coarser deep history) at O(buckets)
+  // instead of O(rows). The v1.4.35 shape blew past Codex's 10 MB
+  // string limit on power users (~25.9 MB observed); the bucketed
+  // shape lands at ~50–500 KB per account.
   if (includeRaw) {
+    const bucketedMeasurements = await readBucketedSeries(userId, now);
     const rawFeatures: RawFeatures = {
       ...features,
-      rawMeasurements: measurements.map((m) => ({
-        type: m.type,
-        value: m.value,
-        dayOffset: Math.round(
-          (now - m.measuredAt.getTime()) / (24 * 60 * 60 * 1000),
-        ),
-      })),
+      bucketedMeasurements,
     };
+    enforceSizeGuard(rawFeatures);
     return rawFeatures;
   }
 
+  enforceSizeGuard(features);
   return features;
 }
+
+/**
+ * Read every BUCKETED_TYPES × BUCKET_WINDOWS combination from the
+ * persistent rollup table and project to the wire shape. Empty
+ * (type, granularity) combinations are dropped so the payload never
+ * carries a labelled-but-empty series.
+ */
+async function readBucketedSeries(
+  userId: string,
+  now: number,
+): Promise<BucketedSeries[]> {
+  const series: BucketedSeries[] = [];
+  for (const type of BUCKETED_TYPES) {
+    for (const window of BUCKET_WINDOWS) {
+      const from = new Date(now - window.toDays * 24 * 60 * 60 * 1000);
+      const to = new Date(now - window.fromDays * 24 * 60 * 60 * 1000);
+      const rows = await readRollupBuckets(
+        userId,
+        type,
+        window.granularity,
+        from,
+        to,
+      );
+      if (rows.length === 0) continue;
+      series.push({
+        type,
+        granularity: window.granularity,
+        buckets: rows.map((r) => ({
+          bucketStart: r.bucketStart.toISOString(),
+          mean: Math.round(r.mean * 100) / 100,
+          count: r.count,
+        })),
+      });
+    }
+  }
+  return series;
+}
+
+/**
+ * Hard ceiling on the serialised payload. Throws
+ * `FeaturesPayloadTooLargeError` (tagged with `code` so the route
+ * handler can pattern-match) when the JSON dump crosses
+ * `FEATURES_MAX_BYTES`. The route handler downgrades to the non-raw
+ * shape and annotates so the regression is observable; the model
+ * call still succeeds.
+ */
+function enforceSizeGuard(payload: AggregatedFeatures | RawFeatures): void {
+  const sizeBytes = Buffer.byteLength(JSON.stringify(payload), "utf8");
+  if (sizeBytes > FEATURES_MAX_BYTES) {
+    throw new FeaturesPayloadTooLargeError(sizeBytes, FEATURES_MAX_BYTES);
+  }
+}