v1.4.36 — perf, charts, AI payload trim, UX punch

Heavy-aggregate skip on the rollup-fresh + fully-covered path,
daily chart fetches opt into rollup buckets via source=rollup,
Insights extractFeatures swap drops the prompt payload from
double-digit MB to the low hundreds of kB on power-user accounts,
page-level isLoading gate replaced with per-section early-skeleton
paint, trends-row chart slot pinned for CLS, IntakeHistoryListV2
restored on the medication detail page, About card folded into the
Admin Console with an auto-checking update badge, cumulative metric
tiles read the day's running sum, IP-whois fallback surfaces city
+ country when the ASN lookup misses. Migration 0068 adds an
additive users.insights_exclude_metrics column. Unit suite 4354
passing; typecheck + lint clean.
This commit is contained in:
Marc-André Bombeck committed 2026-05-17 13:22:51 +02:00
1 parent 5a3ca330ac
commit 0e2a99f8e7
70 files changed
+4982 -810

No files matched your search

@@ -266,8 +266,12 @@ describe("GET /api/analytics", () => {
// contract the dashboard tile strip reads.
it("returns the slim summaries slice when ?slice=summaries is set", async () => {
const fiveDaysAgo = new Date(Date.now() - 5 * 24 * 60 * 60 * 1000);
// Aggregate pass + latest pass. The mock returns one row per pass.
// v1.4.36 — slim slice now opens with a cheap `SELECT COUNT(*) FROM
// measurement_rollups` probe. Force `n: 0` so the route takes the
// legacy heavy-aggregate fallback path that this test's fixtures
// expect (aggregate + latest, two `$queryRaw` passes).
vi.mocked(prisma.$queryRaw)
.mockResolvedValueOnce([{ n: BigInt(0) }] as never)
.mockResolvedValueOnce([
{
type: "WEIGHT",
+61 -14
View File
@@ -29,10 +29,38 @@ import type {
} from "@/generated/prisma/client";
import { measurementTypeEnum } from "@/lib/validations/measurement";
import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority";
import type { SourcePriorityMetricKey } from "@/lib/validations/source-priority";
import { ensureUserRollupsFresh } from "@/lib/measurements/rollups";
import {
isCumulativeDaySumType,
pickCumulativeDaySum,
} from "@/lib/measurements/cumulative-day-sum";
export const dynamic = "force-dynamic";
/**
* v1.4.36 W4c — cumulative MeasurementType → SourcePriorityMetricKey
* for `pickCanonicalSourceRows`. Returns `null` for types without a
* dedicated priority ladder (e.g. TIME_IN_DAYLIGHT), which fall
* through the picker's "no ladder" pass-through branch.
*/
function cumulativeMetricKey(
type: MeasurementType,
): SourcePriorityMetricKey | null {
switch (type) {
case "ACTIVITY_STEPS":
return "steps";
case "ACTIVE_ENERGY_BURNED":
return "activeEnergy";
case "WALKING_RUNNING_DISTANCE":
return "walkingRunningDistance";
case "FLIGHTS_CLIMBED":
return "flightsClimbed";
default:
return null;
}
}
/**
* v1.4.33 C1 — pull `?slice=…` from either a NextRequest (the
* route's production wrapper) or a plain `Request` (the integration
@@ -208,21 +236,40 @@ async function buildAnalyticsResponse(user: AuthedUser) {
sourcePriorityJson,
(d) => userDayKey(d, userTz),
).canonicalRows;
const byDay = new Map<string, { total: number; date: Date }>();
for (const m of sleepRows) {
const key = userDayKey(m.measuredAt, userTz);
const slot = byDay.get(key) ?? {
total: 0,
date: m.measuredAt,
};
slot.total += m.value;
if (m.measuredAt > slot.date) slot.date = m.measuredAt;
byDay.set(key, slot);
}
datapoints = Array.from(byDay.values()).map(
(s): DataPoint => ({ date: s.date, value: s.total }),
datapoints = pickCumulativeDaySum(sleepRows, (d) =>
userDayKey(d, userTz),
);
} else if (isCumulativeDaySumType(type)) {
// v1.4.36 W4c — broaden the SLEEP_DURATION day-bucket-sum
// pattern to every cumulative metric (steps, active energy,
// walking + running distance, flights climbed, time in
// daylight). Apple Health writes minute-level slices for
// these series; the dashboard tile was reading the
// *latest slice* via `summary.latest` instead of the day's
// total, which Marc reported as "Steps tile shows
// last-measurement-not-day-sum". After the per-day sum
// collapse, `summary.latest` is the most-recent day's
// total — exactly what the tile expects.
//
// Map cumulative MeasurementType → SourcePriorityMetricKey
// for the canonical-row picker. TIME_IN_DAYLIGHT has no
// dedicated priority ladder (no clinical-grade competitor
// to Apple Health for daylight minutes today) so it falls
// through `pickCanonicalSourceRows`'s "no ladder"
// pass-through branch — we ALWAYS bucket-and-sum it
// regardless of source.
const metricKey = cumulativeMetricKey(type);
const canonicalRows = metricKey
? pickCanonicalSourceRows(
measurements,
metricKey,
sourcePriorityJson,
(d) => userDayKey(d, userTz),
).canonicalRows
: measurements;
datapoints = pickCumulativeDaySum(canonicalRows, (d) =>
userDayKey(d, userTz),
);
datapoints.sort((a, b) => a.date.getTime() - b.date.getTime());
} else {
datapoints = measurements.map(
(m): DataPoint => ({
+10 -1
View File
@@ -57,9 +57,18 @@ export const PUT = apiHandler(async (req: Request) => {
// Persist the canonical defaulted form so the column shape stays
// stable regardless of which subset of keys the caller supplied.
//
// v1.4.36 W3 T3 — mirror the excludeMetrics list onto the dedicated
// `insightsExcludeMetrics` column so the Insights generator and
// Coach snapshot share a single source of truth. Keeps the user's
// privacy contract symmetric across the two surfaces without
// requiring a separate Insights settings sheet.
await prisma.user.update({
where: { id: user.id },
data: { coachPrefsJson: parsed.data },
data: {
coachPrefsJson: parsed.data,
insightsExcludeMetrics: parsed.data.excludeMetrics,
},
});
annotate({
@@ -80,9 +80,16 @@ vi.mock("@/lib/logging/context", () => ({
annotate: vi.fn(),
}));
vi.mock("@/lib/insights/features", () => ({
extractFeatures: vi.fn(async () => ({ stub: true })),
}));
vi.mock("@/lib/insights/features", async () => {
const actual =
await vi.importActual<typeof import("@/lib/insights/features")>(
"@/lib/insights/features",
);
return {
...actual,
extractFeatures: vi.fn(async () => ({ stub: true })),
};
});
vi.mock("@/lib/insights/prompt", () => ({
buildUserPrompt: vi.fn(() => "user"),
@@ -97,6 +104,11 @@ import { resolveProvider } from "@/lib/ai/provider";
import { checkRateLimit } from "@/lib/rate-limit";
import { prisma } from "@/lib/db";
import { clearLastWorkingProviderCache } from "@/lib/ai/provider-runner";
import {
extractFeatures,
FeaturesPayloadTooLargeError,
} from "@/lib/insights/features";
import { annotate } from "@/lib/logging/context";
beforeEach(() => {
vi.clearAllMocks();
@@ -344,3 +356,51 @@ describe("POST /api/insights/generate — per-status cache eviction (A7.2)", ()
expect(prisma.auditLog.deleteMany).not.toHaveBeenCalled();
});
});
// v1.4.36 QA H1 — when both the raw-mode extraction AND the aggregated
// retry cross the 5 MB ceiling, the route used to throw uncaught → 500.
// It now wraps the retry, downgrades further via the exclude filter,
// and on a third failure returns 422 with an annotate event for ops.
describe("POST /api/insights/generate — payload-size hard downgrade (H1)", () => {
it("returns 422 with an annotate event when every fallback shape is oversized", async () => {
makeWorkingProvider();
vi.mocked(prisma.user.findUnique).mockResolvedValueOnce({
// Raw mode so the route's first extractFeatures call is the
// wide shape; the retry asks for the aggregated shape.
insightsPrivacyMode: "raw",
insightsCachedAt: null,
insightsCachedText: null,
insightsExcludeMetrics: [],
locale: "en",
} as never);
// Each call mimics features.ts blowing past the ceiling. Three
// shots: raw, aggregated retry, hard-downgrade aggregated.
vi.mocked(extractFeatures).mockReset();
vi.mocked(extractFeatures)
.mockRejectedValueOnce(
new FeaturesPayloadTooLargeError(6_000_000, 5_242_880),
)
.mockRejectedValueOnce(
new FeaturesPayloadTooLargeError(5_500_000, 5_242_880),
)
.mockRejectedValueOnce(
new FeaturesPayloadTooLargeError(5_300_000, 5_242_880),
);
const res = await POST(jsonRequest({ force: true }) as never);
expect(res.status).toBe(422);
const body = (await res.json()) as ApiErrorEnvelope;
expect(body.error).toMatch(/payload size/i);
// The annotate event fires with insights_payload_too_large so ops
// can spot the regression in the wide-event pipeline.
const calls = vi.mocked(annotate).mock.calls;
const annotated = calls.find(
(call) =>
(call[0] as { meta?: Record<string, unknown> })?.meta
?.insights_payload_too_large === true,
);
expect(annotated, "annotate event with insights_payload_too_large").toBeTruthy();
});
});
+106 -3
View File
@@ -1,7 +1,12 @@
import { prisma } from "@/lib/db";
import { auditLog } from "@/lib/auth/audit";
import { apiSuccess, apiError, getClientIp } from "@/lib/api-response";
import { extractFeatures } from "@/lib/insights/features";
import {
extractFeatures,
FeaturesPayloadTooLargeError,
} from "@/lib/insights/features";
import { applyInsightsExcludeFilter } from "@/lib/insights/exclude-filter";
import { compactSections } from "@/lib/insights/prompt-compact";
import {
detectGlp1Plateau,
buildGlp1PlateauPrompt,
@@ -188,6 +193,10 @@ export const POST = apiHandler(async (request: NextRequest) => {
insightsPrivacyMode: true,
insightsCachedAt: true,
insightsCachedText: true,
// v1.4.36 W3 T3 — per-user opt-out list mirroring the Coach
// settings. Filtered off `features` before serialisation so the
// LLM never sees the excluded blocks.
insightsExcludeMetrics: true,
locale: true,
},
});
@@ -266,8 +275,102 @@ export const POST = apiHandler(async (request: NextRequest) => {
}
const includeRaw = dbUser?.insightsPrivacyMode === "raw";
const features = await extractFeatures(userId, includeRaw);
const featuresJson = JSON.stringify(features, null, 2);
// v1.4.36 W3 T1 — `extractFeatures` enforces a 5 MB ceiling on the
// serialised payload. If the raw (bucketed) shape ever blows past
// it (regression watch — the v1.4.35 rawMeasurements shape hit
// 25.9 MB on Marc's account), the helper throws
// `FeaturesPayloadTooLargeError`. We downgrade to the aggregated
// shape rather than 500-ing so the user still gets an insight; the
// annotate event surfaces the regression to ops via the Logflare
// pipeline.
let features: Awaited<ReturnType<typeof extractFeatures>>;
let payloadDowngraded = false;
let payloadOversizeBytes: number | null = null;
// v1.4.36 QA H1 — extra fallback when even the aggregated shape
// crosses the 5 MB ceiling (very large medication history + multi-
// year context). Strips anthropometrics + medications + every other
// exclude-token-mapped block and tries one more time. On a third
// failure we return 422 with an annotate event for ops visibility
// rather than 500-ing out of an uncaught throw.
const MAX_DOWNGRADE_TOKENS: ReadonlyArray<string> = [
"anthropometrics",
"medications",
"compliance",
"sleep",
"steps",
"hrv",
"resting_hr",
];
let payloadHardDowngraded = false;
try {
features = await extractFeatures(userId, includeRaw);
} catch (err) {
if (err instanceof FeaturesPayloadTooLargeError) {
payloadDowngraded = true;
payloadOversizeBytes = err.sizeBytes;
try {
features = await extractFeatures(userId, false);
} catch (retryErr) {
if (retryErr instanceof FeaturesPayloadTooLargeError) {
// Aggregated shape ALSO crossed the ceiling. Drop optional
// context blocks (anthropometrics, medications, sleep,
// steps, hrv, resting_hr) via the existing exclude filter
// and try once more.
payloadHardDowngraded = true;
payloadOversizeBytes = retryErr.sizeBytes;
try {
const aggregated = await extractFeatures(userId, false);
features = applyInsightsExcludeFilter(
aggregated,
MAX_DOWNGRADE_TOKENS,
);
} catch (finalErr) {
// Even the aggregated read itself blew up. Annotate and
// return 422 rather than let the uncaught throw 500 out.
annotate({
meta: {
insights_payload_too_large: true,
insights_payload_oversize_bytes:
finalErr instanceof FeaturesPayloadTooLargeError
? finalErr.sizeBytes
: payloadOversizeBytes,
},
});
return apiError(
"Your data exceeds the AI payload size limit. Reduce the data window or exclude optional blocks in Settings > AI.",
422,
);
}
} else {
throw retryErr;
}
}
} else {
throw err;
}
}
// v1.4.36 W3 T3 — apply the user's exclude-metrics opt-out list
// BEFORE serialisation so the model never sees the dropped blocks.
const excludeList = dbUser?.insightsExcludeMetrics ?? [];
features = applyInsightsExcludeFilter(features, excludeList);
// v1.4.36 W3 T4 — drop zero-row blocks so the prompt never carries
// labelled-empty sections (`"sleep": []`, `"medications": []`,
// etc.). Prevents the model from narrating "there are no
// medications in your data" when the user explicitly excluded the
// block.
const compactFeatures = compactSections(
features as unknown as Record<string, unknown>,
);
const featuresJson = JSON.stringify(compactFeatures, null, 2);
if (payloadDowngraded) {
annotate({
meta: {
insights_features_downgraded: true,
insights_features_oversize_bytes: payloadOversizeBytes,
insights_features_hard_downgraded: payloadHardDowngraded,
},
});
}
// v1.4.16 phase B8 — pick up the user's persisted comparison toggle
// and, when active, build a compact prior-period snapshot from the
+28 -3
View File
@@ -25,6 +25,8 @@ import type {
} from "@/generated/prisma/client";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { annotate } from "@/lib/logging/context";
import { cached, caches, type ServerCache } from "@/lib/cache/server-cache";
import { NO_STORE_BUT_BFCACHE } from "@/lib/http/cache-headers";
import {
getEffectiveRange,
type ThresholdOverridesJson,
@@ -105,9 +107,32 @@ interface TargetPageSummary {
streakHighlight: { metric: string; days: number } | null;
}
/**
* v1.4.36 W1 — `/api/insights/targets` body, lifted out of the route
* handler so `cached()` can wrap it. The handler attaches the
* bfcache-friendly Cache-Control header afterward. 60 s TTL keyed on
* the user id matches the analytics cache so multiple Insights mounts
* inside a minute all hit a warm cache. Invalidated alongside the
* analytics bucket on measurement / mood / medication writes.
*/
type AuthedUser = Awaited<ReturnType<typeof requireAuth>>["user"];
export const GET = apiHandler(async () => {
const { user } = await requireAuth();
const body = await cached(
caches.insightsTargets as ServerCache<
Awaited<ReturnType<typeof buildTargetsResponse>>
>,
user.id,
() => buildTargetsResponse(user),
annotate,
);
const response = apiSuccess(body);
response.headers.set("Cache-Control", NO_STORE_BUT_BFCACHE);
return response;
});
async function buildTargetsResponse(user: AuthedUser) {
const userId = user.id;
// v1.4.25 W7b — every per-day bucket key in this route resolves
@@ -1179,7 +1204,7 @@ export const GET = apiHandler(async () => {
},
});
return apiSuccess({
return {
targets,
pageSummary,
// Extra diastolic data for BP display
@@ -1194,5 +1219,5 @@ export const GET = apiHandler(async () => {
gender,
glucoseUnit,
},
});
});
};
}
@@ -20,6 +20,9 @@ vi.mock("@/lib/db", () => ({
findMany: vi.fn(),
count: vi.fn(),
},
measurementRollup: {
findMany: vi.fn(),
},
$queryRaw: vi.fn(),
},
}));
@@ -193,6 +196,71 @@ describe("GET /api/measurements — all-time semantics (SD-H1)", () => {
expect(months.size).toBe(24);
});
it("source=rollup + aggregate=daily reads measurement_rollups without firing the heavy date_trunc query", async () => {
// v1.4.36 W1 — Insights trends row routes the three daily chart
// fetches through the persistent DAY buckets via `source=rollup`.
// The heavy `date_trunc` $queryRaw must NOT fire on the happy
// path where the rollup has rows for the requested window.
const buckets = Array.from({ length: 30 }, (_, i) => ({
type: "WEIGHT",
bucketStart: new Date(Date.UTC(2026, 3, 16 + i, 0, 0, 0)),
mean: 81 + i * 0.05,
count: 5,
}));
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue(
buckets as never,
);
const res = await GET(
getRequest(
"type=WEIGHT&from=2026-04-15T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
),
);
expect(res.status).toBe(200);
const json = (await res.json()) as {
data: {
measurements: Array<{ type: string; value: number; count: number }>;
meta: { aggregate?: string };
};
};
expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(1);
expect(prisma.$queryRaw).not.toHaveBeenCalled();
expect(json.data.measurements.length).toBe(30);
// First bucket reflects the rollup mean — round-trip-stable.
expect(json.data.measurements[0].type).toBe("WEIGHT");
expect(json.data.measurements[0].value).toBeCloseTo(81, 5);
expect(json.data.measurements[0].count).toBe(5);
expect(json.data.meta.aggregate).toBe("daily");
});
it("falls back to live date_trunc when source=rollup returns zero buckets", async () => {
// Empty rollup ⇒ heavy aggregate runs so brand-new accounts still
// see a correct chart on their first render.
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue(
[] as never,
);
const liveBuckets = Array.from({ length: 7 }, (_, i) => ({
type: "WEIGHT",
bucket_start: new Date(Date.UTC(2026, 4, 8 + i, 0, 0, 0)),
avg: 81 + i * 0.05,
cnt: 2,
}));
vi.mocked(prisma.$queryRaw).mockResolvedValue(liveBuckets as never);
const res = await GET(
getRequest(
"type=WEIGHT&from=2026-05-08T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
),
);
expect(res.status).toBe(200);
expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(1);
expect(prisma.$queryRaw).toHaveBeenCalledTimes(1);
const json = (await res.json()) as {
data: { measurements: Array<unknown> };
};
expect(json.data.measurements.length).toBe(7);
});
it("returns a weekly series when the all-time window is under two years", async () => {
// Three months past one year → falls in the weekly grain band.
const weeks = Array.from({ length: 60 }, (_, i) => ({
+86 -1
View File
@@ -42,7 +42,7 @@ export const GET = apiHandler(async (request: NextRequest) => {
return apiError(parsed.error.issues[0].message, 422);
}
const { type, from, to, limit, offset, sortBy, sortDir, aggregate } =
const { type, from, to, limit, offset, sortBy, sortDir, aggregate, source } =
parsed.data;
const where = {
@@ -77,6 +77,91 @@ export const GET = apiHandler(async (request: NextRequest) => {
// The route's response is bounded by the `BUCKET_CAP` ceiling per
// grain (monthly: 24, weekly: 105, daily: 365) so a multi-decade
// account never paints an unbounded series.
// v1.4.36 W1 — rollup-table read for daily aggregates.
//
// The Insights trends row fires three parallel
// `GET /api/measurements?type=…&aggregate=daily&limit=5000` requests
// (BP_SYS / BP_DIA / WEIGHT). The legacy `date_trunc('day', …)`
// path scanned the measurements table each time — a full year window
// on Marc's account hit 3 × ~3 s on the v1.4.35 HAR. Reading from
// the persistent `measurement_rollups` DAY buckets drops each call
// to a small indexed read against the ~5 k-row rollup table.
//
// Gating:
// - `source=rollup` is an explicit client opt-in (`@/lib/validations/measurement`
// constrains the enum to `["rollup"]`).
// - We require `aggregate=daily` because the rollup populator only
// keeps DAY buckets on the synchronous write hook; the WEEK /
// MONTH paths still depend on the async pg-boss recomputes which
// may lag behind a recent write.
// - `type` is mandatory so the response shape mirrors the per-type
// ask the chart code makes.
// - When the rollup window returns zero rows we fall through to
// the live `date_trunc` path so a brand-new account whose
// populator hasn't caught up still sees a correct chart on its
// first render.
if (
source === "rollup" &&
aggregate === "daily" &&
type &&
from &&
to
) {
const cap = Math.min(limit, BUCKET_CAP.daily);
const rollupRows = await prisma.measurementRollup.findMany({
where: {
userId: user.id,
type: type as MeasurementType,
granularity: "DAY",
bucketStart: { gte: from, lte: to },
},
orderBy: { bucketStart: "asc" },
take: cap,
select: {
type: true,
bucketStart: true,
mean: true,
count: true,
},
});
if (rollupRows.length > 0) {
// Cumulative metrics (steps, active energy, distance, flights,
// daylight) store `mean` per bucket but the chart needs the
// daily SUM. Reconstruct as `mean * count` — algebraically
// equivalent to SUM(value) for a single-source day because
// AVG = SUM / COUNT.
const useSum =
type != null && CUMULATIVE_HK_TYPES.has(type as MeasurementType);
const measurements = rollupRows.map((r) => ({
type: r.type,
value: useSum ? r.mean * r.count : r.mean,
measuredAt: r.bucketStart.toISOString(),
count: r.count,
}));
annotate({
action: { name: "measurement.list" },
meta: {
total: measurements.length,
type,
aggregate: "daily",
source: "rollup",
},
});
return apiSuccess({
measurements,
meta: {
total: measurements.length,
limit: cap,
offset: 0,
aggregate: "daily",
},
});
}
// Fall through to the live aggregate path when the rollup is
// empty for the requested window — covers brand-new accounts +
// the race between a recent write hook and the read.
}
if (aggregate && aggregate !== "raw" && from && to) {
const grain: AggregateGrain = aggregate;
const cap = Math.min(limit, BUCKET_CAP[grain]);