tom/HealthLog
1
0 Fork 0
mirror of https://github.com/MBombeck/HealthLog.git synced 2026-11-08 12:32:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix/v1397-units

Browse Source
This commit is contained in:
Marc-André Bombeck authored and GitHub committed 2026-10-01 10:12:47 +02:00
2 parents a1fa84dcdf 1200d1bb05
commit 4a012be37e
51 files changed
+3265 -423
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/api/openapi.yaml
+16 -9
View File
@@ -41066,9 +41066,9 @@ components:
properties:
id:
type: string
description: Measurement row id; `sleep:<wake-day>` for a sleep night; `day:<local-day>` for a day bucket (`pulse` /
`glucose` beyond 90 days); `hour:<ISO start>` for an hour bucket (`pulse` / `glucose` within 90 days
when the window holds more than 10 000 readings).
description: Measurement row id; `sleep:<wake-day>` for a sleep night; `day:<local-day>` for a day bucket (`pulse`,
`glucose`, `heartRateVariability`, `oxygenSaturation` beyond 90 days); `hour:<ISO start>` for an hour
bucket (the same kinds within 90 days when the window holds more than 10 000 readings).
at:
type: string
format: date-time
@@ -41141,12 +41141,13 @@ components:
- stats
additionalProperties: false
description: "iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an
explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and
`glucose` beyond 90 days per local day, and `pulse` / `glucose` within 90 days per local hour when the window
holds more than 10 000 readings (a continuous glucose sensor, a watch recording heart rate every minute; a cuff
or fingerstick meter stays raw). A bucket carries the mean as `value`; a `pulse` bucket also carries its range
as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats`
is always computed over the underlying readings, never over the buckets."
explicit `unit` token, and a summary `stats` block. Sample-dense kinds (`pulse`, `glucose`,
`heartRateVariability`, `oxygenSaturation`) are bucketed server-side: beyond 90 days per local day, and within
90 days per local hour when the window holds more than 10 000 readings (a continuous glucose sensor, a watch or
ring recording every minute; a cuff, a fingerstick meter or a few readings a night stay raw). A bucket carries
the mean as `value` and otherwise has the shape of a reading of its kind; a `pulse` bucket also carries its
range as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading.
`stats` is always computed over the underlying readings, never over the buckets."
ListMeasurementsResponse:
type: object
properties:
@@ -41339,6 +41340,12 @@ components:
type: integer
minimum: 0
maximum: 9007199254740991
windowFrom:
description: "Set on the per-night SLEEP_DURATION mode when the requested window was longer than 365 days: the instant
the read started from. The window is a year ending at `to` (or now); page further back by moving `to`."
type: string
format: date-time
pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$
required:
- total
- limit
src/__tests__/dense-measurement-read-guard.test.ts
+299 -57
View File
@@ -1,6 +1,5 @@
/**
* Structural guard: background and insight readers must not materialise a
* dense measurement stream.
* Structural guard: no reader materialises a dense measurement stream.
*
* #1023: a single account syncing heart rate from a watch took the worker
* down within half an hour. The status-card read was bounded in time
@@ -10,17 +9,34 @@
* SQL (`readDayAggregates`, `readLiveBuckets`), so the reader's size tracks
* the number of days, not the sampling rate.
*
* What this guard pins, in the directories that run on the worker or feed an
* insight: every `measurement.findMany` without a `take` reads a type that is
* named as a literal and is not one a device streams. A read whose type is a
* variable could be handed a dense type by the next caller, so it must be
* listed below with the reason it is safe. The list is frozen both ways: a new
* unlisted read fails, and an entry whose read is gone fails too.
* The same class kept turning up outside the directories this guard first
* walked: the targets page read a year of glucose and a month of heart rate,
* the cycle insights read a year of every outcome channel, the per-kind series
* read heart-rate variability and blood oxygen raw for ten years, all from
* `app/api` or a helper beside it. So the guard now covers the request
* handlers, import, export, the targets and the doctor report as well, and
* the raw SQL reads of `measurements` anywhere in the tree.
*
* It is a tripwire, not a proof: the matcher reads source text, and a read
* written some other way (a raw SELECT of rows, a helper in another
* directory) passes it. The floor below fails the guard if the matcher stops
* finding reads at all.
* Two checks.
*
* 1. `measurement.findMany` without a `take`, in the scoped directories,
* must be bounded by its `where`: a type named as a literal (or a list of
* literals) none of which a device streams, or a key list (`id`,
* `externalId` or `measuredAt` `in: [...]`, or an exact `measuredAt`).
* Anything else is listed below with the reason it stays bounded.
*
* 2. A raw `$queryRaw` / `$queryRawUnsafe` whose statement reads from
* `measurements` must fold: `GROUP BY`, `DISTINCT`, `LIMIT`, or a select
* list of aggregates only. A statement that returns rows is listed below
* with its reason. A statement held in a variable is invisible to the
* matcher; that is the check's known blind spot.
*
* Both lists are frozen both ways: a new unlisted read fails, and an entry
* whose read is gone fails too. The floors below fail the guard if a matcher
* stops finding reads at all, and the "plants" cases run each matcher over a
* planted offender, which is the mutation proof kept in the suite: the
* planted findMany of `BLOOD_GLUCOSE` over a year and the planted raw SELECT of
* every pulse row are both caught. It is a tripwire, not a proof.
*/
import { describe, expect, it } from "vitest";
import { readFileSync } from "node:fs";
@@ -42,6 +58,12 @@ const SCOPED_DIRS = [
"lib/analytics/",
"lib/dashboard/",
"lib/daily/",
// Widened in v1.39.7: the request handlers and the helpers they lean on.
"app/api/",
"lib/import",
"lib/export",
"lib/targets/",
"lib/doctor-report",
];
/** Types a device writes at sampling rate rather than a few times a day. */
@@ -55,8 +77,8 @@ const DENSE_TYPES: ReadonlySet<string> = new Set([
]);
/**
* Untaken reads whose type is not a sparse literal, keyed
* `<file>::<type expression>`, with why each stays bounded.
* Untaken reads whose `where` neither names sparse literal types nor a key
* list, keyed `<file>::<type expression>`, with why each stays bounded.
*/
const ALLOWED: Record<string, string> = {
"lib/insights/comprehensive-generate.ts::(shorthand)":
@@ -69,8 +91,6 @@ const ALLOWED: Record<string, string> = {
"A constant for six-minute-walk distance, written at most daily.",
"lib/insights/derived/wellness-scores.ts::measurementType":
"Daily computed / provider scores (recovery, strain, readiness), one row per day.",
"lib/jobs/reaction-line.ts::{ in: [...types]":
"Exact-instant match (`measuredAt: row.occurredAt`), not a window.",
"lib/jobs/step-consolidation-repair.ts::STEP_TYPE":
"Repair job over one user's step rows for a bounded day range; runs once per repair, not per status refresh.",
'lib/ai/coach/snapshot.ts::"BLOOD_GLUCOSE" as never':
@@ -79,6 +99,34 @@ const ALLOWED: Record<string, string> = {
"Same 30-day clinical glucose panel as the coach snapshot (per-reading metrics), grouped by meal context.",
"lib/analytics/score/reader.ts::{ in: types":
"Health-score inputs: steps (drained to one row per day nightly), sleep, waist, weight, blood pressure, fasting glucose only.",
'app/api/analytics/route.ts::"BLOOD_GLUCOSE"':
"The 30-day per-context glucose summary, the same window and bound as the dashboard and coach glucose panels: at a sensor's fixed rate, below 45 000 rows.",
"app/api/measurements/series/route.ts::(shorthand)":
"The raw branch of the per-kind series. Every dense kind reaches it only inside 90 days and under 10 000 rows (counted first); the sparse kinds are a few readings a day.",
"app/api/measurements/batch/route.ts::p.row.type as MeasurementType":
"Cross-source merge probe: one ±tolerance window around each posted reading's instant, so the rows found are bounded by the batch size.",
"app/api/measurements/route.ts::(shorthand)":
"Cross-source merge probe for one posted sleep segment: SLEEP_DURATION in a ±tolerance window around its end.",
"lib/targets/build-response.ts::{ in: recentTypes":
"Thirty days of weight, blood pressure, resting heart rate, body fat and steps (drained to one row per day). PULSE is filtered out of `recentTypes` and folds in SQL.",
"lib/doctor-report/collect.ts::{ notIn: rawExcluded":
"The report window's sparse types. Pulse, heart-rate variability and blood oxygen over 10 000 readings or any window over 90 days, and glucose over 90 days, are in `rawExcluded` and read as day buckets; glucose inside 90 days stays raw for the clinical panel (per-reading metrics), below 26 000 rows at a sensor's rate.",
};
/**
* Raw statements that read rows of `measurements` without folding them, keyed
* `<file>::<ordinal among this file's unfolded statements>`, with why each
* stays bounded.
*/
const ALLOWED_SQL: Record<string, string> = {
"lib/analytics/bp-in-target-fast-path.ts::1":
"Canonical-source blood pressure, one type over 365 days: a cuff, a few readings a day. The in-target pairing needs the individual readings.",
"lib/insights/comprehensive-aggregator.ts::1":
"Canonical-source blood pressure and weight over 90 days: sparse types, a few rows a day.",
"lib/measurements/reconcile-external-measurement.ts::1":
"Row locks (`FOR UPDATE`) on an id list the caller already read; returns those ids only.",
"lib/rollups/measurement-coverage.ts::1":
"Selects from `unnest` of the caller's type list; `measurements` is read inside `EXISTS`, one row per type at most.",
};
interface UntakenRead {
@@ -87,63 +135,199 @@ interface UntakenRead {
typeExpr: string;
}
function scopedFiles(): string[] {
function lineOf(src: string, index: number): number {
return src.slice(0, index).split("\n").length;
}
/** Index just past the bracket that closes the one at `open`. */
function closeOf(src: string, open: number): number {
const pair: Record<string, string> = { "(": ")", "{": "}", "[": "]" };
const want = pair[src[open]];
let depth = 1;
let i = open + 1;
while (depth > 0 && i < src.length) {
if (src[i] === src[open]) depth += 1;
else if (src[i] === want) depth -= 1;
i += 1;
}
return i;
}
/** The `where: { … }` object of a call, or null when `where` is a variable. */
function whereObject(call: string): string | null {
const m = /\bwhere\s*:\s*\{/.exec(call);
if (!m) return null;
const open = m.index + m[0].length - 1;
return call.slice(open, closeOf(call, open));
}
/** The type expression a `where` object constrains on. */
function typeExprOf(call: string): string {
const where = whereObject(call);
if (where === null) {
return /\bwhere\s*[,}]/.test(call) ? "(where variable)" : "(none)";
}
const typed = /\btype\s*:\s*([^,\n}]+)/.exec(where);
if (typed) return typed[1].trim();
return /\btype\s*[,}\n]/.test(where) ? "(shorthand)" : "(none)";
}
/** A `where` that pins rows by key rather than by a window. */
function keyBounded(call: string): boolean {
const where = whereObject(call) ?? "";
return (
/\b(?:id|externalId|measuredAt)\s*:\s*\{\s*in\s*:/.test(where) ||
/\bmeasuredAt\s*:\s*(?![\s{])/.test(where)
);
}
function sparseLiteral(typeExpr: string): boolean {
const lit = /^"([A-Z_0-9]+)"/.exec(typeExpr);
if (lit) return !DENSE_TYPES.has(lit[1]);
const list = /^\{\s*in\s*:\s*\[([^\]]*)/.exec(typeExpr);
if (!list) return false;
const items = list[1]
.split(",")
.map((s) => s.trim())
.filter(Boolean);
return (
items.length > 0 &&
items.every((item) => {
const m = /^"([A-Z_0-9]+)"$/.exec(item);
return m !== null && !DENSE_TYPES.has(m[1]);
})
);
}
/** Every untaken `measurement.findMany` in `src`, with its type expression. */
function untakenFindManyReads(
file: string,
src: string,
): { all: number; untaken: UntakenRead[] } {
let all = 0;
const untaken: UntakenRead[] = [];
// Whitespace-tolerant: a call split across lines still matches.
const re = /\.measurement\s*\.\s*findMany\s*\(/g;
for (let m = re.exec(src); m; m = re.exec(src)) {
all += 1;
const call = src.slice(m.index, closeOf(src, m.index + m[0].length - 1));
if (/\btake\b/.test(call)) continue;
if (keyBounded(call)) continue;
// A type expression can itself contain a full bracket (`{ in: [...] }`);
// the regex above stops at the first `,` / `}` / newline, which is the
// stable key the allowlist uses.
untaken.push({
file,
line: lineOf(src, m.index),
typeExpr: typeExprOf(call),
});
}
return { all, untaken };
}
/** The statement text of the raw call starting at `at`, or null if opaque. */
function statementAt(src: string, at: number): string | null {
let i = at;
// Skip a type argument: `$queryRaw<Array<{ … }>>`.
if (src[i] === "<") {
let depth = 1;
i += 1;
while (depth > 0 && i < src.length) {
if (src[i] === "<") depth += 1;
else if (src[i] === ">") depth -= 1;
i += 1;
}
}
while (/\s/.test(src[i] ?? "")) i += 1;
if (src[i] === "`") {
// Tagged template: up to the matching backtick, skipping `${ … }`.
let j = i + 1;
while (j < src.length && src[j] !== "`") {
if (src[j] === "$" && src[j + 1] === "{") j = closeOf(src, j + 1);
else j += 1;
}
return src.slice(i, j + 1);
}
if (src[i] === "(") return src.slice(i, closeOf(src, i));
return null;
}
const READS_MEASUREMENTS =
/\bFROM\s+"?measurements"?(?![_\w])|canonicalMeasurementsFrom\s*\(/i;
const FOLDS =
/GROUP\s+BY|\bDISTINCT\b|\bLIMIT\b|^[\s`(]*(?:Prisma\.sql`)?\s*SELECT\s+(?:COUNT|SUM|AVG|MIN|MAX|STDDEV_POP)\s*\(/i;
interface RawRead {
file: string;
line: number;
folds: boolean;
}
/** Every raw statement in `src` that reads from `measurements`. */
function rawMeasurementReads(file: string, src: string): RawRead[] {
const out: RawRead[] = [];
const re = /\.\$queryRaw(?:Unsafe)?\b/g;
for (let m = re.exec(src); m; m = re.exec(src)) {
const statement = statementAt(src, m.index + m[0].length);
if (statement === null || !READS_MEASUREMENTS.test(statement)) continue;
out.push({
file,
line: lineOf(src, m.index),
folds: FOLDS.test(statement),
});
}
return out;
}
function sourceFiles(): string[] {
return walkSourceFiles(SRC, { floor: 3000 })
.filter((p) => SCOPED_DIRS.some((d) => p.startsWith(d)))
.filter((p) => !p.startsWith("generated/"))
.filter((p) => !p.includes("__tests__"))
.filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx"))
.sort();
}
function untakenReads(): { all: number; untaken: UntakenRead[] } {
function scan() {
let all = 0;
const untaken: UntakenRead[] = [];
for (const file of scopedFiles()) {
const raw: RawRead[] = [];
for (const file of sourceFiles()) {
const src = readFileSync(join(SRC, file), "utf8");
// Whitespace-tolerant: a call split across lines still matches.
const re = /\.measurement\s*\.\s*findMany\s*\(/g;
for (let m = re.exec(src); m; m = re.exec(src)) {
all += 1;
let i = m.index + m[0].length;
let depth = 1;
while (depth > 0 && i < src.length) {
if (src[i] === "(") depth += 1;
else if (src[i] === ")") depth -= 1;
i += 1;
}
const call = src.slice(m.index, i);
if (/\btake\b/.test(call)) continue;
const typed = /\btype\s*:\s*([^,\n}]+)/.exec(call);
const typeExpr = typed
? typed[1].trim()
: /\btype\s*,/.test(call)
? "(shorthand)"
: "(none)";
untaken.push({
file,
line: src.slice(0, m.index).split("\n").length,
typeExpr,
});
}
raw.push(...rawMeasurementReads(file, src));
if (!SCOPED_DIRS.some((d) => file.startsWith(d))) continue;
const reads = untakenFindManyReads(file, src);
all += reads.all;
untaken.push(...reads.untaken);
}
return { all, untaken };
return { all, untaken, raw };
}
function sparseLiteral(typeExpr: string): boolean {
const lit = /^"([A-Z_0-9]+)"/.exec(typeExpr);
return lit !== null && !DENSE_TYPES.has(lit[1]);
/** `<file>::<n>` for the n-th unfolded statement of each file. */
function unfoldedKeys(raw: readonly RawRead[]): Map<string, RawRead> {
const keys = new Map<string, RawRead>();
const perFile = new Map<string, number>();
for (const read of raw) {
if (read.folds) continue;
const n = (perFile.get(read.file) ?? 0) + 1;
perFile.set(read.file, n);
keys.set(`${read.file}::${n}`, read);
}
return keys;
}
describe("dense measurement reads stay bounded (#1023)", () => {
const { all, untaken } = untakenReads();
const { all, untaken, raw } = scan();
it("finds the reads it is meant to police", () => {
// Guard against a matcher that silently matches nothing.
expect(all).toBeGreaterThanOrEqual(40);
expect(untaken.length).toBeGreaterThanOrEqual(15);
// Pinned below the counts on 2026-10-01 (82 / 45 / 37 / 33).
expect(all).toBeGreaterThanOrEqual(75);
expect(untaken.length).toBeGreaterThanOrEqual(40);
expect(raw.length).toBeGreaterThanOrEqual(33);
expect(raw.filter((r) => r.folds).length).toBeGreaterThanOrEqual(30);
});
it("every untaken read names a sparse literal type or is listed with its reason", () => {
it("every untaken read names sparse literal types, a key list, or is listed with its reason", () => {
const offenders = untaken
.filter((r) => !sparseLiteral(r.typeExpr))
.filter((r) => !(`${r.file}::${r.typeExpr}` in ALLOWED))
@@ -156,20 +340,78 @@ describe("dense measurement reads stay bounded (#1023)", () => {
expect(Object.keys(ALLOWED).filter((k) => !present.has(k))).toEqual([]);
});
it("every raw read of measurements folds or is listed with its reason", () => {
const keys = unfoldedKeys(raw);
const offenders = [...keys]
.filter(([key]) => !(key in ALLOWED_SQL))
.map(([key, read]) => `${key} (line ${read.line})`);
expect(offenders).toEqual([]);
expect(Object.keys(ALLOWED_SQL).filter((k) => !keys.has(k))).toEqual([]);
});
it("the readers fixed for #1023 fold in SQL rather than reading rows", () => {
for (const file of [
"lib/insights/graded-series.ts",
"lib/rollups/tiered-context.ts",
"lib/jobs/coach-plan-review.ts",
"lib/measurements/daily-series-read.ts",
"lib/insights/derived/baseline.ts",
"lib/targets/glucose-read.ts",
]) {
const src = readFileSync(join(SRC, file), "utf8");
expect(src, file).not.toMatch(/\.measurement\s*\.\s*findMany\s*\(/);
}
const baseline = readFileSync(
join(SRC, "lib/insights/derived/baseline.ts"),
"utf8",
});
it("plants: an unbounded findMany of a dense type is caught", () => {
const planted = `
const rows = await prisma.measurement.findMany({
where: { userId, type: "BLOOD_GLUCOSE", measuredAt: { gte: oneYearAgo } },
select: { value: true, measuredAt: true },
});`;
const { all: n, untaken: found } = untakenFindManyReads(
"app/api/planted/route.ts",
planted,
);
expect(baseline).not.toMatch(/\.measurement\s*\.\s*findMany\s*\(/);
expect(n).toBe(1);
expect(found.map((r) => r.typeExpr)).toEqual(['"BLOOD_GLUCOSE"']);
expect(sparseLiteral(found[0].typeExpr)).toBe(false);
// The same read with a key list or a take is bounded.
expect(
untakenFindManyReads(
"x.ts",
planted.replace("measuredAt: { gte", "id: { in: ids }, x: { gte"),
).untaken,
).toEqual([]);
expect(
untakenFindManyReads(
"x.ts",
planted.replace("select:", "take: 10, select:"),
).untaken,
).toEqual([]);
// A list of sparse literals passes; one dense member does not.
expect(sparseLiteral('{ in: ["WEIGHT", "BODY_FAT"]')).toBe(true);
expect(sparseLiteral('{ in: ["WEIGHT", "PULSE"]')).toBe(false);
});
it("plants: a raw SELECT of rows from measurements is caught", () => {
const planted =
"const rows = await prisma.$queryRaw<Array<{ v: number }>>`\n" +
' SELECT m."value" FROM measurements m\n' +
' WHERE m."user_id" = ${userId} AND m."type" = \'PULSE\'\n' +
"`;";
const reads = rawMeasurementReads("lib/planted.ts", planted);
expect(reads).toHaveLength(1);
expect(reads[0].folds).toBe(false);
const folded = rawMeasurementReads(
"lib/planted.ts",
planted.replace("WHERE", "WHERE true GROUP BY 1 HAVING true AND"),
);
expect(folded[0].folds).toBe(true);
const aggregate = rawMeasurementReads(
"lib/planted.ts",
planted.replace('SELECT m."value"', 'SELECT COUNT(*), AVG(m."value")'),
);
expect(aggregate[0].folds).toBe(true);
});
});
src/__tests__/job-queue-runtime-guard.test.ts
+371
View File
@@ -0,0 +1,371 @@
/**
* Structural guard: every pg-boss queue declares how long its job can run, and
* a long one is safe against pg-boss's expiry.
*
* pg-boss gives a job fifteen minutes unless the send says otherwise, then
* fails it, fires its abort signal and retries it while the handler keeps
* going. The full-history imports ran beside themselves that way (#1066), and
* so could every other job that legitimately needs longer: the nightly status
* passes, the pre-generation, the record scan, the key rotation, the backups.
*
* `src/lib/jobs/queue-runtime.ts` declares every queue `short`, `budgeted` or
* `long`. This guard holds the tree to it:
*
* - every queue bound with `createAndWork` is in the table, and every table
* entry is bound;
* - every `boss.send` and every schedule tuple names a queue in the table;
* - a `long` queue has an expiry between the default and pg-boss's 24-hour
* ceiling, every send and schedule for it carries `expireInSeconds` (or an
* identifier the table names as its carrier), it stops on the job's budget
* where the table says (`jobBudget` / `jobDeadline`, or a declared
* one-shot), and it is exclusive per identity where the table says
* (`lockedPass` at the binding, or the named marker);
* - a `budgeted` queue stops on the budget and relies on the default expiry;
* - a `short` queue's sends set no expiry of their own, which would mean the
* job is longer than the table claims.
*
* Floors fail the guard if a matcher stops matching, and the "plants" cases
* run the checks over a planted long queue that lacks each protection in turn
* (the mutation proof kept in the suite). Removing `lockedPass` from the
* pr-detection binding, or the expiry from its cron tuple, fails the real
* checks below.
*/
import { describe, expect, it } from "vitest";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { walkSourceFiles } from "./helpers/source-files";
import {
DEFAULT_JOB_EXPIRE_SECONDS,
QUEUE_RUNTIME,
type QueueRuntime,
type SourceSite,
} from "@/lib/jobs/queue-runtime";
const SRC = join(process.cwd(), "src");
const REGISTRARS = [
"lib/jobs/reminder/register-integration-sync.ts",
"lib/jobs/reminder/register-maintenance.ts",
"lib/jobs/reminder/register-reminders.ts",
"lib/jobs/reminder/register-rollup.ts",
"lib/jobs/reminder/register-status.ts",
];
/** pg-boss refuses an expiry of 24 hours or more. */
const MAX_EXPIRE_SECONDS = 24 * 60 * 60;
const files = walkSourceFiles(SRC, { floor: 3000 })
.filter((p) => !p.startsWith("generated/"))
.filter((p) => !p.includes("__tests__"))
.filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx"));
const source = new Map(
files.map((f) => [f, readFileSync(join(SRC, f), "utf8")]),
);
/** Strip comments so documentation that quotes code never matches. */
function code(text: string): string {
return text
.replace(/\/\*[\s\S]*?\*\//g, "")
.replace(/(^|[^:])\/\/.*$/gm, "$1");
}
/** Index just past the bracket that closes the one at `open`. */
function closeOf(text: string, open: number): number {
const pair: Record<string, string> = { "(": ")", "[": "]", "{": "}" };
const want = pair[text[open]];
let depth = 1;
let i = open + 1;
while (depth > 0 && i < text.length) {
if (text[i] === text[open]) depth += 1;
else if (text[i] === want) depth -= 1;
i += 1;
}
return i;
}
type Definitions = Map<string, Array<{ file: string; name: string }>>;
/** `const NAME = "queue-name"` definitions, per file. */
function definitionsOf(texts: ReadonlyMap<string, string>): Definitions {
const definitions: Definitions = new Map();
for (const [file, text] of texts) {
for (const m of text.matchAll(
/\bconst\s+([A-Z][A-Z0-9_]*)\s*(?::\s*string\s*)?=\s*"([a-z0-9-]+)"/g,
)) {
const list = definitions.get(m[1]) ?? [];
list.push({ file, name: m[2] });
definitions.set(m[1], list);
}
}
return definitions;
}
/** Resolve a queue identifier as seen from `file`, preferring its own. */
function resolve(
definitions: Definitions,
file: string,
ident: string,
): string | null {
const defs = definitions.get(ident) ?? [];
const own = defs.find((d) => d.file === file);
if (own) return own.name;
const names = new Set(defs.map((d) => d.name));
return names.size === 1 ? [...names][0] : null;
}
interface Site {
file: string;
queue: string;
text: string;
}
/** Every `createAndWork(boss, QUEUE, …)` in the registrars. */
function bindings(
texts: ReadonlyMap<string, string>,
definitions: Definitions,
): Site[] {
const out: Site[] = [];
for (const file of REGISTRARS) {
const text = code(texts.get(file) ?? "");
for (const m of text.matchAll(
/createAndWork\s*(?:<[^(]*?>)?\(\s*boss\s*,\s*([A-Z][A-Z0-9_]*)/g,
)) {
const open = text.indexOf("(", m.index);
const queue = resolve(definitions, file, m[1]);
if (queue) {
out.push({ file, queue, text: text.slice(open, closeOf(text, open)) });
}
}
}
return out;
}
/** Every `boss.send(QUEUE, …)` / `getGlobalBoss()?.send(QUEUE, …)`. */
function sends(
texts: ReadonlyMap<string, string>,
definitions: Definitions,
): Site[] {
const out: Site[] = [];
for (const [file, raw] of texts) {
const text = code(raw);
for (const m of text.matchAll(
/\b(?:boss|getGlobalBoss\(\)\??)\s*\.\s*send\s*(?:<[^(]*?>)?\(\s*([A-Z][A-Z0-9_]*)/g,
)) {
const open = text.indexOf("(", text.indexOf("send", m.index));
const queue = resolve(definitions, file, m[1]);
if (queue) {
out.push({ file, queue, text: text.slice(open, closeOf(text, open)) });
}
}
}
return out;
}
/** Every `[QUEUE, CRON, options?]` tuple in a registrar's `schedules`. */
function schedules(
texts: ReadonlyMap<string, string>,
definitions: Definitions,
): Site[] {
const out: Site[] = [];
for (const file of REGISTRARS) {
const text = code(texts.get(file) ?? "");
const m = /const\s+schedules\s*:\s*ScheduleEntry\[\]\s*=\s*\[/.exec(text);
if (!m) continue;
const open = m.index + m[0].length - 1;
const body = text.slice(open + 1, closeOf(text, open) - 1);
for (let i = 0; i < body.length; i += 1) {
if (body[i] !== "[") continue;
const end = closeOf(body, i);
const tuple = body.slice(i, end);
const ident = /^\[\s*([A-Z][A-Z0-9_]*)/.exec(tuple)?.[1];
const queue = ident ? resolve(definitions, file, ident) : null;
if (queue) out.push({ file, queue, text: tuple });
i = end - 1;
}
}
return out;
}
/** The text of `fn` (a function or a const) in `file`. */
function bodyOf(
texts: ReadonlyMap<string, string>,
site: { file: string; fn: string },
): string {
const text = code(texts.get(site.file) ?? "");
const m = new RegExp(
`(?:function\\s+${site.fn}\\b|const\\s+${site.fn}\\b)`,
).exec(text);
if (!m) return "";
const end = text.indexOf("\n}", m.index);
return text.slice(m.index, end < 0 ? undefined : end + 2);
}
function siteText(
texts: ReadonlyMap<string, string>,
site: SourceSite,
binding: Site | undefined,
): string {
return site === "binding" ? (binding?.text ?? "") : bodyOf(texts, site);
}
const STOPS = /\bjob(?:Budget|Deadline)\s*\(/;
/** Every finding for one queue's declaration against the tree. */
function check(
queue: string,
entry: QueueRuntime,
texts: ReadonlyMap<string, string>,
all: { bindings: Site[]; sends: Site[]; schedules: Site[] },
): string[] {
const problems: string[] = [];
const binding = all.bindings.find((b) => b.queue === queue);
const carriers = [...all.sends, ...all.schedules].filter(
(s) => s.queue === queue,
);
if (entry.runtime === "long") {
if (
!(entry.expireInSeconds > DEFAULT_JOB_EXPIRE_SECONDS) ||
!(entry.expireInSeconds < MAX_EXPIRE_SECONDS)
) {
problems.push(`${queue}: expiry ${entry.expireInSeconds}s out of range`);
}
for (const site of carriers) {
const carried =
/\bexpireInSeconds\b/.test(site.text) ||
entry.expiryVia.some((v) => new RegExp(`\\b${v}\\b`).test(site.text));
if (!carried) {
problems.push(
`${queue}: ${site.file} sends or schedules it without an expiry`,
);
}
}
if (typeof entry.stop === "object" && "oneShot" in entry.stop) {
if (entry.stop.oneShot.length < 20) {
problems.push(`${queue}: a one-shot needs its reason`);
}
} else if (!STOPS.test(siteText(texts, entry.stop, binding))) {
problems.push(`${queue}: does not stop on the job's budget`);
}
if (entry.exclusive === "lockedPass") {
if (!/\blockedPass\s*\(/.test(binding?.text ?? "")) {
problems.push(`${queue}: binding holds no lockedPass`);
}
} else if (
!siteText(texts, entry.exclusive.at, binding).includes(
entry.exclusive.marker,
)
) {
problems.push(`${queue}: no "${entry.exclusive.marker}" where declared`);
}
} else {
if (entry.runtime === "budgeted") {
if (!STOPS.test(siteText(texts, entry.budget, binding))) {
problems.push(`${queue}: does not stop on the job's budget`);
}
}
for (const site of carriers) {
if (/\bexpireInSeconds\b/.test(site.text)) {
problems.push(
`${queue}: ${site.file} sets an expiry on a queue declared ${entry.runtime}`,
);
}
}
}
return problems;
}
function scan(texts: ReadonlyMap<string, string>) {
const definitions = definitionsOf(texts);
return {
bindings: bindings(texts, definitions),
sends: sends(texts, definitions),
schedules: schedules(texts, definitions),
};
}
describe("every queue declares its runtime", () => {
const all = scan(source);
const bound = new Set(all.bindings.map((b) => b.queue));
it("finds the bindings, sends and schedules it is meant to police", () => {
// Pinned below the counts on 2026-10-01 (112 / 64 / 74).
expect(all.bindings.length).toBeGreaterThanOrEqual(105);
expect(all.sends.length).toBeGreaterThanOrEqual(50);
expect(all.schedules.length).toBeGreaterThanOrEqual(70);
expect(
Object.values(QUEUE_RUNTIME).filter((e) => e.runtime === "long").length,
).toBeGreaterThanOrEqual(15);
});
it("every bound queue is in the table, and every entry is bound", () => {
expect([...bound].filter((q) => !(q in QUEUE_RUNTIME)).sort()).toEqual([]);
expect(
Object.keys(QUEUE_RUNTIME)
.filter((q) => !bound.has(q))
.sort(),
).toEqual([]);
});
it("every send and schedule names a queue in the table", () => {
const unknown = [...all.sends, ...all.schedules]
.filter((s) => !(s.queue in QUEUE_RUNTIME))
.map((s) => `${s.file} → ${s.queue}`);
expect(unknown).toEqual([]);
});
it("every queue meets its declaration", () => {
const problems = Object.entries(QUEUE_RUNTIME).flatMap(([queue, entry]) =>
check(queue, entry, source, all),
);
expect(problems).toEqual([]);
});
it("plants: a long queue missing any protection is caught", () => {
const registrar = REGISTRARS[1];
const planted = (binding: string, schedule: string) =>
new Map([
...source,
[
registrar,
`const PLANT_QUEUE = "planted-long";\n` +
`const schedules: ScheduleEntry[] = [\n ${schedule},\n];\n` +
`async function register(boss) {\n await createAndWork(boss, PLANT_QUEUE, { localConcurrency: 1 }, ${binding});\n}\n`,
],
]);
const entry: QueueRuntime = {
runtime: "long",
expireInSeconds: 3600,
expiryVia: [],
stop: "binding",
exclusive: "lockedPass",
why: "planted",
};
const findings = (binding: string, schedule: string) => {
const texts = planted(binding, schedule);
return check("planted-long", entry, texts, scan(texts));
};
const safe = [
"lockedPass(PLANT_QUEUE, () => WHOLE_PASS, async (jobs) => run(jobBudget(jobs)))",
"[PLANT_QUEUE, CRON, { expireInSeconds: 3600 }]",
] as const;
expect(findings(...safe)).toEqual([]);
expect(findings(safe[0], "[PLANT_QUEUE, CRON]")).toEqual([
`planted-long: ${registrar} sends or schedules it without an expiry`,
]);
expect(
findings("lockedPass(PLANT_QUEUE, () => WHOLE_PASS, run)", safe[1]),
).toEqual(["planted-long: does not stop on the job's budget"]);
expect(findings("async (jobs) => run(jobBudget(jobs))", safe[1])).toEqual([
"planted-long: binding holds no lockedPass",
]);
expect(
check(
"planted-long",
{ ...entry, expireInSeconds: 900 },
...((): [ReadonlyMap<string, string>, ReturnType<typeof scan>] => {
const texts = planted(...safe);
return [texts, scan(texts)];
})(),
),
).toEqual(["planted-long: expiry 900s out of range"]);
});
});
src/app/api/admin/encryption/rotate/route.ts
+2
View File
@@ -33,6 +33,7 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import {
ENCRYPTION_KEY_ROTATE_QUEUE,
ENCRYPTION_KEY_ROTATE_SINGLETON,
ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS,
} from "@/lib/jobs/encryption-key-rotate";
import { getActiveKeyId } from "@/lib/crypto";
import { NextRequest } from "next/server";
@@ -57,6 +58,7 @@ export const POST = apiHandler(async (request: NextRequest) => {
singletonKey: ENCRYPTION_KEY_ROTATE_SINGLETON,
retryLimit: 2,
retryDelay: 60,
expireInSeconds: ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS,
},
);
src/app/api/cycle/insights/__tests__/route-locale.test.ts
+3
View File
@@ -37,6 +37,9 @@ vi.mock("@/lib/cycle/phase-crosstab", async (importOriginal) => ({
discoverPhaseCorrelations(args),
}));
vi.mock("@/lib/measurements/day-aggregates", () => ({
readSourceDayAggregates: vi.fn().mockResolvedValue([]),
}));
vi.mock("@/lib/db", () => ({
prisma: {
auditLog: { create: vi.fn() },
src/app/api/cycle/insights/route.ts
+18 -22
View File
@@ -43,6 +43,7 @@ import {
type SymptomDay,
} from "@/lib/cycle/symptom-phase";
import { addDays } from "@/lib/cycle/day-math";
import { readSourceDayAggregates } from "@/lib/measurements/day-aggregates";
import { moodDateKey } from "@/lib/mood/date-key";
import { DEFAULT_TIMEZONE } from "@/lib/tz/format";
@@ -130,27 +131,19 @@ export const GET = apiHandler(async () => {
orderBy: { measuredAt: "asc" },
select: { measuredAt: true, value: true },
}),
// The outcome metrics the phase contrast compares — soft-delete-scoped,
// canonical-source deduped per day inside `metricDayMap`. Skipped whole
// for a caller outside `measurements`.
// The outcome metrics the phase contrast compares, soft-delete-scoped.
// A year of steps, heart-rate variability and sensor glucose is far too
// many readings to hold as objects (#1023), and the crosstab uses only
// each day's sum and count after picking one source and device per day.
// So the fold runs in SQL, one row per type, local day, source and
// device, the read the coach's cycle snapshot already makes. Skipped
// whole for a caller outside `measurements`.
seesMeasurements
? prisma.measurement.findMany({
where: {
userId: user.id,
deletedAt: null,
type: { in: PHASE_CROSSTAB_METRIC_TYPES },
measuredAt: {
gte: new Date(Date.parse(`${from}T00:00:00Z`)),
},
},
orderBy: { measuredAt: "asc" },
select: {
type: true,
value: true,
measuredAt: true,
source: true,
deviceType: true,
},
? readSourceDayAggregates({
userId: user.id,
types: PHASE_CROSSTAB_METRIC_TYPES,
since: new Date(Date.parse(`${from}T00:00:00Z`)),
timeZone: tz,
})
: [],
prisma.user.findUnique({
@@ -200,10 +193,13 @@ export const GET = apiHandler(async () => {
today,
);
// Each row stands for its day's readings: `value` is their sum and
// `count` their number, so `metricDayMap` gets the raw readings' day mean.
const measurements: CrossMetricMeasurement[] = measurementRows.map((m) => ({
type: m.type,
value: m.value,
measuredAt: m.measuredAt,
value: m.sum,
count: m.n,
measuredAt: m.firstAt,
source: m.source,
deviceType: m.deviceType,
}));
src/app/api/export/health-record/__tests__/route.test.ts
+6 -2
View File
@@ -19,8 +19,12 @@ import { unzipSync } from "fflate";
vi.mock("@/lib/db", () => ({
prisma: {
// `count` sizes the report's pulse window (#1023): a sparse account.
measurement: { findMany: vi.fn(), count: vi.fn(async () => 0) },
// `groupBy` sizes the report's dense windows (#1023): a sparse account.
measurement: {
findMany: vi.fn(),
count: vi.fn(async () => 0),
groupBy: vi.fn(async () => []),
},
medication: { findMany: vi.fn() },
medicationIntakeEvent: { findMany: vi.fn() },
moodEntry: { findMany: vi.fn() },
src/app/api/insights/targets/__tests__/route.test.ts
+43 -6
View File
@@ -30,6 +30,16 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { NextRequest } from "next/server";
// The glucose and heart-rate streams fold in SQL; this file pins the route's
// composition over Prisma mocks, so both readers are mocked at the module.
vi.mock("@/lib/targets/glucose-read", async (importOriginal) => ({
...(await importOriginal<typeof import("@/lib/targets/glucose-read")>()),
readGlucoseTargetSummaries: vi.fn(),
}));
vi.mock("@/lib/analytics/resting-pulse-read", () => ({
readRestingPulseProxy: vi.fn(),
}));
vi.mock("@/lib/db", () => ({
prisma: {
user: { findUnique: vi.fn() },
@@ -84,6 +94,11 @@ import { GET } from "../route";
import { getSession } from "@/lib/auth/session";
import { prisma } from "@/lib/db";
import { __resetAllCachesForTests } from "@/lib/cache/server-cache";
import {
foldGlucoseTargetRows,
readGlucoseTargetSummaries,
} from "@/lib/targets/glucose-read";
import { readRestingPulseProxy } from "@/lib/analytics/resting-pulse-read";
const SESSION_USER = {
id: "user-targets-1",
@@ -153,6 +168,8 @@ beforeEach(() => {
// v1.28.25 — the latest-ever-per-type read is a raw `DISTINCT ON`
// (Prisma's `distinct` dedups client-side after pulling every row).
(prisma.$queryRawUnsafe as ReturnType<typeof vi.fn>).mockResolvedValue([]);
vi.mocked(readGlucoseTargetSummaries).mockResolvedValue([]);
vi.mocked(readRestingPulseProxy).mockResolvedValue([]);
});
describe("GET /api/insights/targets — mood-rollup tier swap", () => {
@@ -338,16 +355,26 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => {
it("bounds the glucose read with the same one-year floor (v1.16.8)", async () => {
// The glucose section used to scan the user's entire BLOOD_GLUCOSE
// history on every cold build (no `measuredAt` filter at all) just
// to read the latest value per context. The read now carries the
// same 365-day floor as the latest-ever-per-type query.
// to read the latest value per context. It now folds in SQL, newest
// per context over a year and per day over thirty days, and no glucose
// or heart-rate reading is read as a row.
await callGet(makeReq());
const calls = (
prisma.measurement.findMany as ReturnType<typeof vi.fn>
).mock.calls.map((c) => c[0]);
const glucoseCall = calls.find((c) => c?.where?.type === "BLOOD_GLUCOSE");
expect(glucoseCall).toBeDefined();
expect(glucoseCall?.where?.measuredAt?.gte).toBeInstanceOf(Date);
expect(
calls.find((c) => c?.where?.type === "BLOOD_GLUCOSE"),
).toBeUndefined();
expect(
calls.filter((c) => c?.where?.type?.in?.includes?.("PULSE")),
).toEqual([]);
const [args] = vi.mocked(readGlucoseTargetSummaries).mock.calls[0];
const day = 86_400_000;
expect(Date.now() - args.since.getTime()).toBeGreaterThan(364 * day);
expect(Date.now() - args.since.getTime()).toBeLessThan(366 * day);
expect(Date.now() - args.recentSince.getTime()).toBeGreaterThan(29 * day);
expect(Date.now() - args.recentSince.getTime()).toBeLessThan(31 * day);
});
it("pins the complete ordered public response for representative populated data", async () => {
@@ -408,10 +435,20 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => {
).mockImplementation(
async (args: { where?: { type?: string | { in: string[] } } }) => {
if (args.where?.type === "SLEEP_DURATION") return sleepRows;
if (args.where?.type === "BLOOD_GLUCOSE") return glucoseRows;
return genericRows;
},
);
vi.mocked(readGlucoseTargetSummaries).mockImplementation(
async ({ recentSince, timeZone }) =>
foldGlucoseTargetRows(
glucoseRows.map((row) => ({
...row,
glucoseContext: row.glucoseContext as "FASTING",
})),
recentSince,
timeZone,
),
);
(prisma.$queryRawUnsafe as ReturnType<typeof vi.fn>).mockResolvedValue([
{ type: "WEIGHT", value: 70 },
{ type: "BLOOD_PRESSURE_SYS", value: 125 },
src/app/api/measurements/route.ts
+19 -4
View File
@@ -543,9 +543,18 @@ async function sleepListResponse(
loadUserSourcePriority(user.id),
]);
// Bound the read window: honour an explicit from/to, otherwise cap to
// the last year so the per-stage scan stays small.
const since = from ?? new Date(Date.now() - SLEEP_LIST_MAX_DAYS * 86_400_000);
// Bound the read window to a year ending at `to` (or now). An explicit
// `from` narrows it but never widens it: an early `from` used to walk every
// per-stage row back to that date, which on a multi-year, multi-source
// account is the unbounded read this cap exists to stop. Older nights are
// reached by moving `to` back, and `meta.windowFrom` says where a cut
// window started so a client can tell a cut from an empty history.
const windowEnd = to ?? new Date();
const floor = new Date(
windowEnd.getTime() - SLEEP_LIST_MAX_DAYS * 86_400_000,
);
const since = from && from > floor ? from : floor;
const windowCut = from !== undefined && from < floor;
const rows = await prisma.measurement.findMany({
where: {
userId: user.id,
@@ -690,7 +699,13 @@ async function sleepListResponse(
});
return apiSuccess({
measurements,
meta: { total: totalNights, limit, offset, groupBy: "night" },
meta: {
total: totalNights,
limit,
offset,
groupBy: "night",
...(windowCut ? { windowFrom: since.toISOString() } : {}),
},
});
}
src/app/api/measurements/series/route.ts
+23 -9
View File
@@ -49,11 +49,23 @@ const SLEEP_SERIES_MAX_DAYS = 365;
* byte-identical. Sparse kinds (weight, BP, …) are untouched.
*/
const DENSE_SERIES_RAW_WINDOW_DAYS = 90;
const DENSE_SERIES_KINDS: ReadonlySet<string> = new Set(["glucose", "pulse"]);
/**
* The kinds a device can write at sampling rate. Heart-rate variability and
* blood oxygen joined pulse and glucose once watches and rings began writing
* them every few minutes overnight: their 3650-day "Alle" range read every
* raw row, which is the read this rule exists to stop.
*/
const DENSE_SERIES_KINDS: ReadonlySet<string> = new Set([
"glucose",
"pulse",
"heartRateVariability",
"oxygenSaturation",
]);
/**
* #1023 — inside the raw window, a pulse or glucose series with more rows
* than this is bucketed per local hour in SQL instead of sent raw.
* #1023 — inside the raw window, a dense-kind series (pulse, glucose, HRV,
* blood oxygen) with more rows than this is bucketed per local hour in SQL
* instead of sent raw.
*
* A watch that records heart rate once a minute puts 43 000 rows in the
* default 30-day window and 130 000 in 90 days: a 15 MB response the client
@@ -381,7 +393,7 @@ export const GET = apiHandler(async (request: NextRequest) => {
},
})) > DENSE_SERIES_RAW_ROW_CAP)
) {
// Day buckets past the raw window; hour buckets for a pulse or CGM
// Day buckets past the raw window; hour buckets for a dense-kind
// stream too dense to send raw inside it.
const grain = days > DENSE_SERIES_RAW_WINDOW_DAYS ? "day" : "hour";
// v1.28.25 — long-window read of a sample-dense kind (CGM glucose,
@@ -489,17 +501,19 @@ export const GET = apiHandler(async (request: NextRequest) => {
};
}
} else {
// pulse — `value` is the bucket's average; valueMin/valueMax carry the
// bucket's low/high band (folding each hourly import bucket's own
// `value` is the bucket's average. For pulse, valueMin/valueMax carry
// the bucket's low/high band (folding each hourly import bucket's own
// spread via the COALESCE above), same band semantics as the per-hour
// shape.
// shape. The other dense kinds keep the bare point shape their raw rows
// have, so a client reads a bucket the way it reads a reading.
points = bucketRows.map((r) => ({
id: dayId(r.bucket_start),
at: r.bucket_start.toISOString(),
value: round2(r.mean),
secondary: null,
valueMin: r.min_value,
valueMax: r.max_value,
...(kind === "pulse"
? { valueMin: r.min_value, valueMax: r.max_value }
: {}),
}));
if (
rawAgg !== undefined &&
src/lib/analytics/resting-pulse.ts
+24 -5
View File
@@ -139,6 +139,26 @@ export function resolveRestingPulseSeries(input: {
pulseSamples: ReadonlyArray<PulseSample>;
/** Day key for the proxy buckets: the user's own day. */
dayKeyOf: (d: Date) => string;
}): { series: PulseSample[]; which: "resting" | "proxy" | "none" } {
return mergeRestingWithProxy({
restingSamples: input.restingSamples,
proxySamples: deriveRestingProxyFromPulse(
input.pulseSamples,
input.dayKeyOf,
),
dayKeyOf: input.dayKeyOf,
});
}
/**
* {@link resolveRestingPulseSeries} for a caller that already holds the
* proxy, one point per day, as `readRestingPulseProxy` folds it in Postgres.
* The per-day merge and the label are the same.
*/
export function mergeRestingWithProxy(input: {
restingSamples: ReadonlyArray<PulseSample>;
proxySamples: ReadonlyArray<PulseSample>;
dayKeyOf: (d: Date) => string;
}): { series: PulseSample[]; which: "resting" | "proxy" | "none" } {
const dayKeyOf = input.dayKeyOf;
const byTime = (a: PulseSample, b: PulseSample) =>
@@ -148,14 +168,13 @@ export function resolveRestingPulseSeries(input: {
// Days already covered by a real resting row — the proxy must not
// second-guess them.
const restingDays = new Set(resting.map((s) => dayKeyOf(s.measuredAt)));
const proxy = deriveRestingProxyFromPulse(
input.pulseSamples,
dayKeyOf,
).filter((p) => !restingDays.has(dayKeyOf(p.measuredAt)));
const proxy = input.proxySamples.filter(
(p) => !restingDays.has(dayKeyOf(p.measuredAt)),
);
if (resting.length === 0) {
return proxy.length > 0
? { series: proxy, which: "proxy" }
? { series: [...proxy], which: "proxy" }
: { series: [], which: "none" };
}
if (proxy.length === 0) return { series: resting, which: "resting" };
src/lib/crypto/encryption-corpus.ts
+66 -30
View File
@@ -390,19 +390,29 @@ export interface RotationResult {
dropped: number;
}
/** Re-encrypt one column's stale rows to the active key. */
/** Thrown out of a column walk when the caller's time budget is spent. */
class RotationStopped extends Error {}
/**
* Re-encrypt one column's stale rows to the active key. `shouldStop` is the
* caller's time budget, checked between rows; a stopped column reports
* `stopped` and the rows it did not reach stay on their old key for the next
* run, which skips every row already rotated.
*/
export async function rotateColumn(
client: CorpusClient,
col: EncryptedColumn,
): Promise<RotationResult> {
shouldStop: () => boolean = () => false,
): Promise<RotationResult & { stopped: boolean }> {
const delegate = getDelegate(client, col.model);
const result: RotationResult = {
const result: RotationResult & { stopped: boolean } = {
model: col.model,
field: col.field,
scanned: 0,
rotated: 0,
errors: 0,
dropped: 0,
stopped: false,
};
/**
@@ -432,6 +442,10 @@ export async function rotateColumn(
// (`convertSingleValueBackup`). A row converted, replaced or deleted since
// it was listed is gone, not an error.
for await (const id of singleValueIds(delegate)) {
if (shouldStop()) {
result.stopped = true;
break;
}
result.scanned += 1;
try {
const outcome = await convertSingleValueBackup(
@@ -452,31 +466,37 @@ export async function rotateColumn(
// Idempotent — rows already on the active key are skipped, so an
// interrupted run resumes cleanly on the next invocation. `scanned` counts
// the rows that hold ciphertext, whatever the column.
await walkColumn(delegate, col, async ({ id, value, codec }) => {
result.scanned += 1;
if (walkedKeyId(value, codec, col.kind) === getActiveKeyId()) return;
try {
// The plaintext is never inspected, only re-sealed, so every envelope
// a stored value can legitimately carry survives rotation untouched.
const next =
codec !== null
? reencryptBlob(value as Uint8Array, codec, col.aad)
: fromCiphertext(
encrypt(decrypt(toCiphertext(value, col.kind)!)),
col.kind,
);
await delegate.update({
where: { [pkField(col)]: id },
data: { [col.field]: next },
});
result.rotated += 1;
} catch (err) {
// Deleted between the read and the write (a backup replacing its
// pieces, an account going): nothing is left to rotate.
if (isRowGone(err)) return;
await onUnreadable(id);
}
});
try {
await walkColumn(delegate, col, async ({ id, value, codec }) => {
if (shouldStop()) throw new RotationStopped();
result.scanned += 1;
if (walkedKeyId(value, codec, col.kind) === getActiveKeyId()) return;
try {
// The plaintext is never inspected, only re-sealed, so every envelope
// a stored value can legitimately carry survives rotation untouched.
const next =
codec !== null
? reencryptBlob(value as Uint8Array, codec, col.aad)
: fromCiphertext(
encrypt(decrypt(toCiphertext(value, col.kind)!)),
col.kind,
);
await delegate.update({
where: { [pkField(col)]: id },
data: { [col.field]: next },
});
result.rotated += 1;
} catch (err) {
// Deleted between the read and the write (a backup replacing its
// pieces, an account going): nothing is left to rotate.
if (isRowGone(err)) return;
await onUnreadable(id);
}
});
} catch (err) {
if (!(err instanceof RotationStopped)) throw err;
result.stopped = true;
}
return result;
}
@@ -488,16 +508,31 @@ export interface CorpusRotation {
totalErrors: number;
/** Unreadable rows deleted from `disposable` columns. */
totalDropped: number;
/**
* The time budget ran out before every column was walked. Running again
* finishes the job: rows already on the active key are skipped.
*/
stoppedEarly: boolean;
}
/** Re-encrypt the whole corpus to the active key. Idempotent + active-key-only. */
/**
* Re-encrypt the whole corpus to the active key. Idempotent + active-key-only.
* `shouldStop` is the caller's time budget (`jobBudget` in the worker).
*/
export async function rotateCorpus(
client: CorpusClient,
shouldStop: () => boolean = () => false,
): Promise<CorpusRotation> {
const activeKeyId = getActiveKeyId();
const results: RotationResult[] = [];
let stoppedEarly = false;
for (const col of ENCRYPTED_COLUMNS) {
results.push(await rotateColumn(client, col));
const { stopped, ...result } = await rotateColumn(client, col, shouldStop);
results.push(result);
if (stopped) {
stoppedEarly = true;
break;
}
}
let totalScanned = 0;
let totalRotated = 0;
@@ -516,5 +551,6 @@ export async function rotateCorpus(
totalRotated,
totalErrors,
totalDropped,
stoppedEarly,
};
}
src/lib/doctor-report/__tests__/leaf-gating-sweep.test.ts
+1
View File
@@ -49,6 +49,7 @@ const { calls, rows, prismaMock } = vi.hoisted(() => {
return Promise.resolve(rows[name] ?? []);
},
count: () => Promise.resolve(0),
groupBy: () => Promise.resolve([]),
findUnique: () => {
calls.push(name);
return Promise.resolve(rows[`${name}Unique`]?.[0] ?? null);
src/lib/doctor-report/collect.ts
+46 -18
View File
@@ -77,8 +77,21 @@ import {
const DENSE_REPORT_RAW_WINDOW_DAYS = 90;
/**
* #1023 — inside the raw window, pulse takes the per-day path anyway once the
* window holds more readings than this.
* The types a device writes at sampling rate whose report sections are
* statistics and a series and nothing else: the figures the day path keeps
* exactly (count, mean, minimum, maximum, latest) plus one point per local
* day. Glucose is dense too but has its own rule below, because its clinical
* panel needs the per-reading counts the bucket query carries.
*/
const DENSE_STAT_TYPES = [
"PULSE",
"HEART_RATE_VARIABILITY",
"OXYGEN_SATURATION",
] as const satisfies readonly MeasurementType[];
/**
* #1023 — inside the raw window, a dense statistics type takes the per-day
* path anyway once the window holds more of its readings than this.
*
* A watch that records heart rate once a minute puts 130 000 readings in a
* 90-day report. Read raw, every one of them became an object, went through
@@ -170,22 +183,38 @@ export async function collectDoctorReportData(
const lastDay = new Date(`${userDayKey(end, reportTz)}T00:00:00.000Z`);
const aggregateDenseTypes = days > DENSE_REPORT_RAW_WINDOW_DAYS;
const densePulse =
!excluded.includes("PULSE") &&
(aggregateDenseTypes ||
(await prisma.measurement.count({
where: {
userId,
type: "PULSE",
measuredAt: { gte: start, lte: end },
deletedAt: null,
},
})) > DENSE_REPORT_RAW_ROW_CAP);
const includedDenseStatTypes = DENSE_STAT_TYPES.filter(
(type) => !excluded.includes(type),
);
// Inside the raw window, one grouped count decides each type on its own:
// a watch's minute-by-minute heart rate folds while a few blood-oxygen
// readings a night beside it stay raw.
const denseStatCounts =
aggregateDenseTypes || includedDenseStatTypes.length === 0
? null
: new Map(
(
await prisma.measurement.groupBy({
by: ["type"],
where: {
userId,
type: { in: [...includedDenseStatTypes] },
measuredAt: { gte: start, lte: end },
deletedAt: null,
},
_count: { _all: true },
})
).map((row) => [row.type, row._count._all]),
);
const denseTypes: MeasurementType[] = includedDenseStatTypes.filter(
(type) =>
denseStatCounts === null ||
(denseStatCounts.get(type) ?? 0) > DENSE_REPORT_RAW_ROW_CAP,
);
const denseGlucose =
aggregateDenseTypes && !excluded.includes("BLOOD_GLUCOSE");
const rawExcluded = [...excluded];
if (densePulse) rawExcluded.push("PULSE");
if (denseGlucose) rawExcluded.push("BLOOD_GLUCOSE");
if (denseGlucose) denseTypes.push("BLOOD_GLUCOSE");
const rawExcluded = [...excluded, ...denseTypes];
const wantsMedications = MEDICATION_LEAVES.some((leaf) => gate.admits(leaf));
const wantsIntakeEvents =
@@ -318,8 +347,7 @@ export async function collectDoctorReportData(
start,
end,
reportTz,
densePulse,
denseGlucose,
denseTypes,
}),
]);
src/lib/doctor-report/dense-buckets.ts
+15 -16
View File
@@ -1,16 +1,17 @@
/**
* The per-local-day pre-aggregation for the two sample-dense measurement
* types.
* The per-local-day pre-aggregation for the sample-dense measurement types.
*
* On a long window a CGM or a per-sample heart-rate account produces six
* figures of rows, so pulse and glucose collapse to one row per local
* day/source/device/context in SQL and the canonical-source picker then works
* over that bounded intermediate set. Every other type keeps the raw-row path.
* figures of rows, so the dense types (glucose, pulse, heart-rate variability,
* blood oxygen) collapse to one row per local day/source/device/context in
* SQL and the canonical-source picker then works over that bounded
* intermediate set. Every other type keeps the raw-row path.
*
* Extracted from the aggregator with the selection rework: the query is a job
* of its own, it is the one raw-SQL surface on this path, and keeping it in a
* thousand-line function is how a parameter stops being reviewed.
*/
import { Prisma } from "@/generated/prisma/client";
import { prisma } from "@/lib/db";
import type {
GlucoseContext,
@@ -46,8 +47,8 @@ export interface DenseMeasurementBucket {
}
/**
* Load the dense buckets for whichever of the two types the caller wants. Both
* flags false is not a call: the caller skips it entirely.
* Load the dense buckets for the types the caller names. An empty list is not
* a call: nothing is read.
*
* Every value is bound as a tagged-template parameter — no splice, no
* interpolation of anything that came off the wire.
@@ -57,11 +58,13 @@ export async function loadDenseMeasurementBuckets(params: {
start: Date;
end: Date;
reportTz: string;
densePulse: boolean;
denseGlucose: boolean;
denseTypes: readonly MeasurementType[];
}): Promise<DenseMeasurementBucket[]> {
const { userId, start, end, reportTz, densePulse, denseGlucose } = params;
if (!densePulse && !denseGlucose) return [];
const { userId, start, end, reportTz, denseTypes } = params;
if (denseTypes.length === 0) return [];
const typeList = Prisma.join(
denseTypes.map((type) => Prisma.sql`${type}::"measurement_type"`),
);
return prisma.$queryRaw<DenseMeasurementBucket[]>`
WITH localized AS (
SELECT
@@ -81,11 +84,7 @@ export async function loadDenseMeasurementBuckets(params: {
AND m."measured_at" >= ${start}
AND m."measured_at" <= ${end}
AND m."deleted_at" IS NULL
AND (
(${densePulse} AND m."type" = 'PULSE'::"measurement_type")
OR
(${denseGlucose} AND m."type" = 'BLOOD_GLUCOSE'::"measurement_type")
)
AND m."type" IN (${typeList})
),
scored AS (
SELECT
src/lib/jobs/__tests__/insight-pregenerate.test.ts
+26 -1
View File
@@ -207,6 +207,31 @@ describe("findPregenerateCandidates", () => {
});
});
describe("runInsightPregenerate — the job's time budget", () => {
it("stops between candidates once the budget is spent and says so", async () => {
capabilities = {
briefing: unavailable("user_disabled"),
statusText: unavailable("user_disabled"),
};
const { prisma } = makePrisma([
{ id: "u1", locale: "de" },
{ id: "u2", locale: "de" },
{ id: "u3", locale: "de" },
]);
let checks = 0;
const result = await runInsightPregenerate(prisma as never, {
generate: vi.fn(),
statusGenerators: [],
warmGenericMetrics: vi.fn().mockResolvedValue(0),
// Spent after the first candidate.
shouldStop: () => checks++ >= 1,
});
expect(result).toMatchObject({ total: 3, skipped: 1, stoppedEarly: true });
expect(aiCapabilityForJob).toHaveBeenCalledTimes(2);
});
});
describe("runInsightPregenerate — per-user capabilities", () => {
it("skips a user with neither capability before any budget write or generation", async () => {
capabilities = {
@@ -1337,7 +1362,7 @@ describe("queue registration", () => {
it("schedules the cron in the schedules table (with retry policy)", () => {
expect(workerSrc).toMatch(
/\[\s*INSIGHT_PREGENERATE_QUEUE\s*,\s*INSIGHT_PREGENERATE_CRON\s*,\s*insightRetryOptions\s*\]/,
/\[\s*INSIGHT_PREGENERATE_QUEUE\s*,\s*INSIGHT_PREGENERATE_CRON\s*,\s*\{\s*\.\.\.insightRetryOptions\s*,\s*expireInSeconds:\s*INSIGHT_PREGENERATE_EXPIRE_SECONDS\s*,?\s*\}\s*,?\s*\]/,
);
});
src/lib/jobs/__tests__/long-pass.test.ts
+74
View File
@@ -0,0 +1,74 @@
/**
* `lockedPass`: each job runs under the lock `job:<queue>:<identity>`, and a
* delivery that finds the lock held does no work and succeeds.
*
* The lock itself (a session advisory lock on a connection of its own) is
* exercised against Postgres in
* `tests/integration/integration-backfill-lane-expiry.test.ts`; this pins the
* wrapper around it.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { Job } from "pg-boss";
const held = vi.hoisted(() => new Set<string>());
const keys = vi.hoisted(() => [] as string[]);
vi.mock("@/lib/jobs/job-lock", () => ({
withJobLock: async <T>(key: string, run: () => Promise<T>) => {
keys.push(key);
if (held.has(key)) return { ran: false };
held.add(key);
try {
return { ran: true, result: await run() };
} finally {
held.delete(key);
}
},
}));
import { jobDone, jobFailed } from "@/lib/jobs/job-outcome";
import { lockedPass, WHOLE_PASS } from "@/lib/jobs/long-pass";
type Payload = { userId?: string };
const job = (data: Payload) => ({ id: "j", data }) as Job<Payload>;
beforeEach(() => {
held.clear();
keys.length = 0;
});
describe("lockedPass", () => {
it("runs each job under its queue and identity", async () => {
const handler = vi.fn(async () => jobDone({ generated: 1 }));
const bound = lockedPass(
"q",
(j: Job<Payload>) =>
j.data.userId ? `user:${j.data.userId}` : WHOLE_PASS,
handler,
);
const outcome = await bound([job({ userId: "a" })]);
expect(outcome).toEqual(jobDone({ generated: 1 }));
await bound([job({})]);
expect(keys).toEqual(["job:q:user:a", "job:q:pass"]);
});
it("does nothing beside a run that still holds the lock", async () => {
held.add("job:q:pass");
const handler = vi.fn(async () => jobDone());
const outcome = await lockedPass("q", () => WHOLE_PASS, handler)([job({})]);
expect(handler).not.toHaveBeenCalled();
expect(outcome).toEqual(jobDone({ already_running: 1 }));
});
it("passes a failed outcome through", async () => {
const failed = jobFailed("stopped early", undefined, {
stopped_early: true,
});
const outcome = await lockedPass(
"q",
() => WHOLE_PASS,
async () => failed,
)([job({})]);
expect(outcome).toBe(failed);
});
});
src/lib/jobs/__tests__/period-narrative-warm.test.ts
+1 -1
View File
@@ -250,7 +250,7 @@ describe("queue registration", () => {
it("schedules the cron in the schedules table (with retry policy)", () => {
expect(workerSrc).toMatch(
/\[\s*PERIOD_NARRATIVE_QUEUE\s*,\s*PERIOD_NARRATIVE_CRON\s*,\s*insightRetryOptions\s*\]/,
/\[\s*PERIOD_NARRATIVE_QUEUE\s*,\s*PERIOD_NARRATIVE_CRON\s*,\s*\{\s*\.\.\.insightRetryOptions\s*,\s*expireInSeconds:\s*PERIOD_NARRATIVE_EXPIRE_SECONDS\s*,?\s*\}\s*,?\s*\]/,
);
});
src/lib/jobs/__tests__/status-cron-candidates.test.ts
+2 -2
View File
@@ -190,9 +190,9 @@ describe("mood-status queue registration (dead-queue guard)", () => {
expect(match![1]).toMatch(/\bMOOD_STATUS_QUEUE\b/);
});
it("schedules the 02:30 cron with the insight retry policy", () => {
it("schedules the 02:30 cron with the nightly pass options (retries and expiry)", () => {
expect(workerSrc).toMatch(
/\[\s*MOOD_STATUS_QUEUE\s*,\s*MOOD_STATUS_CRON\s*,\s*insightRetryOptions\s*\]/,
/\[\s*MOOD_STATUS_QUEUE\s*,\s*MOOD_STATUS_CRON\s*,\s*nightlyInsightPassOptions\s*\]/,
);
expect(workerSrc).toMatch(/MOOD_STATUS_CRON\s*=\s*"30 2 \* \* \*"/);
});
src/lib/jobs/backup-restore.ts
+6 -1
View File
@@ -811,7 +811,12 @@ export async function enqueueBackupRestoreSweepAtBoot(
await boss.send(
BACKUP_RESTORE_QUEUE,
{ sweep: true } satisfies BackupRestorePayload,
{ startAfter: Math.ceil(BACKUP_RESTORE_STALE_AFTER_MS / 1000) + 15 },
{
startAfter: Math.ceil(BACKUP_RESTORE_STALE_AFTER_MS / 1000) + 15,
// The sweep is short, but it rides a queue whose every send carries
// the restore's expiry, so no job on it is cut at the default.
expireInSeconds: BACKUP_RESTORE_EXPIRE_SECONDS,
},
);
} catch {
// The admission check fails an abandoned job the next time the account
src/lib/jobs/document-content-index-backfill.ts
+40 -2
View File
@@ -64,6 +64,14 @@ export const CONTENT_INDEX_BACKFILL_CONCURRENCY = 1;
/** Discovery page size (id-only; blobs are loaded one at a time). */
const PAGE_SIZE = 50;
/**
* The job's expiry: three hours. One job transcribes up to 200 documents with
* a provider call each, which outlasted pg-boss's fifteen-minute default and
* was retried beside itself. The run stops at three quarters of this
* (`jobBudget`) and holds a lock per account (`lockedPass`).
*/
export const CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS = 3 * 60 * 60;
/** Max documents one job indexes before yielding (bounds spend + runtime). */
const MAX_DOCS_PER_RUN = 200;
@@ -97,6 +105,11 @@ export interface ContentIndexBackfillSummary {
failed: number;
reason:
"ok" | "no-provider" | "no-consent" | "budget-reached" | "unavailable";
/**
* The job's time budget ran out before the walk did. The documents not
* reached stay un-indexed for the next run.
*/
stoppedEarly?: boolean;
}
/**
@@ -158,6 +171,11 @@ async function retokeniseStaleRows(
*/
export async function runContentIndexBackfillForUser(
userId: string,
/**
* The job's time budget (`jobBudget`), checked before each document: up to
* 200 transcriptions, one provider call each, can outlast the job.
*/
shouldStop: () => boolean = () => false,
): Promise<ContentIndexBackfillSummary> {
// Stale-tokeniser rows first: local work that owes nothing to the
// provider, consent, or budget gates below, so a user without a usable
@@ -222,6 +240,7 @@ export async function runContentIndexBackfillForUser(
let failed = stale.failed;
let reason: ContentIndexBackfillSummary["reason"] = "ok";
let cursor: string | null = null;
let stoppedEarly = false;
outer: for (;;) {
if (indexed >= MAX_DOCS_PER_RUN) break;
@@ -244,6 +263,10 @@ export async function runContentIndexBackfillForUser(
for (const { id } of rows) {
if (indexed >= MAX_DOCS_PER_RUN) break outer;
if (shouldStop()) {
stoppedEarly = true;
break outer;
}
const dateKey = buildDateKey();
const reservation = await reserveBudget(
@@ -325,9 +348,23 @@ export async function runContentIndexBackfillForUser(
annotate({
action: { name: "documents.contentIndex.backfill" },
meta: { indexed, retokenised: stale.retokenised, skipped, failed, reason },
meta: {
indexed,
retokenised: stale.retokenised,
skipped,
failed,
reason,
stopped_early: stoppedEarly,
},
});
return { indexed, retokenised: stale.retokenised, skipped, failed, reason };
return {
indexed,
retokenised: stale.retokenised,
skipped,
failed,
reason,
...(stoppedEarly ? { stoppedEarly } : {}),
};
}
/**
@@ -344,6 +381,7 @@ export async function enqueueContentIndexBackfill(
enqueuedAt: new Date().toISOString(),
};
const jobId = await boss.send(CONTENT_INDEX_BACKFILL_QUEUE, payload, {
expireInSeconds: CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS,
retryLimit: 3,
retryDelay: 60,
retryBackoff: true,
src/lib/jobs/encryption-key-rotate.ts
+20 -3
View File
@@ -25,6 +25,7 @@
import { type Job } from "pg-boss";
import { withBackgroundEvent } from "@/lib/logging/background";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { jobBudget } from "@/lib/jobs/job-budget";
import { auditLog } from "@/lib/auth/audit";
import {
rotateCorpus,
@@ -37,27 +38,40 @@ export const ENCRYPTION_KEY_ROTATE_CONCURRENCY = 1;
/** Fixed key so duplicate admin triggers coalesce into one queued run. */
export const ENCRYPTION_KEY_ROTATE_SINGLETON = "encryption-key-rotate";
/**
* The job's expiry: twelve hours. The pass re-encrypts every encrypted column
* on the instance and outlasted pg-boss's fifteen-minute default, which
* retried it beside itself. The pass stops at three quarters of this
* (`jobBudget`) and holds a lock (`lockedPass`); the admin route's send
* carries it.
*/
export const ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS = 12 * 60 * 60;
export interface EncryptionKeyRotatePayload {
/** The admin who triggered the run (for the audit trail). */
requestedByUserId?: string;
enqueuedAt?: string;
}
export async function runEncryptionKeyRotation(): Promise<{
export async function runEncryptionKeyRotation(
shouldStop: () => boolean = () => false,
): Promise<{
activeKeyId: string;
totalScanned: number;
totalRotated: number;
totalErrors: number;
totalDropped: number;
stoppedEarly: boolean;
}> {
const prisma = getWorkerPrisma();
const out = await rotateCorpus(prisma as unknown as CorpusClient);
const out = await rotateCorpus(prisma as unknown as CorpusClient, shouldStop);
return {
activeKeyId: out.activeKeyId,
totalScanned: out.totalScanned,
totalRotated: out.totalRotated,
totalErrors: out.totalErrors,
totalDropped: out.totalDropped,
stoppedEarly: out.stoppedEarly,
};
}
@@ -67,7 +81,10 @@ export async function handleEncryptionKeyRotate(
return withBackgroundEvent("job.encryption_key_rotate", async (evt) => {
const requestedBy = jobs[0]?.data?.requestedByUserId ?? null;
try {
const result = await runEncryptionKeyRotation();
// A whole-corpus pass: it stops between rows once the job's budget is
// spent, and says so in the audit row, so the operator runs it again
// to finish (rows already rotated are skipped).
const result = await runEncryptionKeyRotation(jobBudget(jobs));
evt.addMeta("rotate_active_key_id", result.activeKeyId);
evt.addMeta("rotate_scanned", result.totalScanned);
evt.addMeta("rotate_rotated", result.totalRotated);
src/lib/jobs/insight-pregenerate-shared.ts
+11
View File
@@ -19,6 +19,15 @@ import { annotate } from "@/lib/logging/context";
export const INSIGHT_PREGENERATE_QUEUE = "insight-pregenerate";
/**
* The job's expiry: three hours. The scheduled pass walks up to 200 accounts
* with a comprehensive generation each, which can outlast pg-boss's fifteen
* minute default; past it pg-boss retried the job beside the pass still
* running. The pass now stops itself at three quarters of this (`jobBudget`)
* and holds a lock (`lockedPass`), and every send and the schedule carry it.
*/
export const INSIGHT_PREGENERATE_EXPIRE_SECONDS = 3 * 60 * 60;
export interface InsightPregeneratePayload {
/** Set on the nightly scheduled tick (informational; the handler keys
* its batch behaviour off the ABSENCE of `userId`/`force`). */
@@ -73,6 +82,7 @@ export async function enqueueForceWarm(payload: {
} satisfies InsightPregeneratePayload,
{
singletonKey: `force:${payload.userId}`,
expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS,
// De-dupe within a 6-minute window. v1.16.8 — widened from 120 s:
// the client's revalidation poll runs up to ~250 s after a stale
// page-open, so a 2-minute singleton let the tail of one poll
@@ -148,6 +158,7 @@ export async function enqueuePregenerateFailureRetry(payload: {
// warm can never swallow the scheduled retry (and vice versa);
// one retry per user per window.
singletonKey: `retry:${payload.userId}`,
expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS,
singletonSeconds: 3600,
// A worker crash mid-warm re-runs the job; a warm that completes
// with a failed comprehensive does NOT throw, so this cannot loop.
src/lib/jobs/insight-pregenerate.ts
+14
View File
@@ -277,6 +277,8 @@ export interface PregenerateRunResult {
skipped: number;
failed: number;
budgetBlocked: number;
/** The job's time budget ran out before every candidate was reached. */
stoppedEarly: boolean;
/**
* Count of per-metric assessment caches written warm across the whole
* run. The comprehensive generator evicts the seven `*-status`
@@ -575,6 +577,12 @@ export async function runInsightPregenerate(
userId: string;
locale: SupportedLocale;
}) => Promise<void>;
/**
* The job's time budget (`jobBudget`). Checked between candidates: a pass
* of 200 provider calls can outlast its job, and the candidates it does
* not reach are the stalest again tomorrow, so they lead the next pass.
*/
shouldStop?: () => boolean;
} = {},
): Promise<PregenerateRunResult> {
const now = options.now ?? new Date();
@@ -606,6 +614,7 @@ export async function runInsightPregenerate(
skipped: 0,
failed: 0,
budgetBlocked: 0,
stoppedEarly: false,
assessmentsWarmed: 0,
metricAssessmentsWarmed: 0,
};
@@ -619,6 +628,10 @@ export async function runInsightPregenerate(
result.total = candidates.length;
for (const candidate of candidates) {
if (options.shouldStop?.()) {
result.stoppedEarly = true;
break;
}
// v1.18.11 P3 — one feature-cache scope per candidate, so the
// comprehensive briefing's bounded feature read is computed once and any
// sibling consumer in this candidate's warm reuses it. Scoped per user so
@@ -819,6 +832,7 @@ export async function runInsightPregenerate(
skipped: result.skipped,
failed: result.failed,
budget_blocked: result.budgetBlocked,
stopped_early: result.stoppedEarly,
assessments_warmed: result.assessmentsWarmed,
metric_assessments_warmed: result.metricAssessmentsWarmed,
},
src/lib/jobs/integration-backfill-lock.ts
+10 -30
View File
@@ -11,8 +11,9 @@
* started beside it in the same worker, so the lane's "one at a time" did not
* hold for the one case it exists for, an import too long for its expiry.
*
* So each run holds a session-level advisory lock keyed on the job's identity
* (kind, provider where there is one, account) for as long as it runs. A
* So each run holds a session-level advisory lock (`withJobLock`) keyed on the
* job's identity (kind, provider where there is one, account) for as long as
* it runs. A
* delivery that finds the lock taken does no work: the run that holds it is
* still importing, and it either stamps its completion marker or leaves the
* account for the next boot's discovery. The lock lives on a connection of its
@@ -24,47 +25,26 @@
* One extra connection per running import; the lane admits one import per
* process at a time, so that is one connection.
*/
import { Client } from "pg";
import { withJobLock, type GuardedRun } from "./job-lock";
/** The lock key for one admission job's identity. */
function integrationBackfillLockKey(identity: string): string {
return `integration-backfill:${identity}`;
}
/** What a guarded run came to. */
type GuardedRun<T> = { ran: true; result: T } | { ran: false };
/**
* Run `run` while holding the import lock for `identity`. Resolves
* `{ ran: false }` at once, without calling `run`, when another run holds it.
* Rejects exactly as `run` does otherwise.
*/
export async function withIntegrationBackfillLock<T>(
export function withIntegrationBackfillLock<T>(
identity: string,
run: () => Promise<T>,
connectionString: string | undefined = process.env.DATABASE_URL,
): Promise<GuardedRun<T>> {
const client = new Client({ connectionString, keepAlive: true });
// An idle client whose server goes away emits `error`; unhandled, that
// takes the whole worker down. The run itself notices the database is
// gone through its own queries.
client.on("error", () => {});
await client.connect();
try {
const key = integrationBackfillLockKey(identity);
const { rows } = await client.query<{ held: boolean }>(
"SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS held",
[key],
);
if (!rows[0]?.held) return { ran: false };
try {
return { ran: true, result: await run() };
} finally {
await client
.query("SELECT pg_advisory_unlock(hashtextextended($1, 0))", [key])
.catch(() => {});
}
} finally {
await client.end().catch(() => {});
}
return withJobLock(
integrationBackfillLockKey(identity),
run,
connectionString,
);
}
src/lib/jobs/job-lock.ts
+58
View File
@@ -0,0 +1,58 @@
/**
* A session-level advisory lock held for the length of one run, across
* processes.
*
* pg-boss holds a job's "one at a time" only while the job is inside its
* expiry. Once the expiry passes it fails the job, fires its abort signal and
* retries it, and it does not stop the handler: measured against pg-boss 12.34
* (`tests/integration/integration-backfill-lane-expiry.test.ts`), the retry of
* the same job started beside the run that was still going. No queue policy
* helps, because the policies constrain job rows and the expired row is no
* longer active. Only something the running handler holds can, so each long
* run holds an advisory lock keyed on its identity, and a delivery that finds
* the lock taken does no work.
*
* The lock lives on a connection of its own, outside the Prisma pool, because
* a session lock has to stay on one connection for the whole run and a pooled
* one is handed back between statements. Ending the connection releases the
* lock, and so does the process dying, so a crash never leaves a key locked.
* One extra connection per running locked job.
*/
import { Client } from "pg";
/** What a guarded run came to. */
export type GuardedRun<T> = { ran: true; result: T } | { ran: false };
/**
* Run `run` while holding the advisory lock for `key`. Resolves
* `{ ran: false }` at once, without calling `run`, when another run holds it.
* Rejects exactly as `run` does otherwise.
*/
export async function withJobLock<T>(
key: string,
run: () => Promise<T>,
connectionString: string | undefined = process.env.DATABASE_URL,
): Promise<GuardedRun<T>> {
const client = new Client({ connectionString, keepAlive: true });
// An idle client whose server goes away emits `error`; unhandled, that
// takes the whole worker down. The run itself notices the database is
// gone through its own queries.
client.on("error", () => {});
await client.connect();
try {
const { rows } = await client.query<{ held: boolean }>(
"SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS held",
[key],
);
if (!rows[0]?.held) return { ran: false };
try {
return { ran: true, result: await run() };
} finally {
await client
.query("SELECT pg_advisory_unlock(hashtextextended($1, 0))", [key])
.catch(() => {});
}
} finally {
await client.end().catch(() => {});
}
}
src/lib/jobs/long-pass.ts
+58
View File
@@ -0,0 +1,58 @@
/**
* The binding for a queue whose job can outlast pg-boss's default expiry.
*
* Three things together make a long job safe, and `queue-runtime.ts` names
* every queue that needs them:
*
* 1. An explicit `expireInSeconds` on every send and schedule, so pg-boss
* does not declare the job dead at fifteen minutes while it is working.
* 2. A time budget (`jobBudget` / `jobDeadline` in `job-budget.ts`) the
* handler checks between units of work, so the pass stops on its own
* before that expiry and the next run picks up what is left.
* 3. A lock the running handler holds (`withJobLock`), so the retry pg-boss
* starts once the expiry has passed does nothing beside a run that is
* still going. Queue policies cannot do this: they constrain job rows,
* and an expired job's row is no longer active.
*
* `lockedPass` is the third. It wraps a handler so each job runs under the
* lock `<queue>:<identity>`, where the identity is what two runs must not
* share: the account for a per-user job, a constant for a pass over everyone.
* A delivery that finds the lock held reports `already_running` and succeeds,
* because the run that holds it is doing the work.
*/
import type { Job } from "pg-boss";
import { withJobLock } from "@/lib/jobs/job-lock";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { annotate } from "@/lib/logging/context";
/** The identity of a pass that walks every account: one run at a time. */
export const WHOLE_PASS = "pass";
export function lockedPass<J extends Job<unknown>>(
queue: string,
identityOf: (job: J) => string,
handler: (jobs: J[]) => Promise<JobOutcome>,
): (jobs: J[]) => Promise<JobOutcome> {
return async (jobs) => {
let outcome: JobOutcome = jobDone({ jobs: 0 });
for (const job of jobs) {
const run = await withJobLock(`job:${queue}:${identityOf(job)}`, () =>
handler([job]),
);
if (!run.ran) {
annotate({
action: { name: "job.long_pass.already_running" },
meta: { queue },
});
outcome = jobDone({ already_running: 1 });
continue;
}
// The first failed job fails the batch; pg-boss hands one job per
// batch here, so this is that job's own outcome.
if (!run.result.ok) return run.result;
outcome = run.result;
}
return outcome;
};
}
src/lib/jobs/medication-intake-import.ts
+38 -5
View File
@@ -4,7 +4,8 @@ import { Prisma, type PrismaClient } from "@/generated/prisma/client";
import { invalidateUserMedications } from "@/lib/cache/invalidate";
import { auditLog } from "@/lib/auth/audit";
import { prisma, toJson } from "@/lib/db";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { jobBudget } from "@/lib/jobs/job-budget";
import { consumeImportedIntakesBatch } from "@/lib/medications/inventory/consumption";
import { queueMedicationIntakeSync } from "@/lib/notifications/medication-intake-sync";
import {
@@ -825,13 +826,25 @@ async function processNextChunk(
);
}
export async function processMedicationIntakeImportJob(
/**
* Work the import chunk by chunk until it is finished. `shouldStop` is the
* job's time budget: between chunks, once it is spent, the run returns
* `"stopped"` and leaves the job row as it is. Every chunk commits its own
* progress under the row lock, so the retry that follows picks up at the next
* chunk, and a second delivery running beside this one waits on that lock and
* continues from the progress this one wrote.
*/
async function workMedicationIntakeImport(
jobId: string,
): Promise<MedicationImportResult | null> {
shouldStop: () => boolean,
): Promise<MedicationImportResult | null | "stopped"> {
const client = getWorkerPrisma();
for (;;) {
const outcome = await processNextChunk(client, jobId);
if (!outcome.terminal) continue;
if (!outcome.terminal) {
if (shouldStop()) return "stopped";
continue;
}
if (outcome.finalized && outcome.userId && outcome.result) {
if (outcome.result.imported > 0) {
invalidateUserMedications(outcome.userId, { evict: true });
@@ -843,6 +856,14 @@ export async function processMedicationIntakeImportJob(
}
}
/** The whole import in one go, with no time budget. */
export async function processMedicationIntakeImportJob(
jobId: string,
): Promise<MedicationImportResult | null> {
const result = await workMedicationIntakeImport(jobId, () => false);
return result === "stopped" ? null : result;
}
async function recordWorkerFailure(
job: Job<MedicationIntakeImportQueuePayload>,
error: unknown,
@@ -879,7 +900,19 @@ export async function handleMedicationIntakeImport(
job: Job<MedicationIntakeImportQueuePayload>,
): Promise<JobOutcome> {
try {
const result = await processMedicationIntakeImportJob(job.data.jobId);
const result = await workMedicationIntakeImport(
job.data.jobId,
jobBudget([job]),
);
// Out of time before the last chunk: fail the delivery so pg-boss
// retries it, and the retry resumes from the committed progress.
if (result === "stopped") {
return jobFailed(
"stopped before the last chunk; the retry resumes",
undefined,
{ stopped_early: true },
);
}
// A null result is a delivery that found the row already terminal or gone.
// There was nothing left to import, which is an absence of work rather
// than a failure to do it.
src/lib/jobs/offhost-backup.ts
+9
View File
@@ -42,6 +42,15 @@ const ALGORITHM = "aes-256-gcm";
const IV_LENGTH = 12;
const TAG_LENGTH = 16;
/**
* The off-host pass's expiry: four hours. It uploads every account and
* already stopped between accounts at three quarters of the default fifteen
* minutes, which on an instance of any size meant a retry per night; one
* large account past the default was retried beside itself. Four hours with
* the same budget and a lock (`lockedPass`) leaves room for the night's pass.
*/
export const OFFHOST_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60;
export interface OffhostBackupConfig {
endpoint: string;
bucket: string;
src/lib/jobs/period-narrative-shared.ts
+10
View File
@@ -23,6 +23,15 @@ import type { Locale } from "@/lib/i18n/config";
export const PERIOD_NARRATIVE_QUEUE = "period-narrative-warm";
/**
* The job's expiry: three hours. On a boundary night the scheduled pass
* writes up to two narratives for each of 200 accounts, which can outlast
* pg-boss's fifteen-minute default. The pass stops itself at three quarters
* of this (`jobBudget`) and holds a lock (`lockedPass`); every send and the
* schedule carry it.
*/
export const PERIOD_NARRATIVE_EXPIRE_SECONDS = 3 * 60 * 60;
/**
* Nightly at 05:05 Europe/Berlin — inside the existing maintenance window,
* after the comprehensive insight pre-generation (04:30) and the computed
@@ -79,6 +88,7 @@ export async function enqueueNarrativeWarm(payload: {
{
singletonKey: `warm:${payload.userId}:${payload.period}:${payload.locale}`,
singletonSeconds: 120,
expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS,
// Transient provider / pool failures retry with backoff instead
// of leaving the narrative cold until the next boundary night.
retryLimit: 3,
src/lib/jobs/period-narrative-warm.ts
+14
View File
@@ -63,6 +63,8 @@ export interface NarrativeWarmRunResult {
insufficient: number;
failed: number;
budgetBlocked: number;
/** The job's time budget ran out before every candidate was reached. */
stoppedEarly: boolean;
}
/**
@@ -152,6 +154,12 @@ export async function runPeriodNarrativeWarm(
cap?: number;
/** Injected for the test — defaults to the real generator. */
generate?: typeof generatePeriodNarrative;
/**
* The job's time budget (`jobBudget`), checked between candidates. The
* candidates are ordered least recently narrated first, so the ones a
* stopped pass did not reach lead the next one.
*/
shouldStop?: () => boolean;
} = {},
): Promise<NarrativeWarmRunResult> {
const now = options.now ?? new Date();
@@ -167,6 +175,7 @@ export async function runPeriodNarrativeWarm(
insufficient: 0,
failed: 0,
budgetBlocked: 0,
stoppedEarly: false,
};
const flags = await getAssistantFlags();
@@ -180,6 +189,10 @@ export async function runPeriodNarrativeWarm(
result.total = candidates.length;
for (const candidate of candidates) {
if (options.shouldStop?.()) {
result.stoppedEarly = true;
break;
}
// The nightly pass exists for the model-written narrative. Unavailable,
// it skips the user before the context build and the budget write; the
// deterministic narrative is written on the next read instead (the
@@ -233,6 +246,7 @@ export async function runPeriodNarrativeWarm(
insufficient: result.insufficient,
failed: result.failed,
budget_blocked: result.budgetBlocked,
stopped_early: result.stoppedEarly,
},
});
src/lib/jobs/pr-detection.ts
+11
View File
@@ -15,6 +15,16 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance";
export const PR_DETECTION_QUEUE = "pr-detection";
/**
* The job's expiry: twenty-five minutes. The fallback cron walks every
* account's whole history every thirty minutes, which on a large instance
* outlasts pg-boss's fifteen-minute default; past it pg-boss retried the pass
* beside itself. The pass stops at three quarters of this (`jobBudget`) and
* holds a lock (`lockedPass`), so it ends before the next tick and a retry or
* a late tick never runs beside it. The ingest sends carry the same expiry.
*/
export const PR_DETECTION_EXPIRE_SECONDS = 25 * 60;
/** Concurrency budget for the worker process — five jobs in flight
* is enough to drain a multi-user backfill spike without crowding
* the reminder check or the daily insights workload that runs on the
@@ -62,6 +72,7 @@ export async function enqueuePrDetection(
triggeredAt: new Date().toISOString(),
};
await boss.send(PR_DETECTION_QUEUE, payload, {
expireInSeconds: PR_DETECTION_EXPIRE_SECONDS,
// Stagger retries gently — the detector is purely read-heavy on
// the user's own data, but a stampede of failed jobs against a
// briefly-flaky DB would compound the problem.
src/lib/jobs/queue-runtime.ts
+398
View File
@@ -0,0 +1,398 @@
/**
* How long one job of each queue can run, and what keeps a long one safe.
*
* pg-boss gives every job an expiry, fifteen minutes unless the send says
* otherwise. Past it pg-boss fails the job, fires its abort signal and retries
* it, and it does not stop the handler, so a job that legitimately needs longer
* ends up running twice beside itself. No queue policy prevents that: the
* policies constrain job rows, and an expired row is no longer active. Every
* queue bound with `createAndWork` is therefore declared here as one of:
*
* - `short`: a job cannot come near fifteen minutes (one account's bounded
* window, one document, a forwarder, a bounded delete). It relies on the
* default expiry, and the reason says why that is enough.
* - `budgeted`: a pass over a long history that stops itself at three
* quarters of the default expiry (`jobBudget`) and hands the rest to its
* next run, so it ends before fifteen minutes by construction.
* - `long`: a job that may run for hours. It carries an explicit
* `expireInSeconds` on every send and schedule, stops itself before that
* expiry (`jobBudget` / `jobDeadline`), and holds something per identity
* that keeps a second run from working beside it: the lock `lockedPass`
* takes, a lock of its own, or a row it claims. The one exception is a
* one-shot job with `retryLimit: 0`, which pg-boss never retries and which
* cannot stop half-way without losing its input.
*
* `src/__tests__/job-queue-runtime-guard.test.ts` holds every binding, send
* and schedule in the tree to this table.
*/
import { INTEGRATION_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/integration-backfill-admission";
import { NIGHTLY_INSIGHT_PASS_EXPIRE_SECONDS } from "@/lib/jobs/reminder/registrar-shared";
import { INSIGHT_PREGENERATE_EXPIRE_SECONDS } from "@/lib/jobs/insight-pregenerate-shared";
import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "@/lib/jobs/period-narrative-shared";
import { PR_DETECTION_EXPIRE_SECONDS } from "@/lib/jobs/pr-detection";
import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key-rotate";
import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill";
import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy";
import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup";
import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore";
import { APPLE_HEALTH_IMPORT_SEND_OPTIONS } from "@/lib/jobs/apple-health-import-worker";
import { MEDICATION_INTAKE_IMPORT_SEND_OPTIONS } from "@/lib/jobs/medication-intake-import";
/** pg-boss's default job expiry, in seconds. */
export const DEFAULT_JOB_EXPIRE_SECONDS = 15 * 60;
/** Where a check lives: `binding` is the `createAndWork` call itself. */
export type SourceSite = "binding" | { file: string; fn: string };
export type QueueRuntime =
| { runtime: "short"; why: string }
| { runtime: "budgeted"; budget: SourceSite; why: string }
| {
runtime: "long";
expireInSeconds: number;
/**
* Identifiers that carry the expiry into a send or a schedule. A site
* that names none of them must set `expireInSeconds` itself.
*/
expiryVia: readonly string[];
/** Where the handler stops on the job's budget, or `oneShot`. */
stop: SourceSite | { oneShot: string };
/**
* What keeps a second run from working beside this one: `lockedPass`
* at the binding, or a marker in the named function.
*/
exclusive: "lockedPass" | { marker: string; at: SourceSite };
why: string;
};
const short = (why: string): QueueRuntime => ({ runtime: "short", why });
const nightlyStatusPass = (fn: string): QueueRuntime => ({
runtime: "long",
expireInSeconds: NIGHTLY_INSIGHT_PASS_EXPIRE_SECONDS,
expiryVia: ["nightlyInsightPassOptions"],
stop: { file: "lib/jobs/reminder/insights-handlers.ts", fn },
exclusive: "lockedPass",
why: "Nightly status pass: one provider call per candidate account, serially, over the whole instance.",
});
const cohortSync = (provider: string): QueueRuntime =>
short(
`${provider} sync: a recent window per connection (incremental since the last sync); a webhook job is one connection.`,
);
const oauthStateCleanup = short("One DELETE of expired OAuth state rows.");
const forwarder = short(
"Forwards each job to integration-backfill-admission; the import itself runs there.",
);
export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
// ── Integration sync ───────────────────────────────────────────────
"integration-backfill-admission": {
runtime: "long",
expireInSeconds: INTEGRATION_BACKFILL_EXPIRE_SECONDS,
expiryVia: ["integrationBackfillAdmissionSendOptions"],
stop: {
file: "lib/jobs/integration-backfill-drain.ts",
fn: "drainIntegrationBackfillAdmission",
},
exclusive: {
marker: "withIntegrationBackfillLock",
at: {
file: "lib/jobs/integration-backfill-drain.ts",
fn: "drainIntegrationBackfillAdmission",
},
},
why: "Full-history provider imports: thousands of pages on a dense account.",
},
"withings-fallback-sync": cohortSync("Withings"),
"withings-activity-sync": cohortSync("Withings activity"),
"withings-sleep-sync": cohortSync("Withings sleep"),
"withings-ecg-sync": cohortSync("Withings ECG"),
"withings-oauth-state-cleanup": oauthStateCleanup,
"whoop-recovery-sync": cohortSync("WHOOP recovery"),
"whoop-sleep-sync": cohortSync("WHOOP sleep"),
"whoop-workout-sync": cohortSync("WHOOP workout"),
"whoop-cycle-sync": cohortSync("WHOOP cycle"),
"whoop-backfill": forwarder,
"whoop-oauth-state-cleanup": oauthStateCleanup,
"oidc-native-handoff-cleanup": short("One DELETE of expired handoff rows."),
"fitbit-sync": cohortSync("Fitbit"),
"fitbit-backfill": forwarder,
"fitbit-sleep-repair": forwarder,
"fitbit-oauth-state-cleanup": oauthStateCleanup,
"google-health-sync": cohortSync("Google Health"),
"google-health-backfill": forwarder,
"google-health-sleep-repair": forwarder,
"google-health-oauth-state-cleanup": oauthStateCleanup,
"sleep-timeline-backfill": forwarder,
"lab-biomarker-backfill": forwarder,
"nightscout-sync": cohortSync("Nightscout"),
"polar-sync": cohortSync("Polar"),
"oura-sync": cohortSync("Oura"),
"strava-sync": cohortSync("Strava"),
"strava-backfill": forwarder,
// ── Status and insights ────────────────────────────────────────────
"insights-general-status": nightlyStatusPass("handleGeneralStatusGenerate"),
"insights-blood-pressure-status": nightlyStatusPass(
"handleBloodPressureStatusGenerate",
),
"insights-weight-status": nightlyStatusPass("handleWeightStatusGenerate"),
"insights-pulse-status": nightlyStatusPass("handlePulseStatusGenerate"),
"insights-bmi-status": nightlyStatusPass("handleBmiStatusGenerate"),
"insights-mood-status": nightlyStatusPass("handleMoodStatusGenerate"),
"insights-medication-compliance-status": nightlyStatusPass(
"handleMedicationComplianceStatusGenerate",
),
"insight-pregenerate": {
runtime: "long",
expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS,
expiryVia: ["INSIGHT_PREGENERATE_EXPIRE_SECONDS"],
stop: {
file: "lib/jobs/reminder/insights-handlers.ts",
fn: "handleInsightPregenerateJob",
},
exclusive: "lockedPass",
why: "Up to 200 accounts, a comprehensive generation each (a provider call bounded by the account's timeout, up to ten minutes) plus the status warm.",
},
"morning-digest-refresh": short(
"One account, one comprehensive generation, bounded by the provider timeout.",
),
"insight-status-generate": short(
"One account, one metric, one provider call.",
),
"data-arrival": short("One arrival's reactions."),
"reaction-line-generate": short("One provider call for one reaction line."),
"workout-insight-generate": short("One workout's insight."),
"recovery-score-compute": short(
"Arithmetic over a recent window, capped at 500 accounts; no provider call.",
),
"stress-score-compute": short(
"Arithmetic over a recent window, capped at 500 accounts; no provider call.",
),
"strain-score-compute": short(
"Arithmetic over a recent window, capped at 500 accounts; no provider call.",
),
"period-narrative-warm": {
runtime: "long",
expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS,
expiryVia: ["PERIOD_NARRATIVE_EXPIRE_SECONDS"],
stop: "binding",
exclusive: "lockedPass",
why: "On a boundary night up to 200 accounts with a provider call per period each; a single-account warm is short.",
},
"coach-memory-refresh": short("One conversation's summary and facts."),
"coach-nudge": short(
"Light deterministic checks per account; the provider part is capped at 25 calls and 90 seconds a tick.",
),
"coach-reminder-sweep": short("Batches of 200 to 500 rows, no provider."),
"coach-plan-review": short("A batch of 100 plans, folded in SQL."),
"medication-low-stock": short("A light inventory check per account."),
"daily-briefing": short("A light cohort scan every fifteen minutes."),
// ── Rollups ────────────────────────────────────────────────────────
"rollup-recompute": short("One bucket."),
"rollup-full-backfill": short(
"One account's rollup tiers: four aggregate statements, each held to the 60-second statement timeout.",
),
"mood-rollup-recompute": short("One bucket."),
"mood-rollup-full-backfill": short(
"One account's mood entries, a few a day.",
),
"medication-compliance-full-backfill": short(
"One account's 90-day compliance window.",
),
"step-consolidation": short(
"One account's legacy step rows, one transaction per day.",
),
"step-consolidation-repair": short(
"One account's tombstoned step rows inside the retention window.",
),
"cumulative-pr-rederive": short(
"One account's record detection; measured at 20 seconds for 1.25 million rows.",
),
"mean-consolidation": {
runtime: "budgeted",
budget: "binding",
why: "One account's whole history, a day at a time; stops at three quarters of the default expiry and resumes.",
},
"dense-intraday-retention": {
runtime: "budgeted",
budget: "binding",
why: "A dense account's history, a day at a time; stops at three quarters of the default expiry and resumes.",
},
"dense-intraday-hourly-rebuild": {
runtime: "budgeted",
budget: "binding",
why: "A dense account's hourly rows, a day at a time; stops at three quarters of the default expiry and resumes.",
},
"drain-per-sample-cumulative": {
runtime: "budgeted",
budget: "binding",
why: "Every account's per-sample drains; stops at three quarters of the default expiry and hands the rest to a continuation job.",
},
// ── Reminders ──────────────────────────────────────────────────────
"medication-reminder-check": short("The due slots of the current minute."),
"mood-reminder-check": short("Accounts whose local reminder hour is now."),
"cycle-reminder-check": short("Accounts whose local reminder hour is now."),
"measurement-reminder-check": short(
"Accounts whose local reminder hour is now.",
),
"reminder-satisfy": short("One account's open reminders."),
// ── Maintenance ────────────────────────────────────────────────────
"data-backup": {
runtime: "long",
expireInSeconds: DATA_BACKUP_SEND_OPTIONS.expireInSeconds,
expiryVia: ["DATA_BACKUP_SEND_OPTIONS"],
stop: {
file: "lib/jobs/reminder/backup-handlers.ts",
fn: "handleDataBackup",
},
exclusive: "lockedPass",
why: "Serialises every account's whole record.",
},
"data-backup-offhost": {
runtime: "long",
expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS,
expiryVia: ["OFFHOST_BACKUP_EXPIRE_SECONDS"],
stop: {
file: "lib/jobs/reminder/backup-handlers.ts",
fn: "handleOffhostBackup",
},
exclusive: "lockedPass",
why: "Uploads every account's record off-host.",
},
"data-restore-drill": short("Fetches and stream-parses one off-host object."),
"offhost-backup-purge": short("At most 200 delete requests per run."),
"backup-restore": {
runtime: "long",
expireInSeconds: BACKUP_RESTORE_EXPIRE_SECONDS,
expiryVia: ["BACKUP_RESTORE_SEND_OPTIONS", "BACKUP_RESTORE_EXPIRE_SECONDS"],
stop: { file: "lib/jobs/backup-restore.ts", fn: "handleBackupRestore" },
exclusive: {
marker: "restore_claimed",
at: { file: "lib/jobs/backup-restore.ts", fn: "runBackupRestoreJob" },
},
why: "Restores a whole account in one transaction.",
},
"host-metric-sample": short("One host sample."),
"feedback-aggregator": short("One aggregate over the feedback table."),
"geo-backfill": short("At most 500 audit rows per pass."),
"geolite2-fetch": short("One database download."),
"tls-pin-monitor": short("One TLS handshake."),
"mood-reminder-cleanup": short("One retention DELETE."),
"push-attempt-cleanup": short("One retention DELETE."),
"arrival-reaction-cleanup": short("One retention DELETE."),
"rate-limit-cleanup": short("One retention DELETE."),
"idempotency-cleanup": short("One retention DELETE."),
"step-up-elevation-cleanup": short("One retention DELETE."),
"coach-message-cleanup": short("One retention DELETE."),
"mcp-token-cleanup": short("One retention DELETE."),
"audit-log-cleanup": short("At most 40 batches of 5 000 rows per leg."),
"measurement-tombstone-cleanup": short(
"At most 40 batches of 5 000 rows per leg.",
),
"document-tombstone-purge": short("One deleteMany."),
"document-summary-reaper": short("One updateMany."),
"cycle-prediction-refresh": short(
"Arithmetic per eligible account, no provider call.",
),
"mood-prognosis-refresh": short(
"Arithmetic per eligible account, no provider call.",
),
"achievement-unlock-sweep": short(
"A bounded achievement read per account, no provider call.",
),
"pr-detection": {
runtime: "long",
expireInSeconds: PR_DETECTION_EXPIRE_SECONDS,
expiryVia: ["PR_DETECTION_EXPIRE_SECONDS"],
stop: {
file: "lib/jobs/reminder/ops-handlers.ts",
fn: "handlePrDetection",
},
exclusive: "lockedPass",
why: "The fallback cron scans every account's all-time history every thirty minutes; an ingest job is one account.",
},
"medication-inventory-expire": short("One updateMany."),
"intake-auto-skip": short("One updateMany."),
"apple-health-import-v2": {
runtime: "long",
expireInSeconds: APPLE_HEALTH_IMPORT_SEND_OPTIONS.expireInSeconds,
expiryVia: ["APPLE_HEALTH_IMPORT_SEND_OPTIONS"],
stop: {
oneShot:
"The run consumes and unlinks its staged upload, so it can neither stop half-way nor be retried; the send sets retryLimit 0.",
},
exclusive: {
marker: "retryLimit: 0",
at: {
file: "lib/jobs/apple-health-import-worker.ts",
fn: "APPLE_HEALTH_IMPORT_SEND_OPTIONS",
},
},
why: "Parses an export that can be several gigabytes.",
},
"apple-health-import": short(
"The legacy queue: moves each job onto apple-health-import-v2.",
),
"apple-health-import-reconcile": short("One updateMany."),
"medication-intake-import": {
runtime: "long",
expireInSeconds: MEDICATION_INTAKE_IMPORT_SEND_OPTIONS.expireInSeconds,
expiryVia: ["MEDICATION_INTAKE_IMPORT_SEND_OPTIONS"],
stop: {
file: "lib/jobs/medication-intake-import.ts",
fn: "handleMedicationIntakeImport",
},
exclusive: {
marker: "FOR UPDATE",
at: {
file: "lib/jobs/medication-intake-import.ts",
fn: "processNextChunk",
},
},
why: "Imports a dose-history file chunk by chunk; its length follows the file.",
},
"intake-slot-dedup": short("One account's duplicate dose slots."),
"note-encryption-backfill": short(
"Pages of 200 rows of one account's plaintext notes, light per row.",
),
"med-notes-encryption-backfill": short(
"Pages of 200 rows of one account's plaintext notes, light per row.",
),
"free-text-encryption-backfill": short(
"Pages of 200 rows of one account's plaintext fields, light per row.",
),
"encryption-key-rotate": {
runtime: "long",
expireInSeconds: ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS,
expiryVia: ["ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS"],
stop: {
file: "lib/jobs/encryption-key-rotate.ts",
fn: "handleEncryptionKeyRotate",
},
exclusive: "lockedPass",
why: "Re-encrypts every encrypted column on the instance.",
},
"document-content-index-backfill": {
runtime: "long",
expireInSeconds: CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS,
expiryVia: ["CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS"],
stop: "binding",
exclusive: "lockedPass",
why: "Up to 200 documents of one account, a provider transcription each.",
},
"document-index": short("One document."),
"document-thumbnail": short("One document."),
"document-summary": short("One document."),
"document-summary-catchup": short("Enqueues at most 200 documents."),
"document-thumbnail-backfill": short("Enqueues at most 1 000 documents."),
"environment-fetch": short(
"A fan-out of sends, or one account's bounded archive range.",
),
};
src/lib/jobs/reminder/__tests__/backup-handlers.test.ts
+42
View File
@@ -70,6 +70,7 @@ function buildPrismaMock(
) {
return {
user: { findMany: vi.fn().mockResolvedValue(users) },
dataBackup: { findMany: vi.fn().mockResolvedValue([]) },
};
}
@@ -236,3 +237,44 @@ describe("handleDataBackup outcome", () => {
expect(outcome).toMatchObject({ ok: true, did: { backed: 0, total: 0 } });
});
});
describe("handleDataBackup on a large instance", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.store.mockResolvedValue(undefined);
});
it("backs up the accounts with the oldest copies first and stops on the budget", async () => {
const prisma = buildPrismaMock([
{ id: "fresh", username: "fresh" },
{ id: "never", username: "never" },
{ id: "stale", username: "stale" },
]);
prisma.dataBackup.findMany.mockResolvedValue([
{ userId: "fresh", createdAt: new Date("2026-09-30T00:00:00Z") },
{ userId: "stale", createdAt: new Date("2026-09-01T00:00:00Z") },
]);
mocks.getWorkerPrisma.mockReturnValue(prisma);
const controller = new AbortController();
// Two accounts in, pg-boss gives up on the job.
mocks.store.mockImplementation(async () => {
if (mocks.store.mock.calls.length === 2) controller.abort();
});
const outcome = await handleDataBackup([
{
expireInSeconds: 7200,
signal: controller.signal,
} as unknown as Parameters<typeof handleDataBackup>[0][number],
]);
const order = mocks.store.mock.calls.map(
([input]) => (input as { userId: string }).userId,
);
expect(order).toEqual(["never", "stale"]);
expect(outcome).toMatchObject({
ok: true,
did: { backed: 2, stopped_early: true },
});
});
});
src/lib/jobs/reminder/backup-handlers.ts
+32 -4
View File
@@ -121,16 +121,39 @@ export async function handleOffhostBackup(
});
}
/**
* The weekly on-host copy of every account.
*
* The pass serialises each account's whole record, so on a large instance it
* can outlast its job. It walks the accounts whose weekly copy is oldest
* first and stops between accounts once the job's budget is spent
* (`jobBudget`), reporting `stopped_early`. The accounts it did not reach have
* the oldest copies, so they lead the next pass (the weekly cron or the
* admin's "run now"), and every pass makes progress where it matters most.
* The binding holds a lock (`lockedPass`), so a retry never runs beside it.
*/
export async function handleDataBackup(
jobs: Job<DataBackupPayload>[],
): Promise<JobOutcome> {
void jobs;
const shouldStop = jobBudget(jobs);
return withBackgroundEvent("job.data_backup", async (evt) => {
const prisma = getWorkerPrisma();
try {
const users = await prisma.user.findMany({
select: { id: true, username: true },
});
const [accounts, copies] = await Promise.all([
prisma.user.findMany({ select: { id: true, username: true } }),
prisma.dataBackup.findMany({
where: { type: "WEEKLY_AUTO" },
select: { userId: true, createdAt: true },
}),
]);
const copiedAt = new Map(
copies.map((copy) => [copy.userId, copy.createdAt.getTime()]),
);
// No copy yet sorts first, then the oldest copy.
const users = [...accounts].sort(
(a, b) => (copiedAt.get(a.id) ?? -1) - (copiedAt.get(b.id) ?? -1),
);
let stoppedEarly = false;
let backed = 0;
let usersFailed = 0;
@@ -142,6 +165,10 @@ export async function handleDataBackup(
// six weeks in.
let lastError: unknown;
for (const user of users) {
if (shouldStop()) {
stoppedEarly = true;
break;
}
reportJobProgress({
backup_user: user.id,
backup_users_done: backed + usersFailed + skippedRestoring,
@@ -207,6 +234,7 @@ export async function handleDataBackup(
users_failed: usersFailed,
records_oversized: oversized,
users_skipped_restoring: skippedRestoring,
stopped_early: stoppedEarly,
};
// A pass that wrote nothing for anybody protected nobody, and saying
src/lib/jobs/reminder/insights-handlers.ts
+71 -17
View File
@@ -21,6 +21,7 @@ import {
runInsightStatusGenerate,
} from "@/lib/jobs/insight-status-generate";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { jobBudget } from "@/lib/jobs/job-budget";
import { withBackgroundEvent } from "@/lib/logging/background";
import { generateBloodPressureStatusForUser } from "@/lib/insights/blood-pressure-status";
import { generateWeightStatusForUser } from "@/lib/insights/weight-status";
@@ -77,6 +78,12 @@ export interface MedicationComplianceStatusPayload {
* The pass is what succeeds or fails. One user whose generation threw is that
* user's cold card, reported as the `failed` count; failing the job would
* re-run the whole cohort over it.
*
* The pass walks every candidate with a provider call each, so on a large
* instance it can outlast its job. `shouldStop` is the job's time budget: the
* pass stops between users once it is spent and reports `stopped_early`. The
* users it did not reach keep yesterday's note until the next night or their
* next visit; nothing is retried beside a pass that is still running.
*/
export async function runStatusCronGenerate(
taskName: string,
@@ -84,6 +91,7 @@ export async function runStatusCronGenerate(
userId: string,
options: { locale: string | null; force: boolean },
) => Promise<unknown>,
shouldStop: () => boolean = () => false,
): Promise<JobOutcome> {
return withBackgroundEvent(taskName, async (evt) => {
const prisma = getWorkerPrisma();
@@ -96,8 +104,13 @@ export async function runStatusCronGenerate(
let generated = 0;
let failed = 0;
let stoppedEarly = false;
for (const user of users) {
if (shouldStop()) {
stoppedEarly = true;
break;
}
// The capability before the generator builds a snapshot. The batch
// entry below checks for itself.
const capability = await aiCapabilityForJob(user.id, "statusText");
@@ -124,9 +137,14 @@ export async function runStatusCronGenerate(
evt.setBackground({
task_name: taskName,
result: { generated, failed, total: users.length },
result: { generated, failed, total: users.length, stoppedEarly },
});
return jobDone({
total: users.length,
generated,
failed,
stopped_early: stoppedEarly,
});
return jobDone({ total: users.length, generated, failed });
} catch (err) {
evt.setError(err);
recordError();
@@ -153,7 +171,10 @@ export async function runStatusCronGenerate(
* inside the batch itself). Net effect: the nightly ladder pays one call per
* user instead of seven, with no queue/registry churn.
*/
async function runStatusBatchCron(taskName: string): Promise<JobOutcome> {
async function runStatusBatchCron(
taskName: string,
shouldStop: () => boolean,
): Promise<JobOutcome> {
return withBackgroundEvent(taskName, async (evt) => {
const prisma = getWorkerPrisma();
try {
@@ -166,7 +187,14 @@ async function runStatusBatchCron(taskName: string): Promise<JobOutcome> {
let generated = 0;
let served = 0;
let failed = 0;
let stoppedEarly = false;
for (const user of users) {
// Same budget as the per-metric passes: stop between users, and
// leave the rest to the later crons and the next night.
if (shouldStop()) {
stoppedEarly = true;
break;
}
try {
const result = await generateStatusBatchForUser(user.id, {
// The stored locale, then the operator default — the prompt names
@@ -186,16 +214,34 @@ async function runStatusBatchCron(taskName: string): Promise<JobOutcome> {
annotate({
action: { name: "insights.status.batch.cron" },
meta: { generated, served, failed, total: users.length },
meta: {
generated,
served,
failed,
total: users.length,
stopped_early: stoppedEarly,
},
});
evt.setBackground({
task_name: taskName,
result: { generated, served, failed, total: users.length },
result: {
generated,
served,
failed,
total: users.length,
stoppedEarly,
},
});
// A user whose batch call threw already fell back to the single-card
// path inside the batch, and the six later per-metric crons cover what
// is still cold. The batch pass ran, so it reports the count.
return jobDone({ total: users.length, generated, served, failed });
return jobDone({
total: users.length,
generated,
served,
failed,
stopped_early: stoppedEarly,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -208,61 +254,66 @@ async function runStatusBatchCron(taskName: string): Promise<JobOutcome> {
export function handleGeneralStatusGenerate(
jobs: Job<GeneralStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusBatchCron("job.insights.batch");
return runStatusBatchCron("job.insights.batch", jobBudget(jobs));
}
export function handleBloodPressureStatusGenerate(
jobs: Job<BloodPressureStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate(
"job.insights.blood_pressure",
generateBloodPressureStatusForUser,
jobBudget(jobs),
);
}
export function handleWeightStatusGenerate(
jobs: Job<WeightStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate(
"job.insights.weight",
generateWeightStatusForUser,
jobBudget(jobs),
);
}
export function handlePulseStatusGenerate(
jobs: Job<PulseStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate(
"job.insights.pulse",
generatePulseStatusForUser,
jobBudget(jobs),
);
}
export function handleBmiStatusGenerate(
jobs: Job<BmiStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate("job.insights.bmi", generateBmiStatusForUser);
return runStatusCronGenerate(
"job.insights.bmi",
generateBmiStatusForUser,
jobBudget(jobs),
);
}
export function handleMoodStatusGenerate(
jobs: Job<MoodStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate("job.insights.mood", generateMoodStatusForUser);
return runStatusCronGenerate(
"job.insights.mood",
generateMoodStatusForUser,
jobBudget(jobs),
);
}
export function handleMedicationComplianceStatusGenerate(
jobs: Job<MedicationComplianceStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate(
"job.insights.medication_compliance",
generateMedicationComplianceStatusForUser,
jobBudget(jobs),
);
}
@@ -310,7 +361,9 @@ export async function handleInsightPregenerateJob(
return jobDone({ forced: forced.length, scheduled: 0 });
}
try {
const summary = await runInsightPregenerate(getWorkerPrisma());
const summary = await runInsightPregenerate(getWorkerPrisma(), {
shouldStop: jobBudget(jobs),
});
evt.setBackground({
task_name: "job.insight_pregenerate",
result: { ...summary },
@@ -325,6 +378,7 @@ export async function handleInsightPregenerateJob(
skipped: summary.skipped,
failed: summary.failed,
budget_blocked: summary.budgetBlocked,
stopped_early: summary.stoppedEarly,
assessments_warmed: summary.assessmentsWarmed,
metric_assessments_warmed: summary.metricAssessmentsWarmed,
});
src/lib/jobs/reminder/ops-handlers.ts
+11
View File
@@ -4,6 +4,7 @@
* Extracted from reminder-worker.ts, which owns the queue names, cron
* schedules, and boss.work registrations.
*/
import { jobBudget } from "@/lib/jobs/job-budget";
import { type Job } from "pg-boss";
import { runGeoBackfill } from "@/lib/jobs/geo-backfill";
import { fetchGeoLite2Databases } from "@/lib/geo/geolite2-fetch";
@@ -223,6 +224,11 @@ export async function handlePrDetection(
let insertedAcrossJobs = 0;
let tiesAcrossJobs = 0;
let usersFailed = 0;
let stoppedEarly = false;
// The cron pass walks every account's all-time history and ticks every
// thirty minutes. It stops between accounts once the job's budget is spent;
// the ingest jobs and the next tick carry the rest.
const shouldStop = jobBudget(jobs);
for (const job of jobs) {
await withBackgroundEvent("job.pr_detection", async (evt) => {
const p = getWorkerPrisma();
@@ -241,6 +247,10 @@ export async function handlePrDetection(
let insertedTotal = 0;
let tiesTotal = 0;
for (const userId of userIds) {
if (shouldStop()) {
stoppedEarly = true;
break;
}
try {
const result = await detectPersonalRecordsForUser(userId, {
silent,
@@ -274,5 +284,6 @@ export async function handlePrDetection(
pr_detection_inserted: insertedAcrossJobs,
pr_detection_ties: tiesAcrossJobs,
pr_detection_users_failed: usersFailed,
stopped_early: stoppedEarly,
});
}
src/lib/jobs/reminder/register-maintenance.ts
+89 -55
View File
@@ -31,8 +31,12 @@ import {
PR_DETECTION_QUEUE,
PR_DETECTION_CONCURRENCY,
PR_DETECTION_FALLBACK_CRON,
PR_DETECTION_EXPIRE_SECONDS,
type PrDetectionPayload,
} from "@/lib/jobs/pr-detection";
import { jobBudget } from "@/lib/jobs/job-budget";
import { lockedPass, WHOLE_PASS } from "@/lib/jobs/long-pass";
import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup";
import {
MEDICATION_INVENTORY_EXPIRE_QUEUE,
MEDICATION_INVENTORY_EXPIRE_CRON,
@@ -531,7 +535,11 @@ const schedules: ScheduleEntry[] = [
STEP_UP_ELEVATION_CLEANUP_CRON,
cronIsTheRetry,
],
[OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON],
[
OFFHOST_BACKUP_QUEUE,
OFFHOST_BACKUP_CRON,
{ expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS },
],
[RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON],
// Nightly backstop for the purge the deletion kicks straight away: a
// request the bucket refused, or one written while the queue was down.
@@ -557,7 +565,11 @@ const schedules: ScheduleEntry[] = [
// cron payload deliberately omits a `userId` so the handler iterates
// every user; per-user push-suppression cannot apply on the cron
// path (the silent flag is set by the ingest hooks).
[PR_DETECTION_QUEUE, PR_DETECTION_FALLBACK_CRON],
[
PR_DETECTION_QUEUE,
PR_DETECTION_FALLBACK_CRON,
{ expireInSeconds: PR_DETECTION_EXPIRE_SECONDS },
],
// v1.4.25 W19b — daily expire-stale pass for the per-pen inventory
// entities. Flips IN_USE rows whose 30-day clock has blown to
// EXPIRED at 03:30 Europe/Berlin (in the existing 02:xx–03:xx
@@ -747,7 +759,7 @@ export async function registerMaintenanceQueues(
boss,
DATA_BACKUP_QUEUE,
{ localConcurrency: 1 },
handleDataBackup,
lockedPass(DATA_BACKUP_QUEUE, () => WHOLE_PASS, handleDataBackup),
);
await createAndWork<RateLimitCleanupPayload>(
boss,
@@ -778,7 +790,7 @@ export async function registerMaintenanceQueues(
boss,
OFFHOST_BACKUP_QUEUE,
{ localConcurrency: 1, includeMetadata: true },
handleOffhostBackup,
lockedPass(OFFHOST_BACKUP_QUEUE, () => WHOLE_PASS, handleOffhostBackup),
);
await createAndWork(
boss,
@@ -935,7 +947,16 @@ export async function registerMaintenanceQueues(
boss,
PR_DETECTION_QUEUE,
{ localConcurrency: PR_DETECTION_CONCURRENCY },
handlePrDetection,
// An ingest job is one account; the cron pass is every account, one run
// at a time.
lockedPass(
PR_DETECTION_QUEUE,
(job) => {
const userId = (job.data as PrDetectionPayload | undefined)?.userId;
return userId ? `user:${userId}` : WHOLE_PASS;
},
handlePrDetection,
),
);
await createAndWork<MedicationInventoryExpirePayload>(
boss,
@@ -1159,7 +1180,11 @@ export async function registerMaintenanceQueues(
boss,
ENCRYPTION_KEY_ROTATE_QUEUE,
{ localConcurrency: ENCRYPTION_KEY_ROTATE_CONCURRENCY },
handleEncryptionKeyRotate,
lockedPass(
ENCRYPTION_KEY_ROTATE_QUEUE,
() => WHOLE_PASS,
handleEncryptionKeyRotate,
),
);
// v1.25 — medication-note encryption backfill worker. The boot enqueue
@@ -1318,56 +1343,65 @@ export async function registerMaintenanceQueues(
boss,
CONTENT_INDEX_BACKFILL_QUEUE,
{ localConcurrency: CONTENT_INDEX_BACKFILL_CONCURRENCY },
async (jobs) => {
let users = 0;
let skipped = 0;
let documentsIndexed = 0;
let documentsRetokenised = 0;
let documentsSkipped = 0;
let documentsFailed = 0;
for (const job of jobs) {
const { userId } = job.data;
if (!userId) {
skipped++;
continue;
lockedPass(
CONTENT_INDEX_BACKFILL_QUEUE,
(job) => `user:${job.data.userId ?? ""}`,
async (jobs) => {
let users = 0;
let stoppedEarly = false;
const shouldStop = jobBudget(jobs);
let skipped = 0;
let documentsIndexed = 0;
let documentsRetokenised = 0;
let documentsSkipped = 0;
let documentsFailed = 0;
for (const job of jobs) {
const { userId } = job.data;
if (!userId) {
skipped++;
continue;
}
try {
const {
indexed,
retokenised,
skipped: docsSkipped,
failed: docsFailed,
reason,
stoppedEarly: stopped,
} = await runContentIndexBackfillForUser(userId, shouldStop);
users++;
stoppedEarly ||= stopped === true;
documentsIndexed += indexed;
documentsRetokenised += retokenised;
documentsSkipped += docsSkipped;
documentsFailed += docsFailed;
workerLog(
"info",
`[document-content-index-backfill] user=${userId} indexed=${indexed} retokenised=${retokenised} skipped=${docsSkipped} failed=${docsFailed} reason=${reason}`,
);
} catch (err) {
recordError();
workerLog(
"error",
`[document-content-index-backfill] user=${userId} failed`,
err,
);
throw err;
}
}
try {
const {
indexed,
retokenised,
skipped: docsSkipped,
failed: docsFailed,
reason,
} = await runContentIndexBackfillForUser(userId);
users++;
documentsIndexed += indexed;
documentsRetokenised += retokenised;
documentsSkipped += docsSkipped;
documentsFailed += docsFailed;
workerLog(
"info",
`[document-content-index-backfill] user=${userId} indexed=${indexed} retokenised=${retokenised} skipped=${docsSkipped} failed=${docsFailed} reason=${reason}`,
);
} catch (err) {
recordError();
workerLog(
"error",
`[document-content-index-backfill] user=${userId} failed`,
err,
);
throw err;
}
}
return jobDone({
jobs: jobs.length,
users,
skipped,
documents_indexed: documentsIndexed,
documents_retokenised: documentsRetokenised,
documents_skipped: documentsSkipped,
documents_failed: documentsFailed,
});
},
return jobDone({
jobs: jobs.length,
users,
skipped,
documents_indexed: documentsIndexed,
documents_retokenised: documentsRetokenised,
documents_skipped: documentsSkipped,
documents_failed: documentsFailed,
stopped_early: stoppedEarly,
});
},
),
);
// Document AI — automatic per-document content indexing. Enqueued on upload
src/lib/jobs/reminder/register-status.ts
+106 -53
View File
@@ -102,11 +102,16 @@ import {
} from "@/lib/jobs/insight-status-generate";
import { withBackgroundEvent } from "@/lib/logging/background";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { jobBudget } from "@/lib/jobs/job-budget";
import { lockedPass, WHOLE_PASS } from "@/lib/jobs/long-pass";
import { INSIGHT_PREGENERATE_EXPIRE_SECONDS } from "@/lib/jobs/insight-pregenerate-shared";
import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "@/lib/jobs/period-narrative-shared";
import { getWorkerPrisma, workerLog } from "./shared";
import {
createAndSchedule,
createAndWork,
insightRetryOptions,
nightlyInsightPassOptions,
type QueuePolicyTable,
type ScheduleEntry,
} from "./registrar-shared";
@@ -232,25 +237,32 @@ const allQueues = [
];
const schedules: ScheduleEntry[] = [
[GENERAL_STATUS_QUEUE, GENERAL_STATUS_CRON, insightRetryOptions],
[GENERAL_STATUS_QUEUE, GENERAL_STATUS_CRON, nightlyInsightPassOptions],
[
BLOOD_PRESSURE_STATUS_QUEUE,
BLOOD_PRESSURE_STATUS_CRON,
insightRetryOptions,
nightlyInsightPassOptions,
],
[WEIGHT_STATUS_QUEUE, WEIGHT_STATUS_CRON, insightRetryOptions],
[PULSE_STATUS_QUEUE, PULSE_STATUS_CRON, insightRetryOptions],
[BMI_STATUS_QUEUE, BMI_STATUS_CRON, insightRetryOptions],
[WEIGHT_STATUS_QUEUE, WEIGHT_STATUS_CRON, nightlyInsightPassOptions],
[PULSE_STATUS_QUEUE, PULSE_STATUS_CRON, nightlyInsightPassOptions],
[BMI_STATUS_QUEUE, BMI_STATUS_CRON, nightlyInsightPassOptions],
// v1.15.20 — mood status nightly, continuing the 02:xx ladder.
[MOOD_STATUS_QUEUE, MOOD_STATUS_CRON, insightRetryOptions],
[MOOD_STATUS_QUEUE, MOOD_STATUS_CRON, nightlyInsightPassOptions],
[
MEDICATION_COMPLIANCE_STATUS_QUEUE,
MEDICATION_COMPLIANCE_STATUS_CRON,
insightRetryOptions,
nightlyInsightPassOptions,
],
// v1.7.0 — nightly 04:30 Europe/Berlin comprehensive-insight
// pre-generation. Budget-gated per user inside the handler.
[INSIGHT_PREGENERATE_QUEUE, INSIGHT_PREGENERATE_CRON, insightRetryOptions],
[
INSIGHT_PREGENERATE_QUEUE,
INSIGHT_PREGENERATE_CRON,
{
...insightRetryOptions,
expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS,
},
],
// v1.10.0 — computed scores (WX-C). Nightly 04:45 Europe/Berlin
// Recovery-score compute + store, after the rollup-feeding consolidation
// + drain so the signals it reads are already folded.
@@ -266,7 +278,14 @@ const schedules: ScheduleEntry[] = [
// v1.11.0 — nightly 05:05 Europe/Berlin period-narrative warm. The
// handler only fans out on a week (Mon) / month (1st) boundary; every
// other night is a cheap no-op. Budget-gated per user inside the runner.
[PERIOD_NARRATIVE_QUEUE, PERIOD_NARRATIVE_CRON, insightRetryOptions],
[
PERIOD_NARRATIVE_QUEUE,
PERIOD_NARRATIVE_CRON,
{
...insightRetryOptions,
expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS,
},
],
// v1.15.20 — daily 05:15 Europe/Berlin proactive Coach nudge, after
// the 04:45–04:55 score crons so the recovery-score trigger reads
// settled rows. Deterministic triggers only — no AI call on this path.
@@ -369,43 +388,59 @@ export async function registerStatusQueues(
boss,
GENERAL_STATUS_QUEUE,
{ localConcurrency: 1 },
handleGeneralStatusGenerate,
lockedPass(
GENERAL_STATUS_QUEUE,
() => WHOLE_PASS,
handleGeneralStatusGenerate,
),
);
await createAndWork<BloodPressureStatusPayload>(
boss,
BLOOD_PRESSURE_STATUS_QUEUE,
{ localConcurrency: 1 },
handleBloodPressureStatusGenerate,
lockedPass(
BLOOD_PRESSURE_STATUS_QUEUE,
() => WHOLE_PASS,
handleBloodPressureStatusGenerate,
),
);
await createAndWork<WeightStatusPayload>(
boss,
WEIGHT_STATUS_QUEUE,
{ localConcurrency: 1 },
handleWeightStatusGenerate,
lockedPass(
WEIGHT_STATUS_QUEUE,
() => WHOLE_PASS,
handleWeightStatusGenerate,
),
);
await createAndWork<PulseStatusPayload>(
boss,
PULSE_STATUS_QUEUE,
{ localConcurrency: 1 },
handlePulseStatusGenerate,
lockedPass(PULSE_STATUS_QUEUE, () => WHOLE_PASS, handlePulseStatusGenerate),
);
await createAndWork<BmiStatusPayload>(
boss,
BMI_STATUS_QUEUE,
{ localConcurrency: 1 },
handleBmiStatusGenerate,
lockedPass(BMI_STATUS_QUEUE, () => WHOLE_PASS, handleBmiStatusGenerate),
);
await createAndWork<MoodStatusPayload>(
boss,
MOOD_STATUS_QUEUE,
{ localConcurrency: 1 },
handleMoodStatusGenerate,
lockedPass(MOOD_STATUS_QUEUE, () => WHOLE_PASS, handleMoodStatusGenerate),
);
await createAndWork<MedicationComplianceStatusPayload>(
boss,
MEDICATION_COMPLIANCE_STATUS_QUEUE,
{ localConcurrency: 1 },
handleMedicationComplianceStatusGenerate,
lockedPass(
MEDICATION_COMPLIANCE_STATUS_QUEUE,
() => WHOLE_PASS,
handleMedicationComplianceStatusGenerate,
),
);
// v1.7.0 — nightly comprehensive-insight pre-generation.
//
@@ -422,7 +457,16 @@ export async function registerStatusQueues(
boss,
INSIGHT_PREGENERATE_QUEUE,
{ localConcurrency: 2 },
handleInsightPregenerateJob,
// The cohort walk is one pass at a time; a forced warm is one per
// account, so it never waits behind the walk.
lockedPass(
INSIGHT_PREGENERATE_QUEUE,
(job) =>
job.data?.force && job.data.userId
? `user:${job.data.userId}`
: WHOLE_PASS,
handleInsightPregenerateJob,
),
);
// S4 — event-driven morning digest refresh. One forced comprehensive
// regeneration per user per local morning, enqueued by the sleep-arrival
@@ -749,44 +793,53 @@ export async function registerStatusQueues(
boss,
PERIOD_NARRATIVE_QUEUE,
{ localConcurrency: 1 },
async (jobs): Promise<JobOutcome> => {
// Two modes on one queue, so the outcome counts them separately: a
// non-boundary night is a scheduled pass that generated nothing, which
// is a success with zero counts, not an idle queue.
let singleUser = 0;
let scheduled = 0;
let generated = 0;
let failed = 0;
for (const job of jobs) {
try {
if (job.data?.userId) {
await warmOneNarrative(job.data);
singleUser++;
} else {
const summary = await runPeriodNarrativeWarm(getWorkerPrisma());
scheduled++;
generated += summary.generated;
failed += summary.failed;
workerLog(
"info",
`[period-narrative] periods=${summary.periods.join(",") || "none"} total=${summary.total} generated=${summary.generated} cached=${summary.cached} skipped=${summary.skipped} insufficient=${summary.insufficient} failed=${summary.failed} budget=${summary.budgetBlocked}`,
);
lockedPass(
PERIOD_NARRATIVE_QUEUE,
(job) => (job.data?.userId ? `user:${job.data.userId}` : WHOLE_PASS),
async (jobs): Promise<JobOutcome> => {
// Two modes on one queue, so the outcome counts them separately: a
// non-boundary night is a scheduled pass that generated nothing, which
// is a success with zero counts, not an idle queue.
let singleUser = 0;
let scheduled = 0;
let generated = 0;
let failed = 0;
let stoppedEarly = false;
for (const job of jobs) {
try {
if (job.data?.userId) {
await warmOneNarrative(job.data);
singleUser++;
} else {
const summary = await runPeriodNarrativeWarm(getWorkerPrisma(), {
shouldStop: jobBudget(jobs),
});
scheduled++;
generated += summary.generated;
failed += summary.failed;
stoppedEarly ||= summary.stoppedEarly;
workerLog(
"info",
`[period-narrative] periods=${summary.periods.join(",") || "none"} total=${summary.total} generated=${summary.generated} cached=${summary.cached} skipped=${summary.skipped} insufficient=${summary.insufficient} failed=${summary.failed} budget=${summary.budgetBlocked}`,
);
}
} catch (err) {
recordError();
await reportWorkerError(PERIOD_NARRATIVE_QUEUE, err, {
mode: job.data?.userId ? "single-user" : "scheduled",
});
throw err;
}
} catch (err) {
recordError();
await reportWorkerError(PERIOD_NARRATIVE_QUEUE, err, {
mode: job.data?.userId ? "single-user" : "scheduled",
});
throw err;
}
}
return jobDone({
single_user: singleUser,
scheduled,
generated,
failed,
});
},
return jobDone({
single_user: singleUser,
scheduled,
generated,
failed,
stopped_early: stoppedEarly,
});
},
),
);
// v1.11.1 — combined Coach memory refresh: rolling conversation summary +
// durable fact extraction for one long conversation. localConcurrency 1 so a
src/lib/jobs/reminder/registrar-shared.ts
+16
View File
@@ -168,6 +168,22 @@ export const insightRetryOptions = {
retryBackoff: true,
} as const;
/**
* Send options for a nightly AI pass over every account (the status crons).
*
* The retries of `insightRetryOptions`, plus a two-hour expiry. Each pass
* makes a provider call per account, and on a large instance that outlasted
* pg-boss's fifteen-minute default: the job was declared dead and retried
* beside the pass still running. The pass now stops itself at three quarters
* of this (`jobBudget`) and holds a lock (`lockedPass`). The options ride on
* the schedule because the cron is the only sender.
*/
export const NIGHTLY_INSIGHT_PASS_EXPIRE_SECONDS = 2 * 60 * 60;
export const nightlyInsightPassOptions = {
...insightRetryOptions,
expireInSeconds: NIGHTLY_INSIGHT_PASS_EXPIRE_SECONDS,
} as const;
/**
* Retry policy for a cron-driven pass whose failure mode is deterministic.
*
src/lib/openapi/routes/measurements.ts
+8 -2
View File
@@ -389,6 +389,12 @@ const listMeasurementsMeta = z
.describe(
"Rows discarded by the cross-source canonical picker on the day-collapse mode.",
),
windowFrom: z.iso
.datetime({ offset: true })
.optional()
.describe(
"Set on the per-night SLEEP_DURATION mode when the requested window was longer than 365 days: the instant the read started from. The window is a year ending at `to` (or now); page further back by moving `to`.",
),
})
.meta({ id: "ListMeasurementsMeta" });
@@ -696,7 +702,7 @@ const seriesPointSchema = z.object({
id: z
.string()
.describe(
"Measurement row id; `sleep:<wake-day>` for a sleep night; `day:<local-day>` for a day bucket (`pulse` / `glucose` beyond 90 days); `hour:<ISO start>` for an hour bucket (`pulse` / `glucose` within 90 days when the window holds more than 10 000 readings).",
"Measurement row id; `sleep:<wake-day>` for a sleep night; `day:<local-day>` for a day bucket (`pulse`, `glucose`, `heartRateVariability`, `oxygenSaturation` beyond 90 days); `hour:<ISO start>` for an hour bucket (the same kinds within 90 days when the window holds more than 10 000 readings).",
),
at: z.iso.datetime({ offset: true }).describe("Point timestamp (ISO-8601)."),
value: z.number().describe("Primary value in the top-level `unit`."),
@@ -751,7 +757,7 @@ const seriesResponse = z
.meta({
id: "MeasurementsSeriesResponse",
description:
"iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and `glucose` beyond 90 days per local day, and `pulse` / `glucose` within 90 days per local hour when the window holds more than 10 000 readings (a continuous glucose sensor, a watch recording heart rate every minute; a cuff or fingerstick meter stays raw). A bucket carries the mean as `value`; a `pulse` bucket also carries its range as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats` is always computed over the underlying readings, never over the buckets.",
"iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an explicit `unit` token, and a summary `stats` block. Sample-dense kinds (`pulse`, `glucose`, `heartRateVariability`, `oxygenSaturation`) are bucketed server-side: beyond 90 days per local day, and within 90 days per local hour when the window holds more than 10 000 readings (a continuous glucose sensor, a watch or ring recording every minute; a cuff, a fingerstick meter or a few readings a night stay raw). A bucket carries the mean as `value` and otherwise has the shape of a reading of its kind; a `pulse` bucket also carries its range as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats` is always computed over the underlying readings, never over the buckets.",
});
// ── Batched daily series (v1.18.6 dashboard fetch coalescing) ────────
src/lib/targets/__tests__/builders.test.ts
+22 -10
View File
@@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest";
import { buildGlucoseTargets } from "../glucose-builder";
import { foldGlucoseTargetRows } from "../glucose-read";
import { buildMedicationTarget } from "../medication-builder";
import { buildMoodTargets } from "../mood-builder";
import { buildSleepTarget } from "../sleep-builder";
@@ -8,11 +9,13 @@ import { buildVitalTargets } from "../vitals-builder";
const NOW = new Date("2026-07-21T12:00:00.000Z");
const TZ = "UTC";
const RECENT = new Date(NOW.getTime() - 30 * 86_400_000);
describe("target section builders", () => {
it("keeps the vital target order and profile-gated cards on an empty profile", () => {
const result = buildVitalTargets({
recentMeasurements: [],
restingPulseProxy: [],
latestByType: {},
average30ByType: {},
heightCm: null,
@@ -108,7 +111,7 @@ describe("target section builders", () => {
},
];
const targets = buildGlucoseTargets({
rows,
summaries: foldGlucoseTargetRows(rows, RECENT, TZ),
profile: {
heightCm: 180,
dateOfBirth: new Date("1985-01-01T00:00:00.000Z"),
@@ -140,10 +143,14 @@ describe("target section builders", () => {
// metadata, so every row arrives with a null context. Iterating the four
// named contexts alone returned no glucose target at all.
const targets = buildGlucoseTargets({
rows: [
{ value: 112, measuredAt: NOW, glucoseContext: null },
{ value: 104, measuredAt: NOW, glucoseContext: null },
],
summaries: foldGlucoseTargetRows(
[
{ value: 112, measuredAt: NOW, glucoseContext: null },
{ value: 104, measuredAt: NOW, glucoseContext: null },
],
RECENT,
TZ,
),
profile: {
heightCm: 180,
dateOfBirth: new Date("1985-01-01T00:00:00.000Z"),
@@ -173,11 +180,15 @@ describe("target section builders", () => {
it("keeps the tagged breakdown and puts the untagged bucket last (#943)", () => {
const targets = buildGlucoseTargets({
rows: [
{ value: 130, measuredAt: NOW, glucoseContext: "RANDOM" as const },
{ value: 90, measuredAt: NOW, glucoseContext: "FASTING" as const },
{ value: 111, measuredAt: NOW, glucoseContext: null },
],
summaries: foldGlucoseTargetRows(
[
{ value: 130, measuredAt: NOW, glucoseContext: "RANDOM" as const },
{ value: 90, measuredAt: NOW, glucoseContext: "FASTING" as const },
{ value: 111, measuredAt: NOW, glucoseContext: null },
],
RECENT,
TZ,
),
profile: {
heightCm: 180,
dateOfBirth: new Date("1985-01-01T00:00:00.000Z"),
@@ -235,6 +246,7 @@ describe("target section builders", () => {
describe("buildVitalTargets — user weight target", () => {
const base = {
recentMeasurements: [],
restingPulseProxy: [],
latestByType: {},
average30ByType: {},
heightCm: 178,
src/lib/targets/build-response.ts
+28 -12
View File
@@ -26,7 +26,9 @@ import {
ensureUserMoodRollupsFresh,
readMoodDayRollups,
} from "@/lib/rollups/mood-rollups";
import { readRestingPulseProxy } from "@/lib/analytics/resting-pulse-read";
import { buildGlucoseTargets } from "./glucose-builder";
import { readGlucoseTargetSummaries } from "./glucose-read";
import { buildMedicationTarget } from "./medication-builder";
import { buildMoodTargets } from "./mood-builder";
import { buildSleepTarget } from "./sleep-builder";
@@ -75,6 +77,10 @@ export async function buildTargetsResponse(user: AuthedUser) {
"BODY_FAT",
"ACTIVITY_STEPS",
];
// Heart rate is the one vital a device streams. Its thirty days reach the
// resting-pulse card as one proxy point per day, folded in SQL below; every
// other vital is written a few times a day and is read as rows.
const recentTypes = types.filter((type) => type !== "PULSE");
const thirtyDaysAgo = new Date(queryNow.getTime() - 30 * 24 * 60 * 60 * 1000);
const oneYearAgo = new Date(queryNow.getTime() - 365 * 24 * 60 * 60 * 1000);
const moodSince = new Date(
@@ -95,7 +101,8 @@ export async function buildTargetsResponse(user: AuthedUser) {
medicationBundle,
moodRollups,
latestMoodEntry,
glucoseRows,
glucoseSummaries,
restingPulseProxy,
] = await Promise.all([
limit(() =>
prisma.user.findUnique({
@@ -115,7 +122,7 @@ export async function buildTargetsResponse(user: AuthedUser) {
prisma.measurement.findMany({
where: {
userId,
type: { in: types },
type: { in: recentTypes },
measuredAt: { gte: thirtyDaysAgo },
deletedAt: null,
},
@@ -211,16 +218,22 @@ export async function buildTargetsResponse(user: AuthedUser) {
select: { score: true, moodLoggedAt: true },
}),
),
// A sensor reporting every five minutes writes 105 000 glucose readings
// a year. The cards need the newest per meal context and the last thirty
// days per local day, so both fold in SQL.
limit(() =>
prisma.measurement.findMany({
where: {
userId,
type: "BLOOD_GLUCOSE",
measuredAt: { gte: oneYearAgo },
deletedAt: null,
},
orderBy: { measuredAt: "desc" },
select: { value: true, measuredAt: true, glucoseContext: true },
readGlucoseTargetSummaries({
userId,
since: oneYearAgo,
recentSince: thirtyDaysAgo,
timeZone: timezone,
}),
),
limit(() =>
readRestingPulseProxy({
userId,
since: thirtyDaysAgo,
timeZone: timezone,
}),
),
]);
@@ -251,6 +264,8 @@ export async function buildTargetsResponse(user: AuthedUser) {
for (const type of types) {
latestByType[type] =
latestEverByType.find((row) => row.type === type)?.value ?? null;
}
for (const type of recentTypes) {
const recent = recentMeasurements.filter(
(measurement) => measurement.type === type,
);
@@ -266,6 +281,7 @@ export async function buildTargetsResponse(user: AuthedUser) {
const vitalSection = buildVitalTargets({
recentMeasurements,
restingPulseProxy,
latestByType,
average30ByType,
heightCm,
@@ -314,7 +330,7 @@ export async function buildTargetsResponse(user: AuthedUser) {
const glucoseUnit = resolveGlucoseUnit(dbUser?.glucoseUnit ?? null);
targets.push(
...buildGlucoseTargets({
rows: glucoseRows,
summaries: glucoseSummaries,
profile: {
heightCm,
dateOfBirth: dbUser?.dateOfBirth ?? null,
src/lib/targets/consistency.ts
+45 -17
View File
@@ -43,12 +43,13 @@ function last7DayKeys(now: Date, timezone: string): string[] {
return keys;
}
function computeStrip(
/** A day's readings folded to their sum and count, keyed by local day. */
export type DaySums = ReadonlyMap<string, { sum: number; count: number }>;
function sumByDay(
events: ReadonlyArray<{ measuredAt: Date; value: number }>,
classify: (mean: number) => DayBand | null,
timezone: string,
now: Date,
) {
): Map<string, { sum: number; count: number }> {
const byDay = new Map<string, { sum: number; count: number }>();
for (const event of events) {
const key = userDayKey(event.measuredAt, timezone);
@@ -57,7 +58,15 @@ function computeStrip(
bucket.count += 1;
byDay.set(key, bucket);
}
return byDay;
}
function computeStrip(
byDay: DaySums,
classify: (mean: number) => DayBand | null,
timezone: string,
now: Date,
) {
const consistency7d: Array<DayBand | null> = [];
let daysInRange7d = 0;
let daysLogged7d = 0;
@@ -77,13 +86,40 @@ function computeStrip(
export function rollupConsistency({
events,
...rest
}: RollupConsistencyInput): TargetConsistency {
return rollupConsistencyFromDays({
...rest,
byDay: sumByDay(events, rest.timezone),
readingCount: events.length,
});
}
interface RollupDaySumsInput extends ConsistencyClock {
/** Each local day's readings as sum and count. */
byDay: DaySums;
/** Readings behind `byDay`, for the thin-data threshold. */
readingCount: number;
classify: (mean: number) => DayBand | null;
totalReadingsThreshold?: number;
}
/**
* The same consistency as {@link rollupConsistency}, from readings already
* folded per local day. Every figure here is a function of each day's mean
* and of the reading count, so a reader that folds in SQL hands over one row
* per day instead of every reading and gets the same answer.
*/
export function rollupConsistencyFromDays({
byDay,
readingCount,
classify,
timezone,
now,
totalReadingsThreshold = 3,
}: RollupConsistencyInput): TargetConsistency {
if (events.length < totalReadingsThreshold) {
const partial = computeStrip(events, classify, timezone, now);
}: RollupDaySumsInput): TargetConsistency {
if (readingCount < totalReadingsThreshold) {
const partial = computeStrip(byDay, classify, timezone, now);
return {
...EMPTY_CONSISTENCY,
...partial,
@@ -91,15 +127,7 @@ export function rollupConsistency({
};
}
const byDay = new Map<string, { sum: number; count: number }>();
for (const event of events) {
const key = userDayKey(event.measuredAt, timezone);
const bucket = byDay.get(key) ?? { sum: 0, count: 0 };
bucket.sum += event.value;
bucket.count += 1;
byDay.set(key, bucket);
}
const strip = computeStrip(events, classify, timezone, now);
const strip = computeStrip(byDay, classify, timezone, now);
let daysInRange30d = 0;
let daysLogged30d = 0;
@@ -136,7 +164,7 @@ export function rollupConsistency({
lastMetGoalAt,
streakDays,
insufficientData:
events.length < totalReadingsThreshold || strip.daysLogged7d < 1,
readingCount < totalReadingsThreshold || strip.daysLogged7d < 1,
consistency7d: strip.consistency7d,
};
}
src/lib/targets/glucose-builder.ts
+35 -21
View File
@@ -4,8 +4,9 @@ import {
type GlucoseContextBucket,
} from "@/lib/glucose";
import { resolveGlucoseTarget } from "./glucose-targets";
import { makeRangeClassifier, rollupConsistency } from "./consistency";
import type { TargetGlucoseRow, TargetItem, TargetProfile } from "./types";
import { makeRangeClassifier, rollupConsistencyFromDays } from "./consistency";
import type { TargetGlucoseContextSummary } from "./glucose-read";
import type { TargetItem, TargetProfile } from "./types";
const LABEL_BY_CONTEXT: Record<GlucoseContextBucket, string> = {
FASTING: "targets.glucoseFasting",
@@ -16,38 +17,53 @@ const LABEL_BY_CONTEXT: Record<GlucoseContextBucket, string> = {
};
interface GlucoseTargetsInput {
rows: TargetGlucoseRow[];
/**
* One summary per stored meal context: the newest reading of the year and
* the last thirty days per local day (`readGlucoseTargetSummaries`).
*/
summaries: TargetGlucoseContextSummary[];
profile: TargetProfile;
timezone: string;
now: Date;
}
export function buildGlucoseTargets({
rows,
summaries,
profile,
timezone,
now,
}: GlucoseTargetsInput): TargetItem[] {
const targets: TargetItem[] = [];
const unit = resolveGlucoseUnit(profile.glucoseUnit);
const thirtyDaysAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
// #943 — the untagged bucket rides the same loop. Iterating the four named
// contexts alone left an account whose source writes no meal-time tag with
// no glucose target at all, the same gap the dashboard tile had.
for (const [context, contextRows] of groupByGlucoseContext(
rows,
(row) => row.glucoseContext,
for (const [context, contextSummaries] of groupByGlucoseContext(
summaries,
(summary) => summary.glucoseContext,
)) {
const latest = contextRows[0].value;
const recent = contextRows.filter((row) => row.measuredAt >= thirtyDaysAgo);
// A bucket is one stored context today; folding several keeps the newest
// reading and adds the days up, should two ever share a bucket.
const newest = contextSummaries.reduce((a, b) =>
b.latestAt > a.latestAt ? b : a,
);
const latest = newest.latest;
const recentByDay = new Map<string, { sum: number; count: number }>();
let recentCount = 0;
let recentSum = 0;
for (const summary of contextSummaries) {
recentCount += summary.recentCount;
for (const [day, bucket] of summary.recentByDay) {
const current = recentByDay.get(day) ?? { sum: 0, count: 0 };
current.sum += bucket.sum;
current.count += bucket.count;
recentByDay.set(day, current);
recentSum += bucket.sum;
}
}
const average30 =
recent.length > 0
? Math.round(
(recent.reduce((sum, row) => sum + row.value, 0) / recent.length) *
10,
) / 10
: null;
recentCount > 0 ? Math.round((recentSum / recentCount) * 10) / 10 : null;
const resolved = resolveGlucoseTarget({
context,
hasDiabetes: profile.hasDiabetes,
@@ -97,11 +113,9 @@ export function buildGlucoseTargets({
: resolved.source === "ADA goal (diabetes)"
? "ADA goal (diabetes)"
: "ADA 2024 / DDG",
...rollupConsistency({
events: recent.map((row) => ({
measuredAt: row.measuredAt,
value: row.value,
})),
...rollupConsistencyFromDays({
byDay: recentByDay,
readingCount: recentCount,
classify: makeRangeClassifier(
range,
effectiveRange
src/lib/targets/glucose-read.ts
+148
View File
@@ -0,0 +1,148 @@
/**
* The glucose rows the targets page needs, folded in Postgres.
*
* The glucose targets used to read every glucose reading of the last year and
* keep three things from them per meal context: the newest value, and for the
* last thirty days each local day's mean and the number of readings. A sensor
* that reports every five minutes puts 105 000 readings in that year, and the
* targets page is read on every visit, on the web and in the app. So the year
* is reduced to one row per context (`DISTINCT ON`) and the thirty days to one
* row per context and local day, and only those rows cross the wire.
*
* Output matches {@link foldGlucoseTargetRows} over the raw readings: the same
* newest value per context (ties broken by id, newest first), and the same
* per-day sums and counts, with days cut in the zone the caller passes by the
* expression every other day fold uses.
*/
import { Prisma, type PrismaClient } from "@/generated/prisma/client";
import type { GlucoseContext } from "@/generated/prisma/client";
import { prisma } from "@/lib/db";
import { isValidTimezone, userDayKey } from "@/lib/tz/format";
import type { TargetGlucoseRow } from "./types";
/** One meal context's newest reading and its last thirty days, per day. */
export interface TargetGlucoseContextSummary {
glucoseContext: GlucoseContext | null;
latest: number;
latestAt: Date;
/** Local day → the day's readings as sum and count, since `recentSince`. */
recentByDay: Map<string, { sum: number; count: number }>;
/** Readings behind `recentByDay`. */
recentCount: number;
}
/**
* The in-memory fold over raw readings, newest first. Pure; the reference the
* SQL reader is tested against.
*/
export function foldGlucoseTargetRows(
rows: readonly TargetGlucoseRow[],
recentSince: Date,
timeZone: string,
): TargetGlucoseContextSummary[] {
const byContext = new Map<string, TargetGlucoseContextSummary>();
for (const row of rows) {
const key = row.glucoseContext ?? "";
let summary = byContext.get(key);
if (!summary) {
summary = {
glucoseContext: row.glucoseContext,
latest: row.value,
latestAt: row.measuredAt,
recentByDay: new Map(),
recentCount: 0,
};
byContext.set(key, summary);
}
if (row.measuredAt < recentSince) continue;
const day = userDayKey(row.measuredAt, timeZone);
const bucket = summary.recentByDay.get(day) ?? { sum: 0, count: 0 };
bucket.sum += row.value;
bucket.count += 1;
summary.recentByDay.set(day, bucket);
summary.recentCount += 1;
}
return [...byContext.values()];
}
export async function readGlucoseTargetSummaries(opts: {
userId: string;
/** Inclusive lower bound for the newest reading per context. */
since: Date;
/** Inclusive lower bound for the per-day fold. */
recentSince: Date;
/** IANA zone the day boundary is cut in. */
timeZone: string;
db?: Pick<PrismaClient, "$queryRaw">;
}): Promise<TargetGlucoseContextSummary[]> {
const db = opts.db ?? prisma;
const timeZone = isValidTimezone(opts.timeZone) ? opts.timeZone : "UTC";
const [latestRows, dayRows] = await Promise.all([
db.$queryRaw<
Array<{
glucose_context: GlucoseContext | null;
value: number;
measured_at: Date;
}>
>(Prisma.sql`
SELECT DISTINCT ON (m."glucose_context")
m."glucose_context" AS glucose_context,
m."value" AS value,
m."measured_at" AS measured_at
FROM measurements m
WHERE m."user_id" = ${opts.userId}
AND m."type" = 'BLOOD_GLUCOSE'::measurement_type
AND m."deleted_at" IS NULL
AND m."measured_at" >= ${opts.since}
ORDER BY m."glucose_context", m."measured_at" DESC, m."id" DESC
`),
db.$queryRaw<
Array<{
glucose_context: GlucoseContext | null;
day: string;
n: number;
sum: number;
}>
>(Prisma.sql`
WITH src AS (
SELECT
m."glucose_context" AS glucose_context,
to_char((m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}, 'YYYY-MM-DD') AS day,
m."value" AS value
FROM measurements m
WHERE m."user_id" = ${opts.userId}
AND m."type" = 'BLOOD_GLUCOSE'::measurement_type
AND m."deleted_at" IS NULL
AND m."measured_at" >= ${opts.recentSince}
)
SELECT
glucose_context,
day,
COUNT(*)::int AS n,
SUM(value)::double precision AS sum
FROM src
GROUP BY glucose_context, day
`),
]);
const byContext = new Map<string, TargetGlucoseContextSummary>();
for (const row of latestRows) {
byContext.set(row.glucose_context ?? "", {
glucoseContext: row.glucose_context,
latest: Number(row.value),
latestAt: row.measured_at,
recentByDay: new Map(),
recentCount: 0,
});
}
for (const row of dayRows) {
// Every recent reading is also inside the year, so its context has a
// newest row unless `recentSince` predates `since`, which no caller does.
const summary = byContext.get(row.glucose_context ?? "");
if (!summary) continue;
const n = Number(row.n);
summary.recentByDay.set(row.day, { sum: Number(row.sum), count: n });
summary.recentCount += n;
}
return [...byContext.values()];
}
src/lib/targets/vitals-builder.ts
+15 -8
View File
@@ -13,7 +13,10 @@ import {
classifyPulseByTarget,
getPersonalizedPulseTarget,
} from "@/lib/analytics/pulse-targets";
import { resolveRestingPulseSeries } from "@/lib/analytics/resting-pulse";
import {
mergeRestingWithProxy,
type PulseSample,
} from "@/lib/analytics/resting-pulse";
import { getBodyFatTargetRange } from "@/lib/analytics/value-bands";
import { binaryReferenceSex, type ProfileSex } from "@/lib/profile/sex";
import { userDayKey } from "@/lib/tz/resolver";
@@ -38,7 +41,15 @@ export interface BloodPressureTargetRange {
}
interface VitalTargetsInput {
/** The last thirty days of every vital type except `PULSE`. */
recentMeasurements: TargetMeasurement[];
/**
* The resting-pulse proxy over the same thirty days, one point per local
* day, folded from raw `PULSE` in Postgres (`readRestingPulseProxy`). Raw
* heart rate is never read here: a watch that records it once a minute puts
* 43 000 readings in thirty days, and the card needs one figure per day.
*/
restingPulseProxy: PulseSample[];
latestByType: TargetValueByType;
average30ByType: TargetValueByType;
heightCm: number | null;
@@ -140,6 +151,7 @@ function buildBpPairsByDay(
export function buildVitalTargets({
recentMeasurements,
restingPulseProxy,
latestByType,
average30ByType,
heightCm,
@@ -273,19 +285,14 @@ export function buildVitalTargets({
}
const pulseTarget = getPersonalizedPulseTarget(age, referenceSex);
const restingResolved = resolveRestingPulseSeries({
const restingResolved = mergeRestingWithProxy({
restingSamples: recentMeasurements
.filter((measurement) => measurement.type === "RESTING_HEART_RATE")
.map((measurement) => ({
measuredAt: measurement.measuredAt,
value: measurement.value,
})),
pulseSamples: recentMeasurements
.filter((measurement) => measurement.type === "PULSE")
.map((measurement) => ({
measuredAt: measurement.measuredAt,
value: measurement.value,
})),
proxySamples: restingPulseProxy,
dayKeyOf: (date) => userDayKey(date, timezone),
});
const restingEvents = restingResolved.series;
tests/integration/cycle-insights-dense-reads.test.ts
+264
View File
@@ -0,0 +1,264 @@
/**
* `GET /api/cycle/insights` reads its outcome channels as per-source day
* aggregates, not as a year of readings.
*
* The route compared a year of resting heart rate, heart-rate variability,
* sleep, steps, weight, temperature and glucose across the cycle phases, and
* read every reading of it. A sensor reporting glucose every five minutes is
* over 100 000 readings in that year. The contrast only uses each local day's
* sum and count after one source and device is picked per day, which is what
* `readSourceDayAggregates` returns.
*
* Two properties on real Postgres: no outcome channel is read as rows, and the
* response is the one the raw readings give. The second runs the route twice,
* once over the SQL fold and once over every raw reading handed in as its own
* one-reading aggregate (exactly the old input), and compares the bodies.
*/
import { beforeAll, describe, expect, it, vi } from "vitest";
import { cookieJar } from "./mock-next-headers";
import { getPrismaClient, truncateAllTables } from "./setup";
import type { SourceDayAggregateRow } from "@/lib/measurements/day-aggregates";
import { userDayKey } from "@/lib/tz/format";
import { invalidateUserTimezone } from "@/lib/tz/resolver";
vi.mock("next/headers", async () => {
const { cookieJar, headerJar } = await import("./mock-next-headers");
return {
headers: vi.fn(async () => ({
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
})),
cookies: vi.fn(async () => ({
get: (name: string) => {
const value = cookieJar.get(name);
return value ? { name, value } : undefined;
},
set: (name: string, value: string) => cookieJar.set(name, value),
delete: (name: string) => cookieJar.delete(name),
})),
};
});
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
/** When set, the fold is replaced by every raw reading as a one-row group. */
const mode = vi.hoisted(() => ({ raw: false }));
vi.mock("@/lib/measurements/day-aggregates", async (importOriginal) => {
const actual =
await importOriginal<typeof import("@/lib/measurements/day-aggregates")>();
return {
...actual,
readSourceDayAggregates: async (
opts: Parameters<typeof actual.readSourceDayAggregates>[0],
): Promise<SourceDayAggregateRow[]> => {
if (!mode.raw) return actual.readSourceDayAggregates(opts);
const { getPrismaClient } = await import("./setup");
const rows = await getPrismaClient().measurement.findMany({
where: {
userId: opts.userId,
deletedAt: null,
type: { in: [...opts.types] },
measuredAt: { gte: opts.since },
},
orderBy: { measuredAt: "asc" },
select: {
type: true,
value: true,
measuredAt: true,
source: true,
deviceType: true,
},
});
return rows.map((r) => ({
type: r.type,
day: userDayKey(r.measuredAt, opts.timeZone),
source: r.source,
deviceType: r.deviceType,
n: 1,
sum: r.value,
firstAt: r.measuredAt,
}));
},
};
});
const prisma = getPrismaClient();
const OWNER = "cycle-dense-owner";
const TZ = "Europe/Berlin";
const CYCLE_LENGTH = 28;
const OBSERVED_CYCLES = 5;
const DAY = 86_400_000;
function isoDay(offsetDays: number): string {
return new Date(Date.now() + offsetDays * DAY).toISOString().slice(0, 10);
}
beforeAll(async () => {
await truncateAllTables(prisma);
await prisma.user.create({
data: { id: OWNER, username: OWNER, gender: "FEMALE", timezone: TZ },
});
invalidateUserTimezone(OWNER);
await prisma.cycleProfile.create({
data: {
userId: OWNER,
cycleTrackingEnabled: true,
predictionEnabled: false,
lutealPhaseLength: 14,
},
});
for (let i = OBSERVED_CYCLES; i >= 1; i--) {
await prisma.menstrualCycle.create({
data: {
userId: OWNER,
startDate: isoDay(-i * CYCLE_LENGTH),
endDate: i > 1 ? isoDay(-(i - 1) * CYCLE_LENGTH - 1) : null,
periodEndDate: isoDay(-i * CYCLE_LENGTH + 4),
tz: "UTC",
},
});
}
const start = `${isoDay(-OBSERVED_CYCLES * CYCLE_LENGTH)} 00:00:00`;
// Sensor glucose every five minutes from two sources, higher in the second
// half of each cycle; heart-rate variability every twenty minutes from a
// watch, lower in the second half; steps every fifteen minutes from a watch
// and a phone; weight once a day next to the Berlin midnight.
await prisma.$executeRawUnsafe(
`INSERT INTO measurements (id, user_id, type, value, unit, source, measured_at, created_at, updated_at, sync_version)
SELECT 'cg' || s || '-' || g, $1, 'BLOOD_GLUCOSE'::measurement_type,
(CASE WHEN (g / 288) % 28 >= 14 THEN 120 ELSE 95 END) + (g % 23) + s * 0.5, 'mg/dL',
(CASE WHEN s = 0 THEN 'APPLE_HEALTH' ELSE 'NIGHTSCOUT' END)::measurement_source,
$2::timestamp + (g * interval '5 minutes') + (s * interval '1 second'),
now(), now(), 1
FROM generate_series(0, ${OBSERVED_CYCLES * CYCLE_LENGTH} * 288 - 1) g, generate_series(0, 1) s`,
OWNER,
start,
);
await prisma.$executeRawUnsafe(
`INSERT INTO measurements (id, user_id, type, value, unit, source, measured_at, created_at, updated_at, sync_version)
SELECT 'hv' || g, $1, 'HEART_RATE_VARIABILITY'::measurement_type,
(CASE WHEN (g / 72) % 28 >= 14 THEN 38 ELSE 52 END) + (g % 11) * 0.7, 'ms',
'APPLE_HEALTH'::measurement_source,
$2::timestamp + (g * interval '20 minutes'), now(), now(), 1
FROM generate_series(0, ${OBSERVED_CYCLES * CYCLE_LENGTH} * 72 - 1) g`,
OWNER,
start,
);
await prisma.$executeRawUnsafe(
`INSERT INTO measurements (id, user_id, type, value, unit, source, device_type, measured_at, created_at, updated_at, sync_version)
SELECT 'st' || d || '-' || g, $1, 'ACTIVITY_STEPS'::measurement_type, 10 + (g % 7), 'count',
'APPLE_HEALTH'::measurement_source, CASE WHEN d = 0 THEN 'watch' ELSE 'phone' END,
$2::timestamp + (g * interval '15 minutes') + (d * interval '2 seconds'),
now(), now(), 1
FROM generate_series(0, ${OBSERVED_CYCLES * CYCLE_LENGTH} * 96 - 1) g, generate_series(0, 1) d
WHERE d = 0 OR g % 3 = 0`,
OWNER,
start,
);
await prisma.$executeRawUnsafe(
`INSERT INTO measurements (id, user_id, type, value, unit, source, measured_at, created_at, updated_at, sync_version)
SELECT 'wt' || g, $1, 'WEIGHT'::measurement_type,
(CASE WHEN g % 28 >= 14 THEN 71.2 ELSE 70.1 END) + (g % 5) * 0.1, 'kg',
'WITHINGS'::measurement_source,
$2::timestamp + (g * interval '1 day') + interval '22 hours 50 minutes', now(), now(), 1
FROM generate_series(0, ${OBSERVED_CYCLES * CYCLE_LENGTH} - 1) g`,
OWNER,
start,
);
await prisma.$executeRawUnsafe(`ANALYZE measurements`);
const session = await prisma.session.create({
data: { userId: OWNER, expiresAt: new Date(Date.now() + 3_600_000) },
});
cookieJar.clear();
cookieJar.set("healthlog_session", session.id);
}, 180_000);
async function readInsights(): Promise<{ status: number; body: unknown }> {
const { GET } = await import("@/app/api/cycle/insights/route");
const handler = GET as unknown as (request: Request) => Promise<Response>;
const res = await handler(new Request("http://localhost/api/cycle/insights"));
return { status: res.status, body: await res.json() };
}
function readsOutcomeRows(args: unknown): boolean {
const type = (args as { where?: { type?: unknown } } | undefined)?.where
?.type;
const list = (type as { in?: unknown } | undefined)?.in;
return (
type === "BLOOD_GLUCOSE" ||
type === "HEART_RATE_VARIABILITY" ||
(Array.isArray(list) &&
(list.includes("BLOOD_GLUCOSE") ||
list.includes("HEART_RATE_VARIABILITY")))
);
}
function expectSameUpToRounding(
actual: unknown,
expected: unknown,
path: string,
): void {
if (typeof expected === "number" && typeof actual === "number") {
const scale = Math.max(Math.abs(expected), Math.abs(actual));
expect(
Math.abs(actual - expected) <= scale * 1e-9,
`${path}: ${actual} vs ${expected}`,
).toBe(true);
return;
}
if (Array.isArray(expected)) {
expect(Array.isArray(actual), path).toBe(true);
expect((actual as unknown[]).length, path).toBe(expected.length);
expected.forEach((e, i) =>
expectSameUpToRounding((actual as unknown[])[i], e, `${path}[${i}]`),
);
return;
}
if (expected !== null && typeof expected === "object") {
expect(actual !== null && typeof actual === "object", path).toBe(true);
const a = actual as Record<string, unknown>;
const e = expected as Record<string, unknown>;
expect(Object.keys(a).sort(), path).toEqual(Object.keys(e).sort());
for (const key of Object.keys(e)) {
expectSameUpToRounding(a[key], e[key], `${path}.${key}`);
}
return;
}
expect(actual, path).toEqual(expected);
}
describe("cycle insights on a dense account", () => {
it("reads no outcome channel as rows", async () => {
mode.raw = false;
const spy = vi.spyOn(prisma.measurement, "findMany");
try {
const started = performance.now();
const { status } = await readInsights();
process.stderr.write(
`[cycle-insights-dense] ${(performance.now() - started).toFixed(0)} ms\n`,
);
expect(status).toBe(200);
expect(spy.mock.calls.filter(([a]) => readsOutcomeRows(a))).toEqual([]);
} finally {
spy.mockRestore();
}
}, 60_000);
it("answers exactly what the raw readings give", async () => {
mode.raw = false;
const folded = await readInsights();
mode.raw = true;
const raw = await readInsights();
mode.raw = false;
expect(folded.status).toBe(200);
const rows = (folded.body as { data: { rows: unknown[] } }).data.rows;
// The fixture is built so the contrast has something to say.
expect(rows.length).toBeGreaterThan(0);
// A day's sum is added in another order in SQL than in JavaScript, so a
// p-value can differ in its last bits; every figure is otherwise equal.
expectSameUpToRounding(folded.body, raw.body, "body");
}, 120_000);
});
tests/integration/dense-series-hrv-spo2.test.ts
+240
View File
@@ -0,0 +1,240 @@
/**
* The per-kind series for heart-rate variability and blood oxygen follows the
* same dense rule as pulse and glucose.
*
* Both used to read every raw row for any window up to 3650 days, and a ring
* or a watch that writes them every minute overnight, or all day, makes that
* the same unbounded read pulse had before #1023. Beyond 90 days they now fold
* per local day; within 90 days a window of more than 10 000 readings folds
* per local hour. A sparse account inside 90 days keeps its raw readings,
* byte for byte, and a bucket has the shape of a reading of its kind (no
* `valueMin` / `valueMax`, which only pulse carries).
*
* The doctor report applies the same rule to both: their sections are a
* statistics row and a series, so a dense window takes the per-day path pulse
* already takes, with exact count, minimum, maximum and latest reading.
*/
import { NextRequest } from "next/server";
import { beforeAll, describe, expect, it, vi } from "vitest";
import { cookieJar } from "./mock-next-headers";
import { getPrismaClient, truncateAllTables } from "./setup";
import { collectDoctorReportData } from "@/lib/doctor-report-data";
import { latestReading } from "@/lib/fhir/resources/common";
import { selectionFromLeaves } from "@/lib/report-selection/selection";
import { invalidateUserTimezone } from "@/lib/tz/resolver";
vi.mock("next/headers", async () => {
const { cookieJar, headerJar } = await import("./mock-next-headers");
return {
headers: vi.fn(async () => ({
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
})),
cookies: vi.fn(async () => ({
get: (name: string) => {
const value = cookieJar.get(name);
return value ? { name, value } : undefined;
},
set: (name: string, value: string) => cookieJar.set(name, value),
delete: (name: string) => cookieJar.delete(name),
})),
};
});
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
const prisma = getPrismaClient();
const TZ = "Europe/Berlin";
const DENSE = "dense-hrv-spo2-owner";
const SPARSE = "sparse-hrv-spo2-owner";
const NOW = new Date(Math.floor(Date.now() / 60_000) * 60_000 - 60_000);
/** Twelve days of one reading a minute: over the 10 000-row cap. */
const DENSE_ROWS = 12 * 24 * 60;
const KINDS = [
{ kind: "heartRateVariability", type: "HEART_RATE_VARIABILITY", unit: "ms" },
{ kind: "oxygenSaturation", type: "OXYGEN_SATURATION", unit: "%" },
] as const;
beforeAll(async () => {
await truncateAllTables(prisma);
for (const id of [DENSE, SPARSE]) {
await prisma.user.create({ data: { id, username: id, timezone: TZ } });
invalidateUserTimezone(id);
}
for (const { type, unit } of KINDS) {
await prisma.$executeRawUnsafe(
`INSERT INTO measurements (
id, user_id, type, value, unit, source, measured_at,
created_at, updated_at, sync_version)
SELECT $4 || lpad(g::text, 7, '0'), $1, $5::measurement_type,
CASE WHEN $5 = 'OXYGEN_SATURATION' THEN 92 + (g % 8) ELSE 30 + (g % 40) * 0.5 END,
$6, 'GOOGLE_HEALTH'::measurement_source,
($2::timestamptz AT TIME ZONE 'UTC') - (g * interval '1 minute'),
now(), now(), 1
FROM generate_series(0, $3::int - 1) g`,
DENSE,
NOW.toISOString(),
DENSE_ROWS,
type === "OXYGEN_SATURATION" ? "ds" : "dh",
type,
unit,
);
// A few readings a night for a month and a half.
await prisma.measurement.createMany({
data: Array.from({ length: 45 * 3 }, (_, i) => ({
id: `sp-${type}-${i}`,
userId: SPARSE,
type,
value: type === "OXYGEN_SATURATION" ? 94 + (i % 5) : 35 + (i % 17),
unit,
source: "MANUAL" as const,
measuredAt: new Date(
NOW.getTime() - Math.floor(i / 3) * 86_400_000 - (i % 3) * 7_200_000,
),
})),
});
}
await prisma.$executeRawUnsafe(`ANALYZE measurements`);
}, 120_000);
async function series(userId: string, kind: string, days: number) {
const s = await prisma.session.create({
data: { userId, expiresAt: new Date(Date.now() + 3_600_000) },
});
cookieJar.clear();
cookieJar.set("healthlog_session", s.id);
const { GET } = await import("@/app/api/measurements/series/route");
const res = await GET(
new NextRequest(
`http://localhost/api/measurements/series?kind=${kind}&days=${days}`,
),
);
expect(res.status).toBe(200);
return (
(await res.json()) as {
data: {
unit: string;
points: Array<Record<string, unknown> & { id: string; value: number }>;
stats: { count: number; mean: number; min: number; max: number };
};
}
).data;
}
describe.each(KINDS)("$kind series", ({ kind, type, unit }) => {
it("sends a dense stream inside 90 days as hour buckets", async () => {
const data = await series(DENSE, kind, 30);
expect(data.unit).toBe(unit);
expect(data.points.length).toBeLessThanOrEqual(12 * 24 + 2);
expect(data.points.length).toBeGreaterThanOrEqual(12 * 24 - 2);
expect(data.points.every((p) => p.id.startsWith("hour:"))).toBe(true);
// A bucket has the shape of a reading of its kind.
expect(
data.points.every((p) => !("valueMin" in p) && !("valueMax" in p)),
).toBe(true);
// Statistics stay over every reading.
const agg = await prisma.measurement.aggregate({
where: {
userId: DENSE,
type,
measuredAt: { gte: new Date(Date.now() - 30 * 86_400_000) },
},
_count: true,
_min: { value: true },
_max: { value: true },
});
expect(data.stats.count).toBe(agg._count);
expect(data.stats.min).toBe(agg._min.value);
expect(data.stats.max).toBe(agg._max.value);
});
it("folds the long ranges per local day", async () => {
const data = await series(SPARSE, kind, 3650);
expect(data.points.length).toBeGreaterThan(40);
expect(data.points.length).toBeLessThanOrEqual(46);
expect(data.points.every((p) => p.id.startsWith("day:"))).toBe(true);
expect(data.stats.count).toBe(45 * 3);
});
it("keeps a sparse series inside 90 days raw", async () => {
const data = await series(SPARSE, kind, 30);
const raw = await prisma.measurement.findMany({
where: {
userId: SPARSE,
type,
measuredAt: { gte: new Date(Date.now() - 30 * 86_400_000) },
},
orderBy: { measuredAt: "asc" },
select: { id: true, value: true, measuredAt: true },
});
expect(data.points).toEqual(
raw.map((r) => ({
id: r.id,
at: r.measuredAt.toISOString(),
value: r.value,
secondary: null,
})),
);
});
});
describe.each(KINDS)("doctor report $type", ({ type }) => {
async function report(userId: string, days: number) {
const end = new Date();
const start = new Date(end.getTime() - days * 86_400_000);
return collectDoctorReportData(
userId,
{ start, end, days },
selectionFromLeaves([type]),
);
}
it("folds a dense window to one point per day, statistics exact", async () => {
const data = await report(DENSE, 30);
const points = data.measurements[type] ?? [];
expect(points.length).toBeLessThanOrEqual(14);
expect(points.length).toBeGreaterThanOrEqual(12);
const agg = await prisma.measurement.aggregate({
where: { userId: DENSE, type },
_count: true,
_avg: { value: true },
_min: { value: true },
_max: { value: true },
});
expect(data.stats[type]?.count).toBe(agg._count);
expect(data.stats[type]?.min).toBe(agg._min.value);
expect(data.stats[type]?.max).toBe(agg._max.value);
expect(data.stats[type]?.avg).toBeCloseTo(agg._avg.value ?? 0, 9);
const newest = await prisma.measurement.findFirstOrThrow({
where: { userId: DENSE, type },
orderBy: { measuredAt: "desc" },
select: { value: true, measuredAt: true },
});
expect(latestReading(data, type)).toEqual({
value: newest.value,
measuredAt: newest.measuredAt.toISOString(),
});
});
it("keeps a sparse window raw", async () => {
const data = await report(SPARSE, 30);
const count = await prisma.measurement.count({
where: {
userId: SPARSE,
type,
measuredAt: { gte: new Date(Date.now() - 30 * 86_400_000) },
},
});
expect(data.measurements[type]).toHaveLength(count);
expect(data.stats[type]?.count).toBe(count);
});
it("folds a window longer than 90 days per day", async () => {
const data = await report(SPARSE, 120);
expect(data.stats[type]?.count).toBe(45 * 3);
expect((data.measurements[type] ?? []).length).toBeLessThanOrEqual(46);
});
});
tests/integration/dense-targets-reads.test.ts
+283
View File
@@ -0,0 +1,283 @@
/**
* The targets page on a dense account: a glucose sensor reporting every five
* minutes and a watch recording heart rate once a minute.
*
* `buildTargetsResponse` read a year of raw glucose readings to keep the
* newest one per meal context, and thirty days of raw heart rate to derive one
* resting figure per day. Both now fold in SQL. These cases pin that neither
* stream is read as rows any more, and that the cards come out exactly as the
* in-memory fold over the raw readings makes them, on a dense account and on
* a sparse one with readings next to the local midnight.
*/
import { beforeAll, describe, expect, it, vi } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import { deriveRestingProxyFromPulse } from "@/lib/analytics/resting-pulse";
import { buildTargetsResponse } from "@/lib/targets/build-response";
import { buildGlucoseTargets } from "@/lib/targets/glucose-builder";
import {
foldGlucoseTargetRows,
readGlucoseTargetSummaries,
} from "@/lib/targets/glucose-read";
import { buildVitalTargets } from "@/lib/targets/vitals-builder";
import { userDayKey } from "@/lib/tz/format";
import { invalidateUserTimezone } from "@/lib/tz/resolver";
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
const prisma = getPrismaClient();
const TZ = "America/New_York";
const DENSE = "dense-targets-owner";
const SPARSE = "sparse-targets-owner";
const NOW = new Date(Math.floor(Date.now() / 60_000) * 60_000 - 60_000);
const DAY = 86_400_000;
beforeAll(async () => {
await truncateAllTables(prisma);
for (const id of [DENSE, SPARSE]) {
await prisma.user.create({
data: {
id,
username: id,
timezone: TZ,
dateOfBirth: new Date("1980-05-01T00:00:00Z"),
gender: "FEMALE",
heightCm: 170,
},
});
invalidateUserTimezone(id);
}
// 200 days of a sensor every five minutes, untagged, with every 37th
// reading tagged fasting so two contexts are present.
await prisma.$executeRawUnsafe(
`INSERT INTO measurements (
id, user_id, type, value, unit, source, glucose_context, measured_at,
created_at, updated_at, sync_version)
SELECT 'dg' || lpad(g::text, 7, '0'), $1, 'BLOOD_GLUCOSE'::measurement_type,
70 + (g % 110) + (g % 3) * 0.25, 'mg/dL', 'NIGHTSCOUT'::measurement_source,
CASE WHEN g % 37 = 0 THEN 'FASTING'::glucose_context ELSE NULL END,
($2::timestamptz AT TIME ZONE 'UTC') - (g * interval '5 minutes'),
now(), now(), 1
FROM generate_series(0, 200 * 288 - 1) g`,
DENSE,
NOW.toISOString(),
);
// Ten days of heart rate once a minute.
await prisma.$executeRawUnsafe(
`INSERT INTO measurements (
id, user_id, type, value, unit, source, measured_at,
created_at, updated_at, sync_version)
SELECT 'dp' || lpad(g::text, 7, '0'), $1, 'PULSE'::measurement_type,
55 + (g % 50), 'bpm', 'GOOGLE_HEALTH'::measurement_source,
($2::timestamptz AT TIME ZONE 'UTC') - (g * interval '1 minute'),
now(), now(), 1
FROM generate_series(0, 10 * 24 * 60 - 1) g`,
DENSE,
NOW.toISOString(),
);
// Sparse: fingerstick readings in three contexts, a few a day, some older
// than thirty days, some within minutes of the New York midnight; a cuff a
// few times a day; a resting heart rate on alternate days.
const contexts = ["FASTING", "POSTPRANDIAL", null] as const;
const sparse: Array<{
type: "BLOOD_GLUCOSE" | "PULSE" | "RESTING_HEART_RATE";
value: number;
at: Date;
ctx?: (typeof contexts)[number];
}> = [];
for (let day = 0; day < 60; day++) {
for (let i = 0; i < 1 + (day % 3); i++) {
sparse.push({
type: "BLOOD_GLUCOSE",
value: 85 + ((day * 13 + i * 29) % 90) + (i % 2 ? 0.5 : 0),
at: new Date(NOW.getTime() - day * DAY - (i * 311 + 7) * 60_000),
ctx: contexts[(day + i) % 3],
});
}
if (day < 25) {
for (let i = 0; i < 2 + (day % 4); i++) {
sparse.push({
type: "PULSE",
value: 54 + ((day * 7 + i * 11) % 40),
at: new Date(NOW.getTime() - day * DAY - (i * 173 + 3) * 60_000),
});
}
}
if (day % 2 === 0 && day < 30) {
sparse.push({
type: "RESTING_HEART_RATE",
value: 58 + (day % 6),
at: new Date(NOW.getTime() - day * DAY - 4 * 3_600_000),
});
}
}
await prisma.measurement.createMany({
data: sparse.map((r, i) => ({
id: `st${String(i).padStart(5, "0")}`,
userId: SPARSE,
type: r.type,
value: r.value,
unit: r.type === "BLOOD_GLUCOSE" ? "mg/dL" : "bpm",
source: "MANUAL" as const,
glucoseContext: r.ctx ?? null,
measuredAt: r.at,
})),
});
await prisma.$executeRawUnsafe(`ANALYZE measurements`);
}, 180_000);
function readsDenseRows(args: unknown): boolean {
const type = (args as { where?: { type?: unknown } } | undefined)?.where
?.type;
if (type === "BLOOD_GLUCOSE" || type === "PULSE") return true;
const list = (type as { in?: unknown } | undefined)?.in;
return (
Array.isArray(list) &&
(list.includes("PULSE") || list.includes("BLOOD_GLUCOSE"))
);
}
describe("targets on a dense account", () => {
it("never reads the glucose or heart-rate stream as rows", async () => {
const spy = vi.spyOn(prisma.measurement, "findMany");
try {
const started = performance.now();
const res = await buildTargetsResponse({ id: DENSE, timezone: TZ });
const elapsed = performance.now() - started;
process.stderr.write(
`[dense-targets] buildTargetsResponse ${elapsed.toFixed(0)} ms\n`,
);
expect(res.targets.length).toBeGreaterThan(0);
const denseReads = spy.mock.calls.filter(([args]) =>
readsDenseRows(args),
);
expect(denseReads).toEqual([]);
} finally {
spy.mockRestore();
}
});
});
describe("targets fold equals the raw-reading fold", () => {
for (const userId of [SPARSE, DENSE]) {
it(`glucose summaries match the in-memory fold (${userId})`, async () => {
const since = new Date(NOW.getTime() - 365 * DAY);
const recentSince = new Date(NOW.getTime() - 30 * DAY);
const raw = await prisma.measurement.findMany({
where: {
userId,
type: "BLOOD_GLUCOSE",
deletedAt: null,
measuredAt: { gte: since },
},
orderBy: [{ measuredAt: "desc" }, { id: "desc" }],
select: { value: true, measuredAt: true, glucoseContext: true },
});
const expected = foldGlucoseTargetRows(raw, recentSince, TZ);
const actual = await readGlucoseTargetSummaries({
userId,
since,
recentSince,
timeZone: TZ,
});
const byCtx = <T extends { glucoseContext: string | null }>(xs: T[]) =>
[...xs].sort((a, b) =>
(a.glucoseContext ?? "").localeCompare(b.glucoseContext ?? ""),
);
expect(actual.length).toBeGreaterThanOrEqual(2);
const exp = byCtx(expected);
const act = byCtx(actual);
expect(act.map((s) => s.glucoseContext)).toEqual(
exp.map((s) => s.glucoseContext),
);
for (let i = 0; i < exp.length; i++) {
expect(act[i].latest).toBe(exp[i].latest);
expect(act[i].latestAt).toEqual(exp[i].latestAt);
expect(act[i].recentCount).toBe(exp[i].recentCount);
expect([...act[i].recentByDay.keys()].sort()).toEqual(
[...exp[i].recentByDay.keys()].sort(),
);
for (const [day, bucket] of exp[i].recentByDay) {
const got = act[i].recentByDay.get(day)!;
expect(got.count, day).toBe(bucket.count);
expect(got.sum, day).toBeCloseTo(bucket.sum, 6);
}
}
});
it(`glucose and resting-pulse cards equal the raw path (${userId})`, async () => {
const res = await buildTargetsResponse({ id: userId, timezone: TZ });
// The builder takes its own clock after the reads; rebuild the raw
// expectation against the same thirty-day cut.
const now = new Date();
const recentSince = new Date(now.getTime() - 30 * DAY);
const raw = await prisma.measurement.findMany({
where: {
userId,
type: "BLOOD_GLUCOSE",
deletedAt: null,
measuredAt: { gte: new Date(now.getTime() - 365 * DAY) },
},
orderBy: [{ measuredAt: "desc" }, { id: "desc" }],
select: { value: true, measuredAt: true, glucoseContext: true },
});
const expectedGlucose = buildGlucoseTargets({
summaries: foldGlucoseTargetRows(raw, recentSince, TZ),
profile: {
heightCm: 170,
dateOfBirth: new Date("1980-05-01T00:00:00Z"),
gender: "FEMALE",
glucoseUnit: null,
hasDiabetes: false,
thresholdsJson: null,
},
timezone: TZ,
now,
});
const glucose = res.targets.filter((t) =>
t.type.startsWith("BLOOD_GLUCOSE_"),
);
expect(glucose.length).toBe(expectedGlucose.length);
for (let i = 0; i < glucose.length; i++) {
const { average30, ...rest } = glucose[i];
const { average30: expAvg, ...expRest } = expectedGlucose[i];
expect(rest).toEqual(expRest);
if (expAvg === null) expect(average30).toBeNull();
else expect(average30).toBeCloseTo(expAvg, 6);
}
const recent = await prisma.measurement.findMany({
where: {
userId,
type: { in: ["PULSE", "RESTING_HEART_RATE"] },
deletedAt: null,
measuredAt: { gte: recentSince },
},
select: { type: true, value: true, measuredAt: true },
});
const expectedPulse = buildVitalTargets({
recentMeasurements: recent.filter(
(r) => r.type === "RESTING_HEART_RATE",
),
restingPulseProxy: deriveRestingProxyFromPulse(
recent.filter((r) => r.type === "PULSE"),
(d) => userDayKey(d, TZ),
),
latestByType: {},
average30ByType: {},
heightCm: 170,
age: res.profile.age,
gender: "FEMALE",
timezone: TZ,
now,
weightTargetOverride: null,
}).targets.find((t) => t.type === "PULSE");
const pulse = res.targets.find((t) => t.type === "PULSE");
expect(pulse?.current).not.toBeNull();
expect(pulse).toEqual(expectedPulse);
});
}
});
tests/integration/measurements-sleep-list.test.ts
+58 -1
View File
@@ -56,7 +56,7 @@ interface ListEnvelope {
napCount?: number;
awakenings?: number;
}>;
meta: { total: number; groupBy?: string };
meta: { total: number; groupBy?: string; windowFrom?: string };
};
}
@@ -160,4 +160,61 @@ describe("GET /api/measurements?type=SLEEP_DURATION", () => {
const stages = body.data.measurements.map((m) => m.sleepStage).sort();
expect(stages).toEqual(["CORE", "DEEP", "REM"]);
});
it("reads at most a year ending at `to`, however early `from` is", async () => {
// An explicit early `from` used to walk every per-stage row back to it.
const prisma = getPrismaClient();
const user = await seedSession("sleep-window-user");
const day = 86_400_000;
const morning = new Date();
morning.setUTCHours(4, 0, 0, 0);
// One night of a single CORE stage every day for two years.
await prisma.measurement.createMany({
data: Array.from({ length: 730 }, (_, i) => ({
userId: user.id,
type: "SLEEP_DURATION" as const,
value: 420,
unit: "minutes",
source: "APPLE_HEALTH" as const,
measuredAt: new Date(morning.getTime() - i * day),
externalId: `uuid-night-${i}`,
sleepStage: "CORE" as const,
})),
});
const spy = vi.spyOn(prisma.measurement, "findMany");
const from = new Date(morning.getTime() - 800 * day).toISOString();
const res = await callList(
`type=SLEEP_DURATION&from=${encodeURIComponent(from)}&limit=1000`,
);
expect(res.status).toBe(200);
const body = (await res.json()) as ListEnvelope;
expect(body.data.meta.total).toBeLessThanOrEqual(366);
expect(body.data.meta.total).toBeGreaterThanOrEqual(364);
expect(body.data.meta.windowFrom).toBeDefined();
const windowFrom = Date.parse(body.data.meta.windowFrom!);
expect(Date.now() - windowFrom).toBeLessThan(366 * day);
expect(Date.now() - windowFrom).toBeGreaterThan(364 * day);
// The read itself was bounded, not just the answer.
const read = spy.mock.calls
.map(([args]) => args as { where?: { measuredAt?: { gte?: Date } } })
.find((args) => args?.where?.measuredAt?.gte);
expect(read?.where?.measuredAt?.gte?.getTime()).toBe(windowFrom);
// Older nights are reached by moving `to` back.
const to = new Date(morning.getTime() - 400 * day).toISOString();
const older = await callList(
`type=SLEEP_DURATION&from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}&limit=1000`,
);
const olderBody = (await older.json()) as ListEnvelope;
expect(olderBody.data.meta.total).toBeGreaterThanOrEqual(329);
expect(olderBody.data.meta.total).toBeLessThanOrEqual(331);
// A window inside the year is untouched and carries no marker.
const recent = await callList(
`type=SLEEP_DURATION&from=${encodeURIComponent(new Date(morning.getTime() - 10.5 * day).toISOString())}`,
);
const recentBody = (await recent.json()) as ListEnvelope;
expect(recentBody.data.meta.total).toBe(11);
expect(recentBody.data.meta.windowFrom).toBeUndefined();
});
});