From bb1acfab642edf35cd44cb494b89a497d22ab1a0 Mon Sep 17 00:00:00 2001 From: BernhardBuckel <33808384+BernhardBuckel@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:18:50 +0200 Subject: [PATCH 01/14] fix(series): follow the source priority ladder in the dense and raw series reads The rollup tier and sleep resolve every (type, day) to the source the person's ladder picks before they aggregate. The dense and raw reads of /api/measurements/series filtered on user and type only, so with a second provider a day's value blended the two, a reading present in both showed twice, and the configured ladder had no effect on those charts. Both reads now take the ladder-winning source per day through the shared canonicalMeasurementsCte helper, as the other live readers do: - the day and hour bucket read of the dense kinds (pulse, glucose, HRV, SpO2), and the stats strip over its rows; - the generic raw read (weight, body fat, steps, resting heart rate, ...). The collapse is per day for the hour grain too, as everywhere else. The helper gains an optional single-type scope so a series scans one metric, not every row the person has in the window. Only a type that has a ladder is collapsed. For a type without one (glucose, body water, bone mass) the helper breaks the tie by source name, which would silently hide one of two real sources, so those keep every source as before. A single-source account reads exactly what is stored. Tests: a real-database test covering the ladder in both directions, the per-day fallback, a reading mirrored into two sources, the day-bucket, hour- bucket and raw reads, a type with no ladder and a single-source account. The route's unit test file reads its fixtures through a small adapter for the new raw statements, and the dense-read guard drops the exception for the removed findMany. The OpenAPI description states the rule. Refs #1108 --- docs/api/openapi.yaml | 5 +- .../dense-measurement-read-guard.test.ts | 2 - .../series/__tests__/route.test.ts | 74 ++++- .../unknown-query-param-open.test.ts | 2 + src/app/api/measurements/series/route.ts | 102 ++++--- src/lib/analytics/source-rank-sql.ts | 20 +- .../__tests__/series-canonical.test.ts | 74 +++++ src/lib/measurements/series-canonical.ts | 56 ++++ src/lib/openapi/routes/measurements.ts | 2 +- .../integration/series-source-ladder.test.ts | 286 ++++++++++++++++++ 10 files changed, 566 insertions(+), 57 deletions(-) create mode 100644 src/lib/measurements/__tests__/series-canonical.test.ts create mode 100644 src/lib/measurements/series-canonical.ts create mode 100644 tests/integration/series-source-ladder.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index f41b8d7cb..59d50a74c 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -3885,7 +3885,10 @@ paths: explicit `unit` token and a summary `stats` block. `bloodPressure` pairs systolic + diastolic (`secondary`) within ±5 min so one fetch renders the dual-line chart; `sleep` collapses per-stage rows into one per-night point carrying time-asleep in hours (`sleepStages` holds the per-stage breakdown) and is internally capped to - 365 days. `glucose` values + `unit` resolve to the user's mg/dL|mmol/L preference. Auth via cookie or Bearer. + 365 days. `glucose` values + `unit` resolve to the user's mg/dL|mmol/L preference. Where a metric has a + source-priority ladder (pulse, weight, SpO2, HRV, ...), each day shows the readings of the source the person's + ladder picks for that day, so a second provider never blends into or duplicates a day; a metric without a ladder + (glucose, body water, bone mass) keeps every source. Auth via cookie or Bearer. parameters: - in: query name: kind diff --git a/src/__tests__/dense-measurement-read-guard.test.ts b/src/__tests__/dense-measurement-read-guard.test.ts index 35060ade2..224bfc15b 100644 --- a/src/__tests__/dense-measurement-read-guard.test.ts +++ b/src/__tests__/dense-measurement-read-guard.test.ts @@ -101,8 +101,6 @@ const ALLOWED: Record = { "Health-score inputs: steps (drained to one row per day nightly), sleep, waist, weight, blood pressure, fasting glucose only.", 'app/api/analytics/route.ts::"BLOOD_GLUCOSE"': "The 30-day per-context glucose summary, the same window and bound as the dashboard and coach glucose panels: at a sensor's fixed rate, below 45 000 rows.", - "app/api/measurements/series/route.ts::(shorthand)": - "The raw branch of the per-kind series. Every dense kind reaches it only inside 90 days and under 10 000 rows (counted first); the sparse kinds are a few readings a day.", "app/api/measurements/batch/route.ts::p.row.type as MeasurementType": "Cross-source merge probe: one ±tolerance window around each posted reading's instant, so the rows found are bounded by the batch size.", "app/api/measurements/route.ts::(shorthand)": diff --git a/src/app/api/measurements/series/__tests__/route.test.ts b/src/app/api/measurements/series/__tests__/route.test.ts index cd220128a..37146ef82 100644 --- a/src/app/api/measurements/series/__tests__/route.test.ts +++ b/src/app/api/measurements/series/__tests__/route.test.ts @@ -9,6 +9,9 @@ vi.mock("@/lib/db", () => ({ // v1.28.25 — dense kinds (glucose / pulse) day-bucket in SQL for // windows beyond the 90-day raw cap. $queryRaw: vi.fn(), + // The series reads go through the ladder-winning source per day (see + // `seriesRowsFrom`), which is raw SQL with the user id as `$1`. + $queryRawUnsafe: vi.fn(), }, })); @@ -65,8 +68,47 @@ function req(query: string): NextRequest { return new NextRequest(`http://localhost/api/measurements/series?${query}`); } +/** + * The route reads its series through `$queryRawUnsafe` (the ladder-winning + * source per day). This file's fixtures are written as rows per type and as the + * two aggregate results, so the adapter routes each statement back to them: + * the day/hour bucket and the stats aggregate take the next `$queryRaw` result, + * and a raw read asks `findMany` for the type named in the SQL and returns the + * rows in the column names the statement selects. The SQL itself (which source + * wins, which rows count) is covered against a real database in + * `tests/integration/series-source-ladder.test.ts`. + */ +function adaptSeriesSql(sql: string): Promise { + if (/GROUP BY local_day|STDDEV_POP/.test(sql)) { + return (prisma.$queryRaw as unknown as () => Promise)(); + } + const type = /'([A-Z0-9_]+)'::"measurement_type"/.exec(sql)?.[1]; + const find = prisma.measurement.findMany as unknown as ( + a: unknown, + ) => Promise< + Array<{ + id: string; + value: number; + measuredAt: Date; + valueMin?: number | null; + valueMax?: number | null; + }> + >; + return find({ where: { type } }).then((rows) => + rows.map((r) => ({ + id: r.id, + value: r.value, + measured_at: r.measuredAt, + value_min: r.valueMin ?? null, + value_max: r.valueMax ?? null, + })), + ); +} + beforeEach(() => { vi.resetAllMocks(); + vi.mocked(prisma.$queryRawUnsafe).mockImplementation(((sql: string) => + adaptSeriesSql(sql)) as never); vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); vi.mocked(prisma.measurement.count).mockResolvedValue(0 as never); vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); @@ -216,14 +258,14 @@ describe("GET /api/measurements/series", () => { expect(body.data.points[1].valueMin).toBeNull(); expect(body.data.points[1].valueMax).toBeNull(); - // The pulse read selects the spread columns. - const selectArg = vi.mocked(prisma.measurement.findMany).mock.calls[0][0]; - expect( - (selectArg as { select: Record }).select.valueMin, - ).toBe(true); - expect( - (selectArg as { select: Record }).select.valueMax, - ).toBe(true); + // The pulse read selects the spread columns, and reads the pulse rows of + // this person's ladder-winning source (the user id is the first parameter). + const [sql, userId] = vi.mocked(prisma.$queryRawUnsafe).mock + .calls[0] as unknown as [string, string]; + expect(sql).toContain('m."value_min"'); + expect(sql).toContain('m."value_max"'); + expect(sql).toContain(`'PULSE'::"measurement_type"`); + expect(userId).toBe("user-1"); }); it("collapses sleep stage rows into one night point in hours (v1.11.4)", async () => { @@ -627,8 +669,12 @@ describe("GET /api/measurements/series — dense-kind day-bucketing (v1.28.25)", expect(body.data.stats.count).toBe(43_200); expect(body.data.stats.mean).toBe(72.1); // The bucket read folds per hour, not per day. - const bucketSql = vi.mocked(prisma.$queryRaw).mock.calls[0] as unknown[]; - expect(bucketSql.slice(1)).toContain("hour"); + const bucketCall = vi + .mocked(prisma.$queryRawUnsafe) + .mock.calls.find(([sql]) => + /GROUP BY local_day/.test(sql as string), + ) as unknown[]; + expect(bucketCall.slice(1)).toContain("hour"); }); it("hour-buckets a CGM glucose window too dense to send raw, in the user's unit", async () => { @@ -690,8 +736,12 @@ describe("GET /api/measurements/series — dense-kind day-bucketing (v1.28.25)", max: 14.4, stdDev: 2, }); - const bucketSql = vi.mocked(prisma.$queryRaw).mock.calls[0] as unknown[]; - expect(bucketSql.slice(1)).toContain("hour"); + const bucketCall = vi + .mocked(prisma.$queryRawUnsafe) + .mock.calls.find(([sql]) => + /GROUP BY local_day/.test(sql as string), + ) as unknown[]; + expect(bucketCall.slice(1)).toContain("hour"); }); it("keeps fingerstick glucose at or under the row cap raw", async () => { diff --git a/src/app/api/measurements/series/__tests__/unknown-query-param-open.test.ts b/src/app/api/measurements/series/__tests__/unknown-query-param-open.test.ts index d51eda7e9..b852744ff 100644 --- a/src/app/api/measurements/series/__tests__/unknown-query-param-open.test.ts +++ b/src/app/api/measurements/series/__tests__/unknown-query-param-open.test.ts @@ -22,6 +22,7 @@ vi.mock("@/lib/db", () => ({ user: { findUnique: vi.fn() }, auditLog: { create: vi.fn() }, $queryRaw: vi.fn(), + $queryRawUnsafe: vi.fn(), }, })); @@ -74,6 +75,7 @@ beforeEach(() => { vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); + vi.mocked(prisma.$queryRawUnsafe).mockResolvedValue([] as never); vi.mocked(prisma.auditLog.create).mockResolvedValue({} as never); vi.mocked(prisma.user.findUnique).mockResolvedValue({ glucoseUnit: "mg/dL", diff --git a/src/app/api/measurements/series/route.ts b/src/app/api/measurements/series/route.ts index f42afb47c..35b22894e 100644 --- a/src/app/api/measurements/series/route.ts +++ b/src/app/api/measurements/series/route.ts @@ -26,6 +26,7 @@ import type { MeasurementType, SleepStage } from "@/generated/prisma/client"; import { reconstructSleepNights } from "@/lib/analytics/sleep-night"; import { VALUE_RANGES } from "@/lib/validations/measurement"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { seriesRowsFrom } from "@/lib/measurements/series-canonical"; import { resolveUserTimezone, userDayKey } from "@/lib/tz/resolver"; import { convertGlucose, resolveGlucoseUnit } from "@/lib/glucose"; @@ -399,49 +400,57 @@ export const GET = apiHandler(async (request: NextRequest) => { // v1.28.25 — long-window read of a sample-dense kind (CGM glucose, // per-sample / hourly pulse). Day-bucket in SQL instead of walking // every raw row into JS: one aggregate pass in Postgres returns at - // most `days` rows regardless of sample density. Parameter-bound - // tagged-template `$queryRaw`, mirroring the rollup tier's - // `date_trunc(... ) GROUP BY` aggregate (measurement-rollups.ts). - const userTz = await resolveUserTimezone(user.id); + // most `days` rows regardless of sample density. Bound parameters + // mirror the rollup tier's `date_trunc(... ) GROUP BY` aggregate + // (measurement-rollups.ts), and the rows are the ladder-winning source per + // day, the same rule the rollup tier and sleep apply. + const [userTz, priorityJson] = await Promise.all([ + resolveUserTimezone(user.id), + loadUserSourcePriority(user.id), + ]); const type = KIND_TO_TYPE[kind]; - const bucketRows = await prisma.$queryRaw< + const seriesRows = seriesRowsFrom(priorityJson, type, days); + const bucketRows = await prisma.$queryRawUnsafe< Array<{ bucket_start: Date; mean: number; min_value: number; max_value: number; }> - >` + >( + ` WITH localized AS ( SELECT m."value", m."value_min", m."value_max", date_trunc( - ${grain}, - (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${userTz} + $2, + (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE $3 ) AS local_day - FROM measurements m - WHERE m."user_id" = ${user.id} - AND m."type" = ${type}::"measurement_type" - AND m."measured_at" >= ${since} - AND m."deleted_at" IS NULL + FROM ${seriesRows} + WHERE m."measured_at" >= $4 ) SELECT - local_day AT TIME ZONE ${userTz} AS bucket_start, + local_day AT TIME ZONE $3 AS bucket_start, AVG("value")::double precision AS mean, MIN(COALESCE("value_min", "value"))::double precision AS min_value, MAX(COALESCE("value_max", "value"))::double precision AS max_value FROM localized GROUP BY local_day ORDER BY bucket_start ASC - `; + `, + user.id, + grain, + userTz, + since, + ); // Stats stay aggregated over the RAW rows (not the day buckets) so the // strip's mean/min/max/stdDev/count are the same figures the raw path // computed: `AVG`/`MIN`/`MAX` over `value`, `STDDEV_POP` matching the // JS population-variance helper, exact row count. Aggregate-only — // no row transfer. - const [rawAgg] = await prisma.$queryRaw< + const [rawAgg] = await prisma.$queryRawUnsafe< Array<{ n: number; mean: number | null; @@ -449,19 +458,20 @@ export const GET = apiHandler(async (request: NextRequest) => { max: number | null; sd: number | null; }> - >` + >( + ` SELECT COUNT(*)::int AS n, AVG(m."value")::double precision AS mean, MIN(m."value")::double precision AS min, MAX(m."value")::double precision AS max, STDDEV_POP(m."value")::double precision AS sd - FROM measurements m - WHERE m."user_id" = ${user.id} - AND m."type" = ${type}::"measurement_type" - AND m."measured_at" >= ${since} - AND m."deleted_at" IS NULL - `; + FROM ${seriesRows} + WHERE m."measured_at" >= $2 + `, + user.id, + since, + ); const round2 = (v: number) => Math.round(v * 100) / 100; const dayId = (d: Date) => grain === "day" @@ -538,21 +548,37 @@ export const GET = apiHandler(async (request: NextRequest) => { // hourly average. Only PULSE rows ever carry a non-null spread (the // hourly HR bucket); every other kind selects the bare value shape. const includeSpread = kind === "pulse"; - const rows = await prisma.measurement.findMany({ - where: { - userId: user.id, - type, - measuredAt: { gte: since }, - deletedAt: null, - }, - orderBy: { measuredAt: "asc" }, - select: { - id: true, - value: true, - measuredAt: true, - ...(includeSpread ? { valueMin: true, valueMax: true } : {}), - }, - }); + // The ladder-winning source per day (see `seriesRowsFrom`), so a second + // provider neither blends into nor duplicates the series. Bounded as before: + // every dense kind reaches this read only inside 90 days and under 10 000 + // rows (counted first), and the sparse kinds are a few readings a day. + const sourcePriority = await loadUserSourcePriority(user.id); + const rawRows = await prisma.$queryRawUnsafe< + Array<{ + id: string; + value: number; + measured_at: Date; + value_min: number | null; + value_max: number | null; + }> + >( + ` + SELECT m."id", m."value", m."measured_at", m."value_min", m."value_max" + FROM ${seriesRowsFrom(sourcePriority, type, days)} + WHERE m."measured_at" >= $2 + ORDER BY m."measured_at" ASC, m."id" ASC + `, + user.id, + since, + ); + const rows = rawRows.map((r) => ({ + id: r.id, + value: r.value, + measuredAt: r.measured_at, + ...(includeSpread + ? { valueMin: r.value_min, valueMax: r.value_max } + : {}), + })); if (kind === "glucose") { // v1.16.16 — glucose is stored canonical mg/dL; convert each point to // the user's display unit AT SERIALIZATION so the wire DTO is unit- diff --git a/src/lib/analytics/source-rank-sql.ts b/src/lib/analytics/source-rank-sql.ts index 664b98170..a0f02cec4 100644 --- a/src/lib/analytics/source-rank-sql.ts +++ b/src/lib/analytics/source-rank-sql.ts @@ -123,11 +123,13 @@ function assertInterval(value: string): string { * The user id is bound as `$1`. `rankUnqualified` must be a CASE built with * unqualified `"type"`/`"source"` columns. `sinceInterval` (e.g. `"90 days"`) * is whitelisted and, when given, scopes both the inner pick and the outer - * filter to a trailing window. + * filter to a trailing window. `onlyType`, when given, scopes both halves to + * that one metric. */ export function canonicalMeasurementsCte( rankUnqualified: string, sinceInterval?: string, + onlyType?: MeasurementType, ): string { const sinceInner = sinceInterval ? `AND "measured_at" >= NOW() - INTERVAL '${assertInterval(sinceInterval)}'` @@ -135,6 +137,15 @@ export function canonicalMeasurementsCte( const sinceOuter = sinceInterval ? `AND mm."measured_at" >= NOW() - INTERVAL '${assertInterval(sinceInterval)}'` : ""; + // A single-type reader (a series for one metric) scopes both halves to that + // type, so the pick scans one metric's rows instead of every row the user + // has in the window. + const typeInner = onlyType + ? `AND "type" = '${assertEnumLiteral(onlyType)}'::"measurement_type"` + : ""; + const typeOuter = onlyType + ? `AND mm."type" = '${assertEnumLiteral(onlyType)}'::"measurement_type"` + : ""; return ` SELECT mm.* FROM measurements mm @@ -147,6 +158,7 @@ export function canonicalMeasurementsCte( WHERE "user_id" = $1 AND "deleted_at" IS NULL ${sinceInner} + ${typeInner} ORDER BY "type", date_trunc('day', "measured_at"), (${rankUnqualified}), "source"::text ) c ON c.t = mm."type" @@ -154,7 +166,8 @@ export function canonicalMeasurementsCte( AND c.canon = mm."source" WHERE mm."user_id" = $1 AND mm."deleted_at" IS NULL - ${sinceOuter}`; + ${sinceOuter} + ${typeOuter}`; } /** @@ -173,7 +186,8 @@ export function canonicalMeasurementsCte( export function canonicalMeasurementsFrom( rankUnqualified: string, sinceInterval?: string, + onlyType?: MeasurementType, ): string { - return `(${canonicalMeasurementsCte(rankUnqualified, sinceInterval)} + return `(${canonicalMeasurementsCte(rankUnqualified, sinceInterval, onlyType)} ) m`; } diff --git a/src/lib/measurements/__tests__/series-canonical.test.ts b/src/lib/measurements/__tests__/series-canonical.test.ts new file mode 100644 index 000000000..6f0da6222 --- /dev/null +++ b/src/lib/measurements/__tests__/series-canonical.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; + +import { canonicalMeasurementsCte } from "@/lib/analytics/source-rank-sql"; + +import { seriesRowsFrom } from "../series-canonical"; + +describe("seriesRowsFrom", () => { + it("collapses a type that has a ladder to the ladder's source per day", () => { + const sql = seriesRowsFrom(null, "PULSE", 30); + expect(sql).toContain("DISTINCT ON"); + // Scoped to the one metric, in both halves of the helper. + expect(sql).toContain(`"type" = 'PULSE'::"measurement_type"`); + expect(sql).toContain(`mm."type" = 'PULSE'::"measurement_type"`); + expect(sql.trimEnd().endsWith(") m")).toBe(true); + }); + + it("reads one day wider than the request, so the edge day is picked from all its readings", () => { + expect(seriesRowsFrom(null, "WEIGHT", 30)).toContain("INTERVAL '31 days'"); + expect(seriesRowsFrom(null, "WEIGHT", 3650)).toContain( + "INTERVAL '3651 days'", + ); + }); + + it("applies the person's ladder, not only the default one", () => { + const first = seriesRowsFrom( + { pulse: ["FITBIT", "APPLE_HEALTH"] }, + "PULSE", + 7, + ); + const second = seriesRowsFrom( + { pulse: ["APPLE_HEALTH", "FITBIT"] }, + "PULSE", + 7, + ); + expect(first).not.toEqual(second); + expect(first).toMatch(/WHEN 'FITBIT' THEN 0 WHEN 'APPLE_HEALTH' THEN 1/); + expect(second).toMatch(/WHEN 'APPLE_HEALTH' THEN 0 WHEN 'FITBIT' THEN 1/); + }); + + it.each(["BLOOD_GLUCOSE", "TOTAL_BODY_WATER", "BONE_MASS"] as const)( + "keeps every source for %s, which has no ladder", + (type) => { + const sql = seriesRowsFrom(null, type, 30); + expect(sql).not.toContain("DISTINCT ON"); + expect(sql).toContain(`mm."type" = '${type}'::"measurement_type"`); + expect(sql).toContain(`mm."user_id" = $1`); + expect(sql).toContain(`mm."deleted_at" IS NULL`); + }, + ); + + it.each([0, -1, 1.5, 3651, Number.NaN])("refuses days = %s", (days) => { + expect(() => seriesRowsFrom(null, "PULSE", days)).toThrow(/days/); + }); +}); + +describe("canonicalMeasurementsCte, one-metric scope", () => { + const rank = `CASE "source" WHEN 'A' THEN 0 ELSE 90 END`; + + it("is unchanged when no type is given", () => { + const sql = canonicalMeasurementsCte(rank, "90 days"); + expect(sql).not.toContain("measurement_type"); + }); + + it("scopes the pick and the join to the type when one is given", () => { + const sql = canonicalMeasurementsCte(rank, "90 days", "PULSE"); + expect(sql.match(/'PULSE'::"measurement_type"/g)).toHaveLength(2); + }); + + it("refuses anything that is not a closed-enum literal", () => { + expect(() => + canonicalMeasurementsCte(rank, undefined, "PULSE'; DROP" as never), + ).toThrow(/unsafe enum literal/); + }); +}); diff --git a/src/lib/measurements/series-canonical.ts b/src/lib/measurements/series-canonical.ts new file mode 100644 index 000000000..963703aeb --- /dev/null +++ b/src/lib/measurements/series-canonical.ts @@ -0,0 +1,56 @@ +/** + * The rows a metric series reads, restricted to the source the user's priority + * ladder picks for each day. + * + * The rollup tier and sleep already resolve every (type, day) to the + * ladder-winning source before they aggregate. The dense and raw reads of + * `/api/measurements/series` filtered on user and type only, so with a second + * provider a day's value blended the two and the configured ladder had no + * effect on those charts. This is the one place that applies the same rule to + * them, through the same SQL helper the other live readers use. + * + * Only a type that HAS a ladder is collapsed (`RANKED_TYPES`). For a type + * without one (glucose, body water, bone mass) the helper would break the tie + * by source name, which silently hides one of two real sources, so those keep + * every source exactly as before. + * + * Returns a FROM-clause subquery aliased `m`; the user id is bound as `$1`. + */ +import type { MeasurementType } from "@/generated/prisma/client"; +import { + RANKED_TYPES, + buildSourceRankCase, + canonicalMeasurementsFrom, +} from "@/lib/analytics/source-rank-sql"; + +const ENUM_RE = /^[A-Z0-9_]+$/; + +export function seriesRowsFrom( + priorityJson: unknown, + type: MeasurementType, + days: number, +): string { + if (!ENUM_RE.test(type)) { + throw new Error(`unsafe enum literal for SQL splice: ${type}`); + } + if (!Number.isInteger(days) || days < 1 || days > 3650) { + throw new Error(`days out of range for SQL splice: ${days}`); + } + if (!RANKED_TYPES.includes(type)) { + return `( + SELECT mm.* + FROM measurements mm + WHERE mm."user_id" = $1 + AND mm."type" = '${type}'::"measurement_type" + AND mm."deleted_at" IS NULL + ) m`; + } + // One day wider than the request: the caller trims to its exact `since`, so + // the edge day's source is picked from all of its readings, not from the + // slice the interval happens to include. + return canonicalMeasurementsFrom( + buildSourceRankCase(priorityJson, '"type"', '"source"'), + `${days + 1} days`, + type, + ); +} diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index dea56ced2..c1a077865 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -840,7 +840,7 @@ export const measurementPaths: NonNullable = { tags: ["Measurements"], summary: "Per-kind time series (iOS chart source)", description: - "Maps a camelCase `kind` to the canonical MeasurementType(s) and returns an ordered point series with an explicit `unit` token and a summary `stats` block. `bloodPressure` pairs systolic + diastolic (`secondary`) within ±5 min so one fetch renders the dual-line chart; `sleep` collapses per-stage rows into one per-night point carrying time-asleep in hours (`sleepStages` holds the per-stage breakdown) and is internally capped to 365 days. `glucose` values + `unit` resolve to the user's mg/dL|mmol/L preference. Auth via cookie or Bearer.", + "Maps a camelCase `kind` to the canonical MeasurementType(s) and returns an ordered point series with an explicit `unit` token and a summary `stats` block. `bloodPressure` pairs systolic + diastolic (`secondary`) within ±5 min so one fetch renders the dual-line chart; `sleep` collapses per-stage rows into one per-night point carrying time-asleep in hours (`sleepStages` holds the per-stage breakdown) and is internally capped to 365 days. `glucose` values + `unit` resolve to the user's mg/dL|mmol/L preference. Where a metric has a source-priority ladder (pulse, weight, SpO2, HRV, ...), each day shows the readings of the source the person's ladder picks for that day, so a second provider never blends into or duplicates a day; a metric without a ladder (glucose, body water, bone mass) keeps every source. Auth via cookie or Bearer.", requestParams: { query: seriesQuerySchema, }, diff --git a/tests/integration/series-source-ladder.test.ts b/tests/integration/series-source-ladder.test.ts new file mode 100644 index 000000000..efd914c06 --- /dev/null +++ b/tests/integration/series-source-ladder.test.ts @@ -0,0 +1,286 @@ +/** + * The per-kind series follows the source-priority ladder. + * + * The rollup tier and sleep resolve every (type, day) to the source the + * person's ladder picks before they aggregate. The dense and raw reads of + * `/api/measurements/series` filtered on user and type only, so with a second + * provider a day's value blended the two, a reading present in both showed + * twice, and the ladder had no effect on those charts. + * + * What is asserted, against a real Postgres: + * - the ladder decides which source a day shows, in both directions; + * - the rule is per day, so a day only the second source covers still shows; + * - a reading mirrored into two sources appears once; + * - the day-bucket, hour-bucket and raw reads all follow it, and the stats + * strip counts the canonical rows; + * - a type with no ladder keeps every source (the collapse would otherwise + * break the tie by source name and hide one of two real sources); + * - a single-source account reads exactly what is stored. + */ +import { NextRequest } from "next/server"; +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { cookieJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { + MeasurementSource, + MeasurementType, + Prisma, +} from "@/generated/prisma/client"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const TZ = "Europe/Berlin"; +const DUAL = "ladder-dual"; +const DENSE = "ladder-dense"; +const SOLO = "ladder-solo"; +const NOW = new Date(Math.floor(Date.now() / 60_000) * 60_000 - 60_000); +const DAY = 86_400_000; + +/** + * Noon UTC of the n-th day back (plus a few minutes), so a reading falls on one + * UTC day and one Berlin day whenever the suite runs. Anchoring on "now" would + * let a reading cross midnight and make the per-day source pick depend on the + * clock. + */ +const TODAY_NOON_UTC = Date.UTC( + NOW.getUTCFullYear(), + NOW.getUTCMonth(), + NOW.getUTCDate(), + 12, +); +const at = (daysBack: number, minuteOffset = 0) => + new Date(TODAY_NOON_UTC - daysBack * DAY + minuteOffset * 60_000); + +let seq = 0; +const row = ( + userId: string, + type: MeasurementType, + unit: string, + source: MeasurementSource, + value: number, + measuredAt: Date, +): Prisma.MeasurementCreateManyInput => ({ + id: `ladder-${++seq}`, + userId, + type, + unit, + source, + value, + measuredAt, +}); + +async function setLadder(userId: string, ladder: Record) { + await prisma.user.update({ + where: { id: userId }, + data: { sourcePriorityJson: ladder }, + }); +} + +async function series(userId: string, kind: string, days: number) { + const s = await prisma.session.create({ + data: { userId, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.clear(); + cookieJar.set("healthlog_session", s.id); + const { GET } = await import("@/app/api/measurements/series/route"); + const res = await GET( + new NextRequest( + `http://localhost/api/measurements/series?kind=${kind}&days=${days}`, + ), + ); + expect(res.status).toBe(200); + return ( + (await res.json()) as { + data: { + points: Array<{ id: string; at: string; value: number }>; + stats: { count: number; mean: number; min: number; max: number }; + }; + } + ).data; +} + +beforeAll(async () => { + await truncateAllTables(prisma); + for (const id of [DUAL, DENSE, SOLO]) { + await prisma.user.create({ data: { id, username: id, timezone: TZ } }); + invalidateUserTimezone(id); + } + + // Pulse: Apple Health (60-64) covers days 1-3, Fitbit (90-94) covers days 3-5, + // so day 3 has both sources and days 4-5 only Fitbit. + const dual: Prisma.MeasurementCreateManyInput[] = []; + for (const d of [1, 2, 3]) { + [60, 62, 64].forEach((v, i) => + dual.push(row(DUAL, "PULSE", "bpm", "APPLE_HEALTH", v, at(d, i))), + ); + } + for (const d of [3, 4, 5]) { + [90, 92, 94].forEach((v, i) => + dual.push(row(DUAL, "PULSE", "bpm", "FITBIT", v, at(d, i + 3))), + ); + } + // Weight: the same weighing recorded by hand AND by the scale, three days. + for (const d of [1, 2, 3]) { + dual.push(row(DUAL, "WEIGHT", "kg", "MANUAL", 80 + d, at(d))); + dual.push(row(DUAL, "WEIGHT", "kg", "WITHINGS", 80 + d, at(d))); + } + // Glucose has no ladder: both sources must stay. + for (const d of [1, 2]) { + dual.push(row(DUAL, "BLOOD_GLUCOSE", "mg/dL", "APPLE_HEALTH", 100, at(d))); + dual.push(row(DUAL, "BLOOD_GLUCOSE", "mg/dL", "NIGHTSCOUT", 110, at(d, 1))); + } + await prisma.measurement.createMany({ data: dual }); + + // A dense two-source pulse stream: past the 10 000-row cap, so it is read + // as hour buckets. Apple is a steady 70, Fitbit a steady 100. + for (const [source, value] of [ + ["APPLE_HEALTH", 70], + ["FITBIT", 100], + ] as const) { + await prisma.$executeRawUnsafe( + `INSERT INTO measurements ( + id, user_id, type, value, unit, source, measured_at, + created_at, updated_at, sync_version) + SELECT $4 || lpad(g::text, 7, '0'), $1, 'PULSE'::measurement_type, + $5, 'bpm', $6::measurement_source, + ($2::timestamptz AT TIME ZONE 'UTC') - (g * interval '1 minute'), + now(), now(), 1 + FROM generate_series(0, $3::int - 1) g`, + DENSE, + NOW.toISOString(), + 6_000, + `dn-${source}-`, + value, + source, + ); + } + + // A single-source account. + await prisma.measurement.createMany({ + data: [1, 2, 3, 4].map((d) => + row(SOLO, "WEIGHT", "kg", "MANUAL", 70 + d, at(d)), + ), + }); + await prisma.$executeRawUnsafe(`ANALYZE measurements`); +}, 120_000); + +describe("pulse follows the ladder (raw read, 30 days)", () => { + it("shows Apple Health when it ranks first, and fills the days only Fitbit covers", async () => { + await setLadder(DUAL, { pulse: ["APPLE_HEALTH", "FITBIT"] }); + const data = await series(DUAL, "pulse", 30); + // Days 1-3: Apple (60-64). Day 3 is shared, so Fitbit's day-3 rows are out. + // Days 4-5: only Fitbit exists, so Fitbit shows. + const values = data.points.map((p) => p.value).sort((a, b) => a - b); + expect(values).toEqual([ + 60, 60, 60, 62, 62, 62, 64, 64, 64, 90, 90, 92, 92, 94, 94, + ]); + expect(data.stats.count).toBe(15); + }); + + it("shows Fitbit when it ranks first, and fills the days only Apple covers", async () => { + await setLadder(DUAL, { pulse: ["FITBIT", "APPLE_HEALTH"] }); + const data = await series(DUAL, "pulse", 30); + const values = data.points.map((p) => p.value).sort((a, b) => a - b); + // Days 3-5: Fitbit. Days 1-2: only Apple exists. + expect(values).toEqual([ + 60, 60, 62, 62, 64, 64, 90, 90, 90, 92, 92, 92, 94, 94, 94, + ]); + expect(data.stats.count).toBe(15); + }); + + it("does not blend the two on a shared day (day-bucket read, 365 days)", async () => { + await setLadder(DUAL, { pulse: ["APPLE_HEALTH", "FITBIT"] }); + const data = await series(DUAL, "pulse", 365); + const byDay = data.points.map((p) => p.value).sort((a, b) => a - b); + // One mean per local day: Apple's days 1-3 are 62, Fitbit's days 4-5 are 92. + expect(byDay).toEqual([62, 62, 62, 92, 92]); + expect(data.stats.count).toBe(15); + }); +}); + +describe("a reading mirrored into two sources", () => { + it("appears once", async () => { + await setLadder(DUAL, { weight: ["WITHINGS", "MANUAL"] }); + const data = await series(DUAL, "weight", 30); + expect(data.points).toHaveLength(3); + expect(data.points.map((p) => p.value).sort()).toEqual([81, 82, 83]); + expect(data.stats.count).toBe(3); + }); + + it("follows the ladder for which copy it keeps", async () => { + await setLadder(DUAL, { weight: ["MANUAL", "WITHINGS"] }); + const manual = await series(DUAL, "weight", 30); + const rows = await prisma.measurement.findMany({ + where: { userId: DUAL, type: "WEIGHT", source: "MANUAL" }, + select: { id: true }, + }); + expect(new Set(manual.points.map((p) => p.id))).toEqual( + new Set(rows.map((r) => r.id)), + ); + }); +}); + +describe("an hour-bucket read (past the 10 000-row cap)", () => { + it("shows only the ladder's source in every bucket", async () => { + await setLadder(DENSE, { pulse: ["APPLE_HEALTH", "FITBIT"] }); + const apple = await series(DENSE, "pulse", 30); + expect(apple.points.length).toBeGreaterThan(50); + expect(apple.points.every((p) => p.id.startsWith("hour:"))).toBe(true); + expect(apple.points.every((p) => p.value === 70)).toBe(true); + expect(apple.stats.count).toBe(6_000); + + await setLadder(DENSE, { pulse: ["FITBIT", "APPLE_HEALTH"] }); + const fitbit = await series(DENSE, "pulse", 30); + expect(fitbit.points.every((p) => p.value === 100)).toBe(true); + expect(fitbit.stats.count).toBe(6_000); + }); +}); + +describe("a type with no ladder", () => { + it("keeps every source (glucose)", async () => { + const data = await series(DUAL, "glucose", 30); + expect(data.points).toHaveLength(4); + expect(data.stats.count).toBe(4); + }); +}); + +describe("a single-source account", () => { + it("reads exactly what is stored", async () => { + const data = await series(SOLO, "weight", 30); + const raw = await prisma.measurement.findMany({ + where: { userId: SOLO, type: "WEIGHT" }, + orderBy: { measuredAt: "asc" }, + select: { id: true, value: true, measuredAt: true }, + }); + expect(data.points).toEqual( + raw.map((r) => ({ + id: r.id, + at: r.measuredAt.toISOString(), + value: r.value, + secondary: null, + })), + ); + }); +}); From b961e82a71dd679b11e2f2cf4d546fb8b3a2aaf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 3 Oct 2026 20:34:51 +0200 Subject: [PATCH 02/14] chore(github): point feature ideas to the upvotable Ideas category --- .github/ISSUE_TEMPLATE/config.yml | 3 +++ .github/ISSUE_TEMPLATE/feature_request.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 054370535..020b985d3 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,8 @@ blank_issues_enabled: false contact_links: + - name: Ideas and upvotes + url: https://github.com/MBombeck/HealthLog/discussions/categories/ideas?discussions_q=category%3AIdeas+sort%3Atop + about: Suggest an improvement or upvote the ones you want. One idea per discussion; the most-voted ones get looked at first. - name: Documentation url: https://docs.healthlog.dev about: Setup, configuration, and feature documentation. Most usage questions are answered here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 5229123be..fbaa1519f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -6,7 +6,7 @@ body: - type: markdown attributes: value: | - Feature ideas welcome. HealthLog is a personal-health PWA — the bar for inclusion is high (privacy-first, offline-capable, self-hostable). Use this template to propose something concrete; for open-ended discussion, prefer a [Discussion](https://github.com/MBombeck/HealthLog/discussions) thread. + Feature ideas welcome. HealthLog is a personal-health PWA, and the bar for inclusion is high (privacy-first, offline-capable, self-hostable). Use this template to propose something concrete. For an idea you'd like others to weigh in on, or a small improvement, start it in [Ideas](https://github.com/MBombeck/HealthLog/discussions/categories/ideas) instead: people can upvote it there, and the most-voted ideas get looked at first (see #1114). - type: textarea id: problem attributes: From eda57768864ae2e8e80cec62f99024c010d48f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 3 Oct 2026 20:51:57 +0200 Subject: [PATCH 03/14] Run document summaries and filing suggestions in the background The summary and suggest routes accept `Prefer: respond-async`: after the same quick refusals they answer 202 with a run id, and the background document AI worker runs the read under two new run kinds (DOCUMENT_SUMMARY, DOCUMENT_SUGGEST, migration 0370). Without the header both routes answer synchronously as before; both paths run one shared body, so they cannot drift. The worker asks the capability for the record and re-checks the wire for the provider it picks before every read. A summary that is to be stored marks the document PENDING while queued, READY when it lands, and UNAVAILABLE when the run fails or the write does not land. A second request attaches to a live run only when it would produce the same thing. The vault asks for both in the background and shows the same calm waiting line the index uses, worded so a transient read never promises that leaving the page keeps it. The egress guard and the route inventory now hold every run kind to its dispatcher re-check and its capability. --- docs/api/openapi.yaml | 662 +++++++++--------- .../migration.sql | 15 + prisma/schema.prisma | 9 + .../ai-egress-capability-guard.test.ts | 47 +- .../ai-capability-route-inventory.test.ts | 34 + .../documents/inbound/[id]/suggest/route.ts | 268 ++++--- .../documents/inbound/[id]/summary/route.ts | 468 +++++-------- .../__tests__/document-ai-section.test.tsx | 32 + .../__tests__/document-summary-block.test.tsx | 12 + .../documents/document-ai-panels.tsx | 34 + .../documents/document-ai-section.tsx | 26 +- .../documents/document-detail-sheet.tsx | 4 + .../documents/document-summary-block.tsx | 14 + .../documents/use-document-assist.ts | 23 +- src/lib/documents/ai-runs/run-support.ts | 74 ++ src/lib/documents/ai-runs/start.ts | 15 +- src/lib/documents/ai-runs/suggest-run.ts | 112 +++ src/lib/documents/ai-runs/summary-run.ts | 250 +++++++ src/lib/documents/ai-runs/types.ts | 46 +- .../jobs/__tests__/document-ai-run.test.ts | 223 +++++- src/lib/jobs/document-ai-run.ts | 185 ++++- src/lib/openapi/routes/ai-runs.ts | 27 +- src/lib/openapi/routes/documents.ts | 48 +- tests/integration/document-ai-runs.test.ts | 233 +++++- 24 files changed, 2061 insertions(+), 800 deletions(-) create mode 100644 prisma/migrations/0370_document_ai_run_kinds_summary_suggest_extract/migration.sql create mode 100644 src/lib/documents/ai-runs/run-support.ts create mode 100644 src/lib/documents/ai-runs/suggest-run.ts create mode 100644 src/lib/documents/ai-runs/summary-run.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index cf0c25eb9..35847ef8a 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -14284,7 +14284,7 @@ paths: schema: $ref: "#/components/schemas/OcrTextExtractRequest" responses: - "202": &a38 + "202": &a36 description: Queued for the background worker. Every refusal the route can answer quickly (capability, provider, rate limit, budget, validation) has already been answered; anything that goes wrong from here is reported on the run. @@ -16722,7 +16722,7 @@ paths: schema: $ref: "#/components/schemas/InboundExtractEnvelope" "401": *a1 - "403": &a36 + "403": &a37 description: "AI refusal: the `documentAi` capability is unavailable for this record. The envelope carries `meta.errorCode`, `meta.capability` and `meta.reason` (one of `AiUnavailableReason`). `assistant.disabled.documentAi` — the operator turned off reading documents, or every AI feature (the master @@ -16746,7 +16746,7 @@ paths: $ref: "#/components/schemas/ErrorEnvelope" "422": *a4 "429": *a5 - "503": &a37 + "503": &a38 description: "`ai.unavailable` — the inputs behind the AI capability could not be read, and the answer failed closed. Nothing was sent. Retry later." content: @@ -16766,160 +16766,18 @@ paths: only consumed when the request reaches the provider, and the 429 carries `Retry-After`, the `X-RateLimit-Limit` / `-Remaining` / `-Reset` triple, and `meta.retryAt`). WRITES NOTHING — never stages facts, never flips status; the user reviews and saves. 422 (`documents.inbound.providerUnsupported`) with no provider configured. Never - interprets or diagnoses; the title is a neutral filing label." + interprets or diagnoses; the title is a neutral filing label. With `Prefer: respond-async` (v1.40) the read runs + in the background: after the same quick refusals the route answers 202 with a run id, a second request while a + suggestion read of this document over the same transport is queued or running answers with that run, and `GET + /api/ai-runs/{id}` serves this body as the run's `result`." parameters: - name: id in: path required: true schema: type: string - requestBody: - required: false - content: - application/json: - schema: - type: object - properties: - mode: - type: string - const: text - text: - type: string - kind: - type: string - enum: - - DOCTOR_REPORT - - DISCHARGE_LETTER - - LAB_RESULT - - IMAGING - - PRESCRIPTION - - REFERRAL - - INSURANCE - - VACCINATION - - OTHER - required: - - mode - - text - responses: - "200": - description: The filing-metadata suggestion (drafts). - content: - application/json: - schema: - $ref: "#/components/schemas/DocumentSuggestEnvelope" - "401": *a1 - "403": *a36 - "422": *a4 - "429": *a5 - "503": *a37 - /api/documents/inbound/{id}/summary: - post: - tags: - - Documents - summary: Summarise or transcribe a document - description: 'On-demand description of a stored document. `?mode=summary` (default) returns a short plain-language - summary of WHAT the document is; it remains transient unless `persist=true` fills the empty stored-summary slot, - while `persist=true&replace=true` explicitly replaces an existing summary. `?mode=text` returns transient raw - transcribed text. Neither result reaches coach memory, snapshots, structured stores, or the search index. The - summary is descriptive only and never a diagnosis. Same VISION (empty body) / TEXT (`application/json` `{ mode: - "text", text }`, opt-in local OCR) dispatch as extract. Answers under the `documentAi` capability (403 with the - capability envelope when it is closed), then rate / budget gated. 422 `documents.inbound.providerUnsupported` - with no provider.' - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: mode - in: query - required: false - schema: - type: string - enum: - - summary - - text - default: summary - description: "`summary` (plain-language description) or `text` (raw)." - - name: persist - in: query - required: false - schema: - type: boolean - default: false - description: "For summary mode only: persist into an empty stored-summary slot." - - name: replace - in: query - required: false - schema: - type: boolean - default: false - description: With `persist=true`, explicitly replace an existing stored summary. - requestBody: - required: false - content: - application/json: - schema: - type: object - properties: - mode: - type: string - const: text - text: - type: string - kind: - type: string - enum: - - DOCTOR_REPORT - - DISCHARGE_LETTER - - LAB_RESULT - - IMAGING - - PRESCRIPTION - - REFERRAL - - INSURANCE - - VACCINATION - - OTHER - required: - - mode - - text - responses: - "200": - description: The summary and its persistence outcome, or transient extracted text. - content: - application/json: - schema: - $ref: "#/components/schemas/DocumentSummaryEnvelope" - "401": *a1 - "403": *a36 - "422": *a4 - "429": *a5 - "503": *a37 - /api/documents/inbound/{id}/index: - post: - tags: - - Documents - summary: Build / refresh the content-search index - description: "Populates or refreshes one document's content-search index so search matches INSIDE its body. VISION - (empty body) decrypts the stored original and runs one provider transcription (answers under the `documentAi` - capability — 403 with the capability envelope when it is closed, including a missing `ai_extraction` / `ai_full` - receipt for a provider that leaves the machine — then rate / budget gated); TEXT (`application/json` `{ mode: - \"text\", text }`, opt-in local OCR) indexes browser-OCR'd text with no provider egress and stays available with - AI off, so search keeps working. Persists ONLY AES-256-GCM ciphertext of the text plus opaque HMAC token hashes - — no plaintext body, no plaintext token. Idempotent; re-indexing overwrites in place. A successful index - continues into the same lab auto-staging the background worker performs (still-STORED lab-looking document with - no facts, both modules on, provider + consent eligible); `labFactsStaged` reports how many facts were staged - PENDING for review — confirmation stays the only write into Labs. With `Prefer: respond-async` (v1.40) the read - runs in the background: after the same quick refusals the route answers 202 with a run id, a second request - while a read of this document is queued or running answers with that run, and `GET /api/ai-runs/{id}` serves - this body as the run's `result`. Text mode honours the preference too, because the lab staging it continues into - is model work." - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: Prefer + - &a39 + name: Prefer in: header required: false schema: @@ -16957,6 +16815,162 @@ paths: required: - mode - text + responses: + "200": + description: The filing-metadata suggestion (drafts). + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSuggestEnvelope" + "202": *a36 + "401": *a1 + "403": *a37 + "422": *a4 + "429": *a5 + "503": *a38 + /api/documents/inbound/{id}/summary: + post: + tags: + - Documents + summary: Summarise or transcribe a document + description: "On-demand description of a stored document. `?mode=summary` (default) returns a short plain-language + summary of WHAT the document is; it remains transient unless `persist=true` fills the empty stored-summary slot, + while `persist=true&replace=true` explicitly replaces an existing summary. `?mode=text` returns transient raw + transcribed text. Neither result reaches coach memory, snapshots, structured stores, or the search index. The + summary is descriptive only and never a diagnosis. Same VISION (empty body) / TEXT (`application/json` `{ mode: + \"text\", text }`, opt-in local OCR) dispatch as extract. Answers under the `documentAi` capability (403 with + the capability envelope when it is closed), then rate / budget gated. 422 + `documents.inbound.providerUnsupported` with no provider. With `Prefer: respond-async` (v1.40) the read runs in + the background: after the same quick refusals the route answers 202 with a run id, and `GET /api/ai-runs/{id}` + serves this body as the run's `result`. A second request while a read with the same `mode`, `persist` and + `replace` is queued or running answers with that run. With `persist=true` the document's `summaryState` is + PENDING while the run is queued or running (unless a summary is already stored) and becomes UNAVAILABLE when the + run fails. `mode=text` over posted text calls no model and is answered synchronously either way (no + `Preference-Applied`)." + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: mode + in: query + required: false + schema: + type: string + enum: + - summary + - text + default: summary + description: "`summary` (plain-language description) or `text` (raw)." + - name: persist + in: query + required: false + schema: + type: boolean + default: false + description: "For summary mode only: persist into an empty stored-summary slot." + - name: replace + in: query + required: false + schema: + type: boolean + default: false + description: With `persist=true`, explicitly replace an existing stored summary. + - *a39 + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + mode: + type: string + const: text + text: + type: string + kind: + type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - OTHER + required: + - mode + - text + responses: + "200": + description: The summary and its persistence outcome, or transient extracted text. + content: + application/json: + schema: + $ref: "#/components/schemas/DocumentSummaryEnvelope" + "202": *a36 + "401": *a1 + "403": *a37 + "422": *a4 + "429": *a5 + "503": *a38 + /api/documents/inbound/{id}/index: + post: + tags: + - Documents + summary: Build / refresh the content-search index + description: "Populates or refreshes one document's content-search index so search matches INSIDE its body. VISION + (empty body) decrypts the stored original and runs one provider transcription (answers under the `documentAi` + capability — 403 with the capability envelope when it is closed, including a missing `ai_extraction` / `ai_full` + receipt for a provider that leaves the machine — then rate / budget gated); TEXT (`application/json` `{ mode: + \"text\", text }`, opt-in local OCR) indexes browser-OCR'd text with no provider egress and stays available with + AI off, so search keeps working. Persists ONLY AES-256-GCM ciphertext of the text plus opaque HMAC token hashes + — no plaintext body, no plaintext token. Idempotent; re-indexing overwrites in place. A successful index + continues into the same lab auto-staging the background worker performs (still-STORED lab-looking document with + no facts, both modules on, provider + consent eligible); `labFactsStaged` reports how many facts were staged + PENDING for review — confirmation stays the only write into Labs. With `Prefer: respond-async` (v1.40) the read + runs in the background: after the same quick refusals the route answers 202 with a run id, a second request + while a read of this document is queued or running answers with that run, and `GET /api/ai-runs/{id}` serves + this body as the run's `result`. Text mode honours the preference too, because the lab staging it continues into + is model work." + parameters: + - name: id + in: path + required: true + schema: + type: string + - *a39 + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + mode: + type: string + const: text + text: + type: string + kind: + type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - OTHER + required: + - mode + - text responses: "200": description: The document was indexed. @@ -16964,12 +16978,12 @@ paths: application/json: schema: $ref: "#/components/schemas/DocumentIndexEnvelope" - "202": *a38 + "202": *a36 "401": *a1 - "403": *a36 + "403": *a37 "422": *a4 "429": *a5 - "503": *a37 + "503": *a38 /api/documents/inbound/{id}/chat: post: tags: @@ -17009,10 +17023,10 @@ paths: type: string description: "SSE frames: `data: \\n\\n`. See the operation description for the per-`type` frame shapes." "401": *a1 - "403": *a36 + "403": *a37 "422": *a4 "429": *a5 - "503": *a37 + "503": *a38 get: tags: - Documents @@ -17075,10 +17089,10 @@ paths: schema: $ref: "#/components/schemas/DocumentReindexEnvelope" "401": *a1 - "403": *a36 + "403": *a37 "422": *a4 "429": *a5 - "503": *a37 + "503": *a38 /api/documents/inbound/{id}/facts/{factId}: patch: tags: @@ -17185,7 +17199,7 @@ paths: schema: $ref: "#/components/schemas/CreateCustomMetricResponse" "401": *a1 - "409": &a40 + "409": &a41 description: A custom metric with this name already exists for the caller. content: application/json: @@ -17214,7 +17228,7 @@ paths: $ref: "#/components/schemas/GetCustomMetricResponse" "401": *a1 "403": *a11 - "404": &a39 + "404": &a40 description: Custom metric not found (or owned by another user). content: application/json: @@ -17248,8 +17262,8 @@ paths: schema: $ref: "#/components/schemas/UpdateCustomMetricResponse" "401": *a1 - "404": *a39 - "409": *a40 + "404": *a40 + "409": *a41 "422": *a4 "429": *a5 delete: @@ -17272,7 +17286,7 @@ paths: schema: $ref: "#/components/schemas/DeleteCustomMetricResponse" "401": *a1 - "404": *a39 + "404": *a40 "422": *a4 "429": *a5 /api/custom-metrics/{id}/entries: @@ -17316,7 +17330,7 @@ paths: $ref: "#/components/schemas/ListCustomMetricEntriesEnvelope" "401": *a1 "403": *a11 - "404": *a39 + "404": *a40 "422": *a4 "429": *a5 post: @@ -17345,7 +17359,7 @@ paths: schema: $ref: "#/components/schemas/CreateCustomMetricEntryResponse" "401": *a1 - "404": *a39 + "404": *a40 "409": *a7 "422": *a4 "429": *a12 @@ -17381,7 +17395,7 @@ paths: schema: $ref: "#/components/schemas/UpdateCustomMetricEntryResponse" "401": *a1 - "404": &a41 + "404": &a42 description: Custom metric entry not found (or owned by another user). content: application/json: @@ -17414,7 +17428,7 @@ paths: schema: $ref: "#/components/schemas/DeleteCustomMetricEntryResponse" "401": *a1 - "404": *a41 + "404": *a42 "422": *a4 "429": *a5 /api/custom-metrics/{id}/entries/restore: @@ -17826,7 +17840,7 @@ paths: schema: $ref: "#/components/schemas/NutrientOverviewEnvelope" "401": *a1 - "403": &a42 + "403": &a43 description: >- The opt-in `nutrients` module is off for this account (errorCode `module.disabled`). Stop syncing and do not retry; offer the enable-in-settings hint. Only run the HealthKit read-authorization flow while the module is @@ -17881,7 +17895,7 @@ paths: schema: $ref: "#/components/schemas/NutrientDailySeriesEnvelope" "401": *a1 - "403": *a42 + "403": *a43 "422": *a4 "429": *a5 /api/nutrients/water: @@ -17910,7 +17924,7 @@ paths: schema: $ref: "#/components/schemas/NutrientWaterWriteEnvelope" "401": *a1 - "403": *a42 + "403": *a43 "409": *a7 "422": *a4 "429": *a5 @@ -18412,26 +18426,26 @@ paths: application/json: schema: $ref: "#/components/schemas/WithingsCredentialsUpdateEnvelope" - "400": &a43 + "400": &a44 description: Body is not parseable JSON. content: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" "401": *a1 - "413": &a44 + "413": &a45 description: Body exceeds 64 KiB. content: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" - "415": &a45 + "415": &a46 description: Content-Type is not `application/json`. content: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" - "422": &a46 + "422": &a47 description: "`clientId` and `clientSecret` are both required and each must be 1–200 characters. The envelope carries a single fixed message, not per-field issues." content: @@ -18617,11 +18631,11 @@ paths: application/json: schema: $ref: "#/components/schemas/FitbitCredentialsUpdateEnvelope" - "400": *a43 + "400": *a44 "401": *a1 - "413": *a44 - "415": *a45 - "422": *a46 + "413": *a45 + "415": *a46 + "422": *a47 "429": *a5 delete: tags: @@ -18764,11 +18778,11 @@ paths: application/json: schema: $ref: "#/components/schemas/GoogleHealthCredentialsUpdateEnvelope" - "400": *a43 + "400": *a44 "401": *a1 - "413": *a44 - "415": *a45 - "422": *a46 + "413": *a45 + "415": *a46 + "422": *a47 "429": *a5 delete: tags: @@ -18921,11 +18935,11 @@ paths: application/json: schema: $ref: "#/components/schemas/OuraCredentialsUpdateEnvelope" - "400": *a43 + "400": *a44 "401": *a1 - "413": *a44 - "415": *a45 - "422": *a46 + "413": *a45 + "415": *a46 + "422": *a47 "429": *a5 delete: tags: @@ -19001,21 +19015,21 @@ paths: schema: $ref: "#/components/schemas/OuraTestEnvelope" "401": *a1 - "422": &a47 + "422": &a48 description: No usable connection for this provider — nothing stored, or the stored grant could no longer be resolved into an access token. `meta.errorCode` = `not_configured`. content: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" - "429": &a48 + "429": &a49 description: More than 5 requests in 60 s from this user. `meta.errorCode` = `rate_limited_self` — this is the local limiter, distinct from a `rate_limited` 502 which is the provider's. content: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" - "502": &a49 + "502": &a50 description: "The probe did not complete. `meta.errorCode` names the class: `credentials_rejected` (the provider answered 401/403 — the stored grant is gone), `rate_limited` (the PROVIDER answered 429, which is not this endpoint's own limiter), `upstream_error` (5xx), `timeout` (no answer within 5 s), `connection_failed` @@ -19060,11 +19074,11 @@ paths: application/json: schema: $ref: "#/components/schemas/PolarCredentialsUpdateEnvelope" - "400": *a43 + "400": *a44 "401": *a1 - "413": *a44 - "415": *a45 - "422": *a46 + "413": *a45 + "415": *a46 + "422": *a47 "429": *a5 delete: tags: @@ -19140,9 +19154,9 @@ paths: schema: $ref: "#/components/schemas/PolarTestEnvelope" "401": *a1 - "422": *a47 - "429": *a48 - "502": *a49 + "422": *a48 + "429": *a49 + "502": *a50 /api/strava/credentials: get: tags: @@ -19178,11 +19192,11 @@ paths: application/json: schema: $ref: "#/components/schemas/StravaCredentialsUpdateEnvelope" - "400": *a43 + "400": *a44 "401": *a1 - "413": *a44 - "415": *a45 - "422": *a46 + "413": *a45 + "415": *a46 + "422": *a47 "429": *a5 delete: tags: @@ -19260,9 +19274,9 @@ paths: schema: $ref: "#/components/schemas/StravaTestEnvelope" "401": *a1 - "422": *a47 - "429": *a48 - "502": *a49 + "422": *a48 + "429": *a49 + "502": *a50 /api/integrations/status: get: tags: @@ -19298,7 +19312,7 @@ paths: $ref: "#/components/schemas/FitbitResumeEnvelope" "401": *a1 "422": *a4 - "429": *a48 + "429": *a49 /api/integrations/fitbit/test: post: tags: @@ -19316,9 +19330,9 @@ paths: schema: $ref: "#/components/schemas/FitbitIntegrationTestEnvelope" "401": *a1 - "422": *a47 - "429": *a48 - "502": *a49 + "422": *a48 + "429": *a49 + "502": *a50 /api/integrations/google-health/resume: post: tags: @@ -19335,7 +19349,7 @@ paths: $ref: "#/components/schemas/GoogleHealthResumeEnvelope" "401": *a1 "422": *a4 - "429": *a48 + "429": *a49 /api/integrations/google-health/test: post: tags: @@ -19368,9 +19382,9 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" - "422": *a47 - "429": *a48 - "502": *a49 + "422": *a48 + "429": *a49 + "502": *a50 /api/integrations/whoop/resume: post: tags: @@ -19387,7 +19401,7 @@ paths: $ref: "#/components/schemas/WhoopResumeEnvelope" "401": *a1 "422": *a4 - "429": *a48 + "429": *a49 /api/integrations/whoop/test: post: tags: @@ -19404,9 +19418,9 @@ paths: schema: $ref: "#/components/schemas/WhoopIntegrationTestEnvelope" "401": *a1 - "422": *a47 - "429": *a48 - "502": *a49 + "422": *a48 + "429": *a49 + "502": *a50 /api/integrations/withings/resume: post: tags: @@ -19424,7 +19438,7 @@ paths: $ref: "#/components/schemas/WithingsResumeEnvelope" "401": *a1 "422": *a4 - "429": *a48 + "429": *a49 /api/integrations/withings/test: post: tags: @@ -19442,8 +19456,8 @@ paths: schema: $ref: "#/components/schemas/WithingsIntegrationTestEnvelope" "401": *a1 - "422": *a47 - "429": *a48 + "422": *a48 + "429": *a49 "502": description: "The probe did not complete. The shared codes apply (`credentials_rejected`, `rate_limited`, `upstream_error`, `timeout`, `connection_failed`) and this path adds one of its own: @@ -19683,7 +19697,7 @@ paths: schema: $ref: "#/components/schemas/RecordActivityEnvelope" "401": *a1 - "403": &a50 + "403": &a51 description: Refused. `meta.errorCode` is `sharing.not_permitted` when the request was made while acting on another account (grant management is never delegable), or `sharing.access.denied` when the caller may not act as the account they named. The latter is byte-identical for an account that does not exist and one that granted @@ -19708,7 +19722,7 @@ paths: schema: $ref: "#/components/schemas/AccountGrantListEnvelope" "401": *a1 - "403": *a50 + "403": *a51 "422": *a4 "429": *a5 post: @@ -19810,7 +19824,7 @@ paths: schema: $ref: "#/components/schemas/AcceptedAccountGrantEnvelope" "401": *a1 - "403": *a50 + "403": *a51 "404": description: No such invitation for this account. content: @@ -19854,7 +19868,7 @@ paths: schema: $ref: "#/components/schemas/RevokedAccountGrantEnvelope" "401": *a1 - "403": *a50 + "403": *a51 "404": description: No such grant on this account's record. content: @@ -19895,7 +19909,7 @@ paths: schema: $ref: "#/components/schemas/RenouncedAccountGrantEnvelope" "401": *a1 - "403": *a50 + "403": *a51 "404": description: No such grant held by this account. content: @@ -20277,7 +20291,7 @@ paths: schema: $ref: "#/components/schemas/ErrorEnvelope" "401": *a1 - "403": *a50 + "403": *a51 "409": description: "The record this browser session is on is not the record the request asserted. `meta.errorCode` is `sharing.session.changed`. It is not a refusal of access — the caller may well be entitled to both records — @@ -20607,7 +20621,7 @@ paths: schema: $ref: "#/components/schemas/McpConnectionListEnvelope" "401": *a1 - "403": &a51 + "403": &a52 description: The operator has disabled the API instance-wide. Every route in this module refuses while that setting is off, for a cookie session as well as for a Bearer token; it is not a per-user permission and retrying will not help. @@ -20640,7 +20654,7 @@ paths: schema: $ref: "#/components/schemas/McpConnectionRevokeEnvelope" "401": *a1 - "403": *a51 + "403": *a52 "404": description: No live connection with this id for this caller — also the answer for an already-revoked one, so a repeat revoke is 404 rather than an idempotent 200. @@ -20667,7 +20681,7 @@ paths: schema: $ref: "#/components/schemas/McpConnectorTokenListEnvelope" "401": *a1 - "403": *a51 + "403": *a52 "422": *a4 "429": *a5 post: @@ -20700,7 +20714,7 @@ paths: schema: $ref: "#/components/schemas/ErrorEnvelope" "401": *a1 - "403": *a51 + "403": *a52 "413": description: Body exceeds 16 KiB. content: @@ -20743,7 +20757,7 @@ paths: schema: $ref: "#/components/schemas/McpConnectorTokenRevokeEnvelope" "401": *a1 - "403": *a51 + "403": *a52 "404": description: No `health:read` token with this id for this caller. An already-revoked token is still found, so a repeat revoke answers 200. @@ -20801,7 +20815,7 @@ paths: description: Returns the named family's resolved settings for the record the session is acting on. Cookie session or Bearer, while acting on a managed record the caller guards. parameters: - - &a52 + - &a53 name: family in: path required: true @@ -20845,7 +20859,7 @@ paths: ride the audit row. Body capped at 64 KiB. Cookie session or Bearer, while acting on a managed record the caller guards. parameters: - - *a52 + - *a53 requestBody: required: true content: @@ -21382,13 +21396,13 @@ paths: application/json: schema: $ref: "#/components/schemas/AdminBackupDocument" - "403": &a53 + "403": &a54 description: Caller is not an admin. `requireAdmin()` is cookie-only, so a Bearer token lands here whatever its scope. content: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" - "404": &a54 + "404": &a55 description: No stored copy with this id. content: application/json: @@ -21462,8 +21476,8 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" - "403": *a53 - "404": *a54 + "403": *a54 + "404": *a55 "409": description: A restore of the same account is already queued or running (`meta.errorCode` = `backup.restore.active`, `meta.jobId` names it), the stored copy was replaced by a newer one while it was being checked @@ -21507,7 +21521,7 @@ paths: application/json: schema: $ref: "#/components/schemas/AdminBackupRestoreJobListResponse" - "403": *a53 + "403": *a54 /api/admin/backups/restores/{jobId}: get: tags: @@ -21530,7 +21544,7 @@ paths: application/json: schema: $ref: "#/components/schemas/AdminBackupRestoreJobResponse" - "403": *a53 + "403": *a54 "404": description: No restore job with this id. content: @@ -21549,7 +21563,7 @@ paths: request leaves the server. Needs neither the Documents module nor the list: with the list unset, `available` is false and any connection still stored is listed (with `originAllowed: false`) so it can be removed. Cookie session only; not delegable." - security: &a55 + security: &a56 - cookieAuth: [] responses: "200": @@ -21559,7 +21573,7 @@ paths: schema: $ref: "#/components/schemas/DocumentSourcesStatusEnvelope" "401": *a1 - "403": &a56 + "403": &a57 description: "Refused before any request to the source: `documents.sources.browserOnly` for a Bearer caller, `sharing.not_permitted` while acting on another person's record (a delegate or guardian never uses the owner's archive), or `module.disabled` when the Documents module is off." @@ -21582,9 +21596,9 @@ paths: `documents.sources.tokenRequired`), so a stored token is never sent to an address it was not saved for. A Papra organization id is letters, digits, `_` and `-`. The token is stored encrypted (AES-256-GCM) and never returned. Ten saves and tests a minute per person. Cookie session only; not delegable." - security: *a55 + security: *a56 parameters: - - &a57 + - &a58 name: system in: path required: true @@ -21623,8 +21637,8 @@ paths: schema: $ref: "#/components/schemas/DocumentSourceConnectionEnvelope" "401": *a1 - "403": *a56 - "404": &a58 + "403": *a57 + "404": &a59 description: "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. `documents.sources.notFound`: the source has no such document, or (Papra) no such organization, including @@ -21643,7 +21657,7 @@ paths: schema: $ref: "#/components/schemas/ErrorEnvelope" "429": *a5 - "502": &a59 + "502": &a60 description: "The source could not answer usefully, with the reason in `meta.errorCode` and, when the source answered at all, its HTTP status in `meta.upstreamStatus` (its body is never passed on): `documents.sources.unreachable` (no connection, timeout, or a destination refused at dial time), `documents.sources.redirected` (the source @@ -21662,9 +21676,9 @@ paths: description: Deletes the caller's connection for this system, and its token with it. Works even after the operator removed the origin from the list, and with the Documents module switched off. `disconnected` is false when there was nothing to delete. Cookie session only; not delegable. - security: *a55 + security: *a56 parameters: - - *a57 + - *a58 responses: "200": description: Removed, or there was nothing to remove. @@ -21673,7 +21687,7 @@ paths: schema: $ref: "#/components/schemas/DocumentSourceDisconnectEnvelope" "401": *a1 - "403": *a56 + "403": *a57 "422": *a4 "429": *a5 /api/documents/sources/{system}/test: @@ -21684,9 +21698,9 @@ paths: description: One request to the source with the stored token. Answers `ok` with the round-trip time and stamps `lastVerifiedAt`, or the reason in `meta.errorCode`. Shares the save's allowance of ten a minute. Cookie session only; not delegable. - security: *a55 + security: *a56 parameters: - - *a57 + - *a58 responses: "200": description: The source answered. @@ -21695,9 +21709,9 @@ paths: schema: $ref: "#/components/schemas/DocumentSourceTestEnvelope" "401": *a1 - "403": *a56 - "404": *a58 - "422": &a60 + "403": *a57 + "404": *a59 + "422": &a61 description: "`documents.sources.originNotAllowed`: the base address's origin is not on the operator's list (checked on every request, so a connection saved earlier is cut off once its origin is removed), or another validation failure." @@ -21706,7 +21720,7 @@ paths: schema: $ref: "#/components/schemas/ErrorEnvelope" "429": *a5 - "502": *a59 + "502": *a60 /api/documents/sources/{system}/tags: get: tags: @@ -21714,9 +21728,9 @@ paths: summary: Tags of a connected archive description: The source's tags for the picker's filter, the first 250 by name, in the source's own ids. Shares the search allowance of sixty a minute. Cookie session only; not delegable. - security: *a55 + security: *a56 parameters: - - *a57 + - *a58 responses: "200": description: The tags. @@ -21725,11 +21739,11 @@ paths: schema: $ref: "#/components/schemas/DocumentSourceTagsEnvelope" "401": *a1 - "403": *a56 - "404": *a58 - "422": *a60 + "403": *a57 + "404": *a59 + "422": *a61 "429": *a5 - "502": *a59 + "502": *a60 /api/documents/sources/{system}/search: get: tags: @@ -21743,9 +21757,9 @@ paths: under that source key: `new`, `imported` with the vault document's `documentId`, or `deleted` (deleted in HealthLog; importing it again stores nothing). Nothing is stored. Sixty searches a minute, shared with the tag read. Cookie session only; not delegable." - security: *a55 + security: *a56 parameters: - - *a57 + - *a58 - name: q in: query required: false @@ -21786,11 +21800,11 @@ paths: schema: $ref: "#/components/schemas/DocumentSourceSearchEnvelope" "401": *a1 - "403": *a56 - "404": *a58 - "422": *a60 + "403": *a57 + "404": *a59 + "422": *a61 "429": *a5 - "502": *a59 + "502": *a60 /api/documents/sources/{system}/import: post: tags: @@ -21809,9 +21823,9 @@ paths: fetched), including a duplicate. `kind` sets the document type (default OTHER). Automatic AI reading follows the person's own setting. Counts against the person's upload allowance (60 an hour) and storage quota. 201 for a new document, 200 otherwise. Cookie session only; not delegable." - security: *a55 + security: *a56 parameters: - - *a57 + - *a58 requestBody: required: true content: @@ -21868,7 +21882,7 @@ paths: schema: $ref: "#/components/schemas/DocumentSourceImportEnvelope" "401": *a1 - "403": *a56 + "403": *a57 "404": description: "`documents.sources.unavailable`: the operator has not listed any origin in `DOCUMENT_SOURCE_ORIGINS`, so the picker is off. `documents.sources.notConnected`: no connection is saved for this system. @@ -21899,7 +21913,7 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" - "422": *a60 + "422": *a61 "429": description: "`documents.inbound.rateLimited` (the person's hourly upload allowance, with the rate-limit headers) or `documents.inbound.uploadBusy` (too many uploads in progress; `Retry-After: 1`)." @@ -21907,7 +21921,7 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" - "502": *a59 + "502": *a60 /api/bugreport: post: security: [] @@ -63653,6 +63667,8 @@ components: enum: - DOCUMENT_INDEX - LABS_OCR_EXTRACT + - DOCUMENT_SUMMARY + - DOCUMENT_SUGGEST documentId: anyOf: - type: string @@ -63670,9 +63686,13 @@ components: - anyOf: - $ref: "#/components/schemas/DocumentIndexResponse" - $ref: "#/components/schemas/OcrExtractResponse" + - $ref: "#/components/schemas/DocumentSummaryResponse" + - $ref: "#/components/schemas/DocumentSuggestResponse" - type: "null" description: "Present once `status` is `SUCCEEDED`: the body the queuing route answers with synchronously. - `DOCUMENT_INDEX` carries `DocumentIndexResponse`, `LABS_OCR_EXTRACT` carries `OcrExtractResponse`." + `DOCUMENT_INDEX` carries `DocumentIndexResponse`, `LABS_OCR_EXTRACT` carries `OcrExtractResponse`, + `DOCUMENT_SUMMARY` carries `DocumentSummaryResponse`, `DOCUMENT_SUGGEST` carries `DocumentSuggestResponse`. + Decode by `kind`; treat an unknown `kind` as a run this client did not start." error: anyOf: - type: object @@ -63881,6 +63901,56 @@ components: reading this row likely duplicates (the review row defaults to unchecked when set); `confidence` is the model's per-field self-score (low fields are flagged for the human). Nothing is written until the commit route confirms." + DocumentSummaryResponse: + type: object + properties: + summary: + type: string + text: + type: string + persistence: + type: string + enum: + - stored + - withheld + - failed + additionalProperties: false + DocumentSuggestResponse: + type: object + properties: + suggestions: + type: object + properties: + title: + anyOf: + - type: string + - type: "null" + kind: + anyOf: + - type: string + enum: + - DOCTOR_REPORT + - DISCHARGE_LETTER + - LAB_RESULT + - IMAGING + - PRESCRIPTION + - REFERRAL + - INSURANCE + - VACCINATION + - OTHER + - type: "null" + documentDate: + anyOf: + - type: string + - type: "null" + required: + - title + - kind + - documentDate + additionalProperties: false + required: + - suggestions + additionalProperties: false ListBiomarkersEnvelope: type: object properties: @@ -67515,42 +67585,6 @@ components: - data - error additionalProperties: false - DocumentSuggestResponse: - type: object - properties: - suggestions: - type: object - properties: - title: - anyOf: - - type: string - - type: "null" - kind: - anyOf: - - type: string - enum: - - DOCTOR_REPORT - - DISCHARGE_LETTER - - LAB_RESULT - - IMAGING - - PRESCRIPTION - - REFERRAL - - INSURANCE - - VACCINATION - - OTHER - - type: "null" - documentDate: - anyOf: - - type: string - - type: "null" - required: - - title - - kind - - documentDate - additionalProperties: false - required: - - suggestions - additionalProperties: false DocumentSummaryEnvelope: type: object properties: @@ -67568,20 +67602,6 @@ components: - data - error additionalProperties: false - DocumentSummaryResponse: - type: object - properties: - summary: - type: string - text: - type: string - persistence: - type: string - enum: - - stored - - withheld - - failed - additionalProperties: false DocumentIndexEnvelope: type: object properties: diff --git a/prisma/migrations/0370_document_ai_run_kinds_summary_suggest_extract/migration.sql b/prisma/migrations/0370_document_ai_run_kinds_summary_suggest_extract/migration.sql new file mode 100644 index 000000000..6d1ede60a --- /dev/null +++ b/prisma/migrations/0370_document_ai_run_kinds_summary_suggest_extract/migration.sql @@ -0,0 +1,15 @@ +-- Two more kinds of background document AI run. +-- +-- Summaries and filing suggestions can now run in the same background worker +-- as "Read with AI" and the lab scan (0366): the request queues the read and +-- answers 202, and the result is polled from the run. The synchronous routes +-- are unchanged; the new kinds only name the run. +-- +-- Purely additive. `ADD VALUE IF NOT EXISTS` makes a rerun safe, and no new +-- value is used elsewhere in this migration, which is the one thing Postgres +-- forbids inside the transaction that adds it. No row is rewritten. +-- +-- Reversibility: Postgres cannot drop an enum value, so the values would stay +-- (inert with no rows that carry them). +ALTER TYPE "document_ai_run_kind" ADD VALUE IF NOT EXISTS 'DOCUMENT_SUMMARY'; +ALTER TYPE "document_ai_run_kind" ADD VALUE IF NOT EXISTS 'DOCUMENT_SUGGEST'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 36dbabbce..473f7c987 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -8031,6 +8031,9 @@ enum InboundDocumentKind { /// every fact, DISCARDED on a soft delete. enum InboundDocumentStatus { STORED + /// @internal: reserved, no writer. An extraction in flight is the state of + /// its background run (`DocumentAiRun`), not of the document, so the vault + /// never shows a status nothing renders. EXTRACTING EXTRACTED FAILED @@ -9033,6 +9036,12 @@ enum DocumentAiRunKind { /// A lab report scan (photo, PDF or browser-read text) into proposed rows /// for the review screen. Nothing is written until the person commits. LABS_OCR_EXTRACT + /// A plain-language summary or the transcribed text of a stored document; + /// stored on the document only when the request asked for that. + DOCUMENT_SUMMARY + /// Filing suggestions (title, kind, date) for a stored document. Drafts + /// only: nothing is written to the document. + DOCUMENT_SUGGEST @@map("document_ai_run_kind") } diff --git a/src/__tests__/ai-egress-capability-guard.test.ts b/src/__tests__/ai-egress-capability-guard.test.ts index 847f41bca..3b06191b0 100644 --- a/src/__tests__/ai-egress-capability-guard.test.ts +++ b/src/__tests__/ai-egress-capability-guard.test.ts @@ -19,6 +19,7 @@ import { describe, expect, it } from "vitest"; import { readFileSync } from "node:fs"; import { join } from "node:path"; +import { DOCUMENT_AI_RUN_KINDS } from "@/lib/documents/ai-runs/types"; import type { runStatusCompletion } from "@/lib/insights/status-provider"; import { walkSourceFiles } from "./helpers/source-files"; @@ -163,7 +164,7 @@ const EGRESS_SITES: Record = { */ const WORKER_REACHERS: Record = { "lib/jobs/document-ai-run.ts": - "Background document reads and lab scans (v1.40): `transcribeDocument` and `runOcrExtraction` through the shared run bodies.", + "Background document reads and lab scans (v1.40): `transcribeDocument`, `runDocumentSummary`, `runDocumentAssist` and `runOcrExtraction` through the shared run bodies.", }; const JOB_CAPABILITY = /\baiCapabilityForJob\s*\(/; @@ -190,6 +191,50 @@ describe("workers that reach an admitted helper", () => { }); }); +/** The body of one top-level `async function name(` in a source text. */ +function functionBody(text: string, name: string): string | null { + const start = text.search(new RegExp(`async function ${name}\\s*\\(`)); + if (start < 0) return null; + const end = text.indexOf("\n}\n", start); + return end < 0 ? null : text.slice(start, end); +} + +describe("every kind of background run re-checks before its read", () => { + const worker = code("lib/jobs/document-ai-run.ts"); + const table = /const RUN_DISPATCH[^=]*=\s*\{([\s\S]*?)\};/.exec(worker); + const entries = [...(table?.[1] ?? "").matchAll(/(\w+):\s*(\w+),/g)].map( + (m) => ({ kind: m[1], dispatcher: m[2] }), + ); + + it("finds the dispatch table (an empty match is a failure, not a pass)", () => { + expect(entries.length).toBeGreaterThanOrEqual(2); + }); + + it("dispatches every run kind", () => { + expect(entries.map((e) => e.kind).sort()).toEqual( + [...DOCUMENT_AI_RUN_KINDS].sort(), + ); + }); + + it("the shared document pick re-checks the wire on both transports", () => { + const pick = functionBody(worker, "pickForDocumentRead") ?? ""; + expect(pick.match(/\bassertRunMayEgress\s*\(/g)?.length ?? 0).toBe(2); + }); + + it.each(DOCUMENT_AI_RUN_KINDS)( + "%s re-checks the capability and the wire before the body runs", + (kind) => { + const entry = entries.find((e) => e.kind === kind); + const body = entry ? functionBody(worker, entry.dispatcher) : null; + expect(body, `${kind}: dispatcher not found`).not.toBeNull(); + expect( + /\b(?:assertRunMayEgress|pickForDocumentRead)\s*\(/.test(body ?? ""), + `${kind}: its dispatcher sends to a model without assertRunMayEgress`, + ).toBe(true); + }, + ); +}); + describe("AI egress call sites", () => { const found = sourceFiles().filter((rel) => EGRESS_CALL.test(code(rel))); diff --git a/src/app/api/__tests__/ai-capability-route-inventory.test.ts b/src/app/api/__tests__/ai-capability-route-inventory.test.ts index a5b256218..91c3c29c1 100644 --- a/src/app/api/__tests__/ai-capability-route-inventory.test.ts +++ b/src/app/api/__tests__/ai-capability-route-inventory.test.ts @@ -20,6 +20,11 @@ * files (`tests/integration/ai-optional-*.test.ts`) prove behaviour; this * proves no route escapes classification and no data route grows a gate. * + * 5. Every kind of background run (`DOCUMENT_AI_RUN_KINDS`) is queued by a + * route on `AI_ROUTES` that asks the capability the worker asks again for + * that kind (`DOCUMENT_AI_RUN_CAPABILITY`), so a run cannot answer to a + * different switch than the request that started it. + * * Mutation checks (each turned this file red by name): adding * `await requireAiCapability("statusText")` to `insights/ecg/route.ts`; * removing the capability call from `insights/chat/route.ts`; swapping @@ -31,6 +36,11 @@ import { join, relative, resolve } from "node:path"; import { describe, expect, it } from "vitest"; import { AI_CAPABILITY_KEYS } from "@/lib/ai/capabilities/types"; +import { + DOCUMENT_AI_RUN_CAPABILITY, + DOCUMENT_AI_RUN_KINDS, + type DocumentAiRunKindValue, +} from "@/lib/documents/ai-runs/types"; import { AI_ROUTES, DATA_ROUTES } from "./ai-route-inventory"; @@ -154,3 +164,27 @@ describe("AI capability route inventory", () => { expect(both).toEqual([]); }); }); + +/** The route that queues each kind of background run. */ +const RUN_QUEUED_BY: Record = { + DOCUMENT_INDEX: "src/app/api/documents/inbound/[id]/index/route.ts", + LABS_OCR_EXTRACT: "src/app/api/labs/ocr/extract/route.ts", + DOCUMENT_SUMMARY: "src/app/api/documents/inbound/[id]/summary/route.ts", + DOCUMENT_SUGGEST: "src/app/api/documents/inbound/[id]/suggest/route.ts", +}; + +describe("background run kinds", () => { + it.each(DOCUMENT_AI_RUN_KINDS)( + "%s is queued by an AI route under the capability the worker re-checks", + (kind) => { + const path = RUN_QUEUED_BY[kind]; + const entry = AI_ROUTES[path]; + expect(entry, `${path} is not on AI_ROUTES`).toBeDefined(); + expect(entry.capabilities).toContain(DOCUMENT_AI_RUN_CAPABILITY[kind]); + expect( + new RegExp(`kind:\\s*"${kind}"`).test(code(path)), + `${path} does not queue ${kind}`, + ).toBe(true); + }, + ); +}); diff --git a/src/app/api/documents/inbound/[id]/suggest/route.ts b/src/app/api/documents/inbound/[id]/suggest/route.ts index c361d2929..28ac50c64 100644 --- a/src/app/api/documents/inbound/[id]/suggest/route.ts +++ b/src/app/api/documents/inbound/[id]/suggest/route.ts @@ -11,13 +11,21 @@ * The document is UNTRUSTED (prompt-injection): the server never acts on an * instruction inside it. With no provider configured this 422s the enhancement; * the stored document and the manual edit form are untouched. + * + * v1.40 — `Prefer: respond-async` (RFC 7240) runs the read in the background + * worker: every refusal that can be answered quickly is still answered here, + * then the route answers 202 with a run id and `GET /api/ai-runs/{id}` serves + * the same body this route answers with synchronously. Without the header the + * route behaves exactly as before; the iPhone app relies on that. Both paths + * run one body (`executeDocumentSuggest`). */ +import { Buffer } from "node:buffer"; + import { NextRequest } from "next/server"; import { apiHandler, requireAuth } from "@/lib/api-handler"; import { apiError, - apiSuccess, apiValidationError, getClientIp, safeJson, @@ -25,33 +33,36 @@ import { } from "@/lib/api-response"; import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; import { requireAiCapability } from "@/lib/ai/capabilities/gate"; +import type { ProviderChainType } from "@/lib/ai/provider-chain"; import { buildDateKey, - reconcileSpend, reserveBudget, resolveCostOwner, resolveDailyCap, } from "@/lib/ai/coach/budget"; -import { auditLog } from "@/lib/auth/audit"; -import { - DocumentAssistError, - runDocumentAssist, - type DocumentSuggestion, -} from "@/lib/documents/assist"; import { checkDocumentAiRateLimit, documentAiRateLimited, DOCUMENT_AI_TEXT_BODY_MAX_BYTES, loadOwnedDocument, - prepareVisionInput, refundDocumentAiSlot, type LoadedDocument, } from "@/lib/documents/ai-route-support"; +import { + acceptedRunResponse, + outcomeResponse, + prefersRespondAsync, +} from "@/lib/documents/ai-runs/http"; +import { findLiveDocumentRun, startAiRun } from "@/lib/documents/ai-runs/start"; +import { + executeDocumentSuggest, + type DocumentSuggestInput, +} from "@/lib/documents/ai-runs/suggest-run"; +import type { AiRunBudget } from "@/lib/documents/ai-runs/types"; import { requireDocumentTextProvider, requireDocumentVisionProvider, } from "@/lib/documents/provider-order"; -import { annotate } from "@/lib/logging/context"; import { requireModuleEnabled } from "@/lib/modules/gate"; import { prisma } from "@/lib/db"; import { inboundTextExtractSchema } from "@/lib/validations/inbound-documents"; @@ -79,38 +90,75 @@ export const POST = apiHandler( }); } + const background = prefersRespondAsync(request); const contentType = request.headers.get("content-type") ?? ""; if (contentType.includes("application/json")) { - return handleTextSuggest(request, user.id, document); + return handleTextSuggest(request, user.id, document, background); } - return handleVisionSuggest(request, user.id, document); + return handleVisionSuggest(request, user.id, document, background); }, ); -/** Persist the AI-budget ledger side effect + the audit trail. NEVER the row. */ -async function finishSuggest( - request: NextRequest, +/** + * A suggestion read of this document over the same transport, already queued + * or running: attach to it rather than charge a second one. Background only. + */ +async function liveRunResponse( userId: string, documentId: string, mode: "vision" | "text", - suggestion: DocumentSuggestion, -): Promise { - await auditLog("documents.inbound.suggest", { +): Promise { + const live = await findLiveDocumentRun( userId, - ipAddress: getClientIp(request), - details: { documentId, mode }, + documentId, + "DOCUMENT_SUGGEST", + (params) => params.mode === mode, + ); + return live ? acceptedRunResponse(live) : null; +} + +/** Reserve the read's budget; null (slot refunded) when the day is spent. */ +async function reserveFor( + userId: string, + providerType: ProviderChainType, +): Promise { + const dateKey = buildDateKey(); + const reservation = await reserveBudget( + userId, + AI_BUDGETS.documentAssist.maxTokens, + dateKey, + resolveDailyCap([{ providerType }]), + resolveCostOwner([{ providerType }]), + "coach", + ); + if (!reservation.allowed) { + await refundDocumentAiSlot(userId); + return null; + } + return { reserved: reservation.reserved, owner: reservation.owner, dateKey }; +} + +function budgetExceeded(): Response { + return apiError("Your AI usage budget for today is reached.", 429, { + errorCode: "documents.inbound.budgetExceeded", }); - annotate({ - action: { name: "documents.assist.suggest" }, - meta: { - documentId, - mode, - hasTitle: suggestion.title !== null, - hasKind: suggestion.kind !== null, - hasDate: suggestion.documentDate !== null, - }, - }); - return apiSuccess({ suggestions: suggestion }); +} + +/** The synchronous path: run the shared body inside this request. */ +async function runNow( + request: NextRequest, + userId: string, + document: LoadedDocument, + input: DocumentSuggestInput, +): Promise { + return outcomeResponse( + await executeDocumentSuggest({ + userId, + document, + input, + origin: { ipAddress: getClientIp(request), worker: false }, + }), + ); } /** TEXT mode — suggest from in-browser-OCR'd text (opt-in local OCR). */ @@ -118,6 +166,7 @@ async function handleTextSuggest( request: NextRequest, userId: string, document: LoadedDocument, + background: boolean, ): Promise { const row = await prisma.user.findUnique({ where: { id: userId }, @@ -131,6 +180,11 @@ async function handleTextSuggest( const pick = await requireDocumentTextProvider(userId); + if (background) { + const live = await liveRunResponse(userId, document.id, "text"); + if (live) return live; + } + const rl = await checkDocumentAiRateLimit(userId); if (!rl.allowed) return documentAiRateLimited(rl); @@ -154,55 +208,28 @@ async function handleTextSuggest( ); } - const dateKey = buildDateKey(); - const reservation = await reserveBudget( - userId, - AI_BUDGETS.documentAssist.maxTokens, - dateKey, - resolveDailyCap([{ providerType: pick.entry.providerType }]), - resolveCostOwner([{ providerType: pick.entry.providerType }]), - "coach", - ); - if (!reservation.allowed) { - await refundDocumentAiSlot(userId); - return apiError("Your AI usage budget for today is reached.", 429, { - errorCode: "documents.inbound.budgetExceeded", + const budget = await reserveFor(userId, pick.entry.providerType); + if (!budget) return budgetExceeded(); + + if (background) { + // The worker picks the provider again and re-checks the wire for it right + // before the text leaves. + return startAiRun({ + userId, + kind: "DOCUMENT_SUGGEST", + documentId: document.id, + params: { mode: "text", budget }, + input: Buffer.from(parsed.data.text, "utf8"), + refundSlot: () => refundDocumentAiSlot(userId), }); } - try { - const suggestion = await runDocumentAssist({ - provider: pick.entry.instance, - providerType: pick.providerType, - ocrText: parsed.data.text, - }); - await reconcileSpend( - userId, - reservation.reserved, - reservation.reserved, - dateKey, - 0, - { servedBy: pick.entry.providerType, reservedOwner: reservation.owner }, - ); - return finishSuggest(request, userId, document.id, "text", suggestion); - } catch (err) { - await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, { - servedBy: null, - reservedOwner: reservation.owner, - }); - if (err instanceof DocumentAssistError) { - return apiError("Couldn't read the document. Try a clearer copy.", 422, { - errorCode: "documents.inbound.extractFailed", - }); - } - annotate({ - action: { name: "documents.assist.failed" }, - meta: { reason: "provider_error", mode: "text" }, - }); - return apiError("Couldn't read the document. Try a clearer copy.", 502, { - errorCode: "documents.inbound.extractFailed", - }); - } + return runNow(request, userId, document, { + mode: "text", + text: parsed.data.text, + pick, + budget, + }); } /** VISION mode — suggest from the stored original via the vision provider. */ @@ -210,83 +237,32 @@ async function handleVisionSuggest( request: NextRequest, userId: string, document: LoadedDocument, + background: boolean, ): Promise { const pick = await requireDocumentVisionProvider(userId); + if (background) { + const live = await liveRunResponse(userId, document.id, "vision"); + if (live) return live; + } + const rl = await checkDocumentAiRateLimit(userId); if (!rl.allowed) return documentAiRateLimited(rl); - const vision = await prepareVisionInput(document, pick.pdfSupported); - if (!vision.ok) { - // Preparation failed before any provider dispatch — the slot goes back. - await refundDocumentAiSlot(userId); - if (vision.reason === "pdfNeedsAnthropic") { - return apiError( - "PDF scanning needs a Claude vision provider; use local OCR instead.", - 422, - { errorCode: "documents.inbound.pdfNeedsAnthropic" }, - ); - } - if (vision.reason === "fileType") { - return apiError( - "This document can't be scanned. Use local OCR (text mode).", - 422, - { errorCode: "documents.inbound.fileType" }, - ); - } - return apiError("Couldn't read the stored document.", 422, { - errorCode: "documents.inbound.extractFailed", - }); - } + const budget = await reserveFor(userId, pick.entry.providerType); + if (!budget) return budgetExceeded(); - const dateKey = buildDateKey(); - const reservation = await reserveBudget( - userId, - AI_BUDGETS.documentAssist.maxTokens, - dateKey, - resolveDailyCap([{ providerType: pick.entry.providerType }]), - resolveCostOwner([{ providerType: pick.entry.providerType }]), - "coach", - ); - if (!reservation.allowed) { - await refundDocumentAiSlot(userId); - return apiError("Your AI usage budget for today is reached.", 429, { - errorCode: "documents.inbound.budgetExceeded", - }); - } - - try { - const suggestion = await runDocumentAssist({ - provider: pick.entry.instance, - providerType: pick.providerType, - images: vision.images, - documents: vision.documents, - }); - await reconcileSpend( + if (background) { + // The worker decrypts the stored original itself, after it has picked + // the provider again and re-checked the wire for it. + return startAiRun({ userId, - reservation.reserved, - reservation.reserved, - dateKey, - 0, - { servedBy: pick.entry.providerType, reservedOwner: reservation.owner }, - ); - return finishSuggest(request, userId, document.id, "vision", suggestion); - } catch (err) { - await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, { - servedBy: null, - reservedOwner: reservation.owner, - }); - if (err instanceof DocumentAssistError) { - return apiError("Couldn't read the document. Try a clearer copy.", 422, { - errorCode: "documents.inbound.extractFailed", - }); - } - annotate({ - action: { name: "documents.assist.failed" }, - meta: { reason: "provider_error", mode: "vision" }, - }); - return apiError("Couldn't read the document. Try a clearer copy.", 502, { - errorCode: "documents.inbound.extractFailed", + kind: "DOCUMENT_SUGGEST", + documentId: document.id, + params: { mode: "vision", budget }, + refundSlot: () => refundDocumentAiSlot(userId), }); } + + return runNow(request, userId, document, { mode: "vision", pick, budget }); } diff --git a/src/app/api/documents/inbound/[id]/summary/route.ts b/src/app/api/documents/inbound/[id]/summary/route.ts index e9e785df7..9c2c7c484 100644 --- a/src/app/api/documents/inbound/[id]/summary/route.ts +++ b/src/app/api/documents/inbound/[id]/summary/route.ts @@ -18,7 +18,18 @@ * Same VISION/TEXT dispatch and gauntlet as the extract route. The document is * UNTRUSTED (prompt-injection): the server never acts on an instruction inside * it. With no provider configured this 422s; nothing is stored either way. + * + * v1.40 — `Prefer: respond-async` (RFC 7240) runs the read in the background + * worker: every refusal that can be answered quickly is still answered here, + * then the route answers 202 with a run id and `GET /api/ai-runs/{id}` serves + * the same body this route answers with synchronously. A summary that is to + * be stored marks the document PENDING while it is queued. Without the header + * the route behaves exactly as before; the iPhone app relies on that. Both + * paths run one body (`executeDocumentSummary`). A transcription of text the + * browser already read calls no model and is answered here either way. */ +import { Buffer } from "node:buffer"; + import { NextRequest } from "next/server"; import { apiHandler, requireAuth } from "@/lib/api-handler"; @@ -32,9 +43,9 @@ import { } from "@/lib/api-response"; import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; import { requireAiCapability } from "@/lib/ai/capabilities/gate"; +import type { ProviderChainType } from "@/lib/ai/provider-chain"; import { buildDateKey, - reconcileSpend, reserveBudget, resolveCostOwner, resolveDailyCap, @@ -45,19 +56,23 @@ import { documentAiRateLimited, DOCUMENT_AI_TEXT_BODY_MAX_BYTES, loadOwnedDocument, - prepareVisionInput, refundDocumentAiSlot, type LoadedDocument, } from "@/lib/documents/ai-route-support"; import { - DocumentDescribeError, - runDocumentSummary, - documentSummaryBlockedCopy, - transcribeDocument, - type DescribeInput, -} from "@/lib/documents/describe"; -import { encryptDocumentSummary } from "@/lib/documents/store"; -import type { OutboundReason } from "@/lib/ai/safety/outbound-screen"; + acceptedRunResponse, + outcomeResponse, + prefersRespondAsync, +} from "@/lib/documents/ai-runs/http"; +import { findLiveDocumentRun, startAiRun } from "@/lib/documents/ai-runs/start"; +import { + executeDocumentSummary, + markSummaryQueued, + type DocumentSummaryInput, + type SummaryPersistence, +} from "@/lib/documents/ai-runs/summary-run"; +import type { AiRunBudget, AiRunParams } from "@/lib/documents/ai-runs/types"; +import { transcribeDocument } from "@/lib/documents/describe"; import { requireDocumentTextProvider, requireDocumentVisionProvider, @@ -78,11 +93,15 @@ export const dynamic = "force-dynamic"; type RouteParams = { params: Promise<{ id: string }> }; -type SummaryPersistenceTarget = { +/** What one request asks for, read once from the query and the headers. */ +interface SummaryRequest { userId: string; - documentId: string; - replaceExisting: boolean; -}; + document: LoadedDocument; + mode: DocumentSummaryMode; + locale: Locale; + persist: SummaryPersistence | null; + background: boolean; +} function resolveMode(request: NextRequest): DocumentSummaryMode { const raw = new URL(request.url).searchParams.get("mode"); @@ -98,100 +117,6 @@ function resolveReplaceExisting(request: NextRequest): boolean { return new URL(request.url).searchParams.get("replace") === "true"; } -/** - * Run the requested describe call (summary or raw text) over the input. - * - * Ordinary action-row summaries and raw transcription remain session-only. - * The document summary block opts into persistence so an empty document can be - * repaired and an existing stored summary can be explicitly regenerated. - */ -async function describe( - mode: DocumentSummaryMode, - input: DescribeInput, - locale: Locale, - persist: SummaryPersistenceTarget | null, -): Promise< - | { - summary: string; - persistence?: "stored" | "withheld" | "failed"; - } - | { text: string } -> { - if (mode === "text") return transcribeDocument(input); - // A normal summary request fills an empty slot only. Replacement is reserved - // for the document detail's explicit "Generate again" action. - const { summary, blocked } = await runDocumentSummary({ ...input, locale }); - const persistence = persist - ? await persistSummary(persist, summary, blocked) - : undefined; - return { - summary: blocked ? documentSummaryBlockedCopy(locale) : summary, - ...(persistence ? { persistence } : {}), - }; -} - -/** - * Store a screened-clean summary, or record the refusal as a state. - * - * A blocked summary NEVER lands as text — the honest statement the caller shows - * is generated copy for this response, not the model's prose, and persisting it - * would put a refusal where a summary belongs. Only the WITHHELD state is kept, - * and it is not terminal: the user can ask again. - * - * Storing is best-effort. The user is waiting on the summary they can already - * read in the response; a failed write must not turn that into an error. - */ -async function persistSummary( - target: SummaryPersistenceTarget, - summary: string, - blocked: OutboundReason | null, -): Promise<"stored" | "withheld" | "failed"> { - try { - if (blocked) { - await prisma.inboundDocument.updateMany({ - where: { - id: target.documentId, - userId: target.userId, - deletedAt: null, - summaryState: { not: "READY" }, - }, - // An explicit request is the person reading it on purpose, so an - // import's hold on automatic AI reading ends here. - data: { summaryState: "WITHHELD", aiReadDeferred: false }, - }); - return "withheld"; - } - // Preserve a previously stored summary unless the caller explicitly chose - // the replacement action. Failed or screened attempts never reach this - // write, so the previous clean summary remains available in those cases. - const written = await prisma.inboundDocument.updateMany({ - where: { - id: target.documentId, - userId: target.userId, - deletedAt: null, - ...(target.replaceExisting ? {} : { summaryEncrypted: null }), - }, - data: { - summaryEncrypted: encryptDocumentSummary(summary), - summaryGeneratedAt: new Date(), - summaryState: "READY", - aiReadDeferred: false, - }, - }); - annotate({ - action: { name: "documents.summary.persisted" }, - meta: { documentId: target.documentId, stored: written.count > 0 }, - }); - return written.count > 0 ? "stored" : "failed"; - } catch { - annotate({ - action: { name: "documents.summary.persistFailed" }, - meta: { documentId: target.documentId }, - }); - return blocked ? "withheld" : "failed"; - } -} - /** The budget the requested mode charges. */ function budgetFor(mode: DocumentSummaryMode): number { return mode === "text" @@ -219,13 +144,11 @@ export const POST = apiHandler( } const mode = resolveMode(request); - const persist: SummaryPersistenceTarget | null = + // A normal summary request fills an empty slot only. Replacement is + // reserved for the document detail's explicit "Generate again" action. + const persist: SummaryPersistence | null = mode === "summary" && resolvePersistRequested(request) - ? { - userId: user.id, - documentId: document.id, - replaceExisting: resolveReplaceExisting(request), - } + ? { replaceExisting: resolveReplaceExisting(request) } : null; // The outbound screen on the summary needs the reader's locale to pick its // pattern banks; resolve it once here and thread it into both describe legs. @@ -233,57 +156,112 @@ export const POST = apiHandler( request, userLocale: user.locale ?? null, }); - const contentType = request.headers.get("content-type") ?? ""; - if (contentType.includes("application/json")) { - return handleTextSummary( - request, - user.id, - document, - mode, - locale, - persist, - ); - } - return handleVisionSummary( - request, - user.id, + const ask: SummaryRequest = { + userId: user.id, document, mode, locale, persist, - ); + background: prefersRespondAsync(request), + }; + const contentType = request.headers.get("content-type") ?? ""; + if (contentType.includes("application/json")) { + return handleTextSummary(request, ask); + } + return handleVisionSummary(request, ask); }, ); -async function finishSummary( - request: NextRequest, - userId: string, - documentId: string, +/** The run parameters for this request (never health data). */ +function runParams( + ask: SummaryRequest, inputMode: "vision" | "text", - mode: DocumentSummaryMode, - result: { summary: string } | { text: string }, + budget: AiRunBudget, +): AiRunParams { + return { + mode: inputMode, + budget, + summary: { + output: ask.mode, + persist: ask.persist !== null, + replace: ask.persist?.replaceExisting ?? false, + locale: ask.locale, + }, + }; +} + +/** + * A run that will produce the same thing, already queued or running: attach + * to it rather than charge a second one. Background path only. + */ +async function liveRunResponse( + ask: SummaryRequest, + inputMode: "vision" | "text", +): Promise { + const live = await findLiveDocumentRun( + ask.userId, + ask.document.id, + "DOCUMENT_SUMMARY", + (params) => + params.mode === inputMode && + params.summary?.output === ask.mode && + params.summary.persist === (ask.persist !== null) && + params.summary.replace === (ask.persist?.replaceExisting ?? false), + ); + return live ? acceptedRunResponse(live) : null; +} + +/** Reserve the read's budget; null (slot refunded) when the day is spent. */ +async function reserveFor( + ask: SummaryRequest, + providerType: ProviderChainType, +): Promise { + const dateKey = buildDateKey(); + const reservation = await reserveBudget( + ask.userId, + budgetFor(ask.mode), + dateKey, + resolveDailyCap([{ providerType }]), + resolveCostOwner([{ providerType }]), + "coach", + ); + if (!reservation.allowed) { + await refundDocumentAiSlot(ask.userId); + return null; + } + return { reserved: reservation.reserved, owner: reservation.owner, dateKey }; +} + +function budgetExceeded(): Response { + return apiError("Your AI usage budget for today is reached.", 429, { + errorCode: "documents.inbound.budgetExceeded", + }); +} + +/** Queue the read; a summary that is to be stored is marked PENDING first. */ +async function queueSummary( + ask: SummaryRequest, + inputMode: "vision" | "text", + budget: AiRunBudget, + text: string | null, ): Promise { - await auditLog("documents.inbound.summary", { - userId, - ipAddress: getClientIp(request), - details: { documentId, mode, inputMode }, + if (ask.persist) await markSummaryQueued(ask.userId, ask.document.id); + return startAiRun({ + userId: ask.userId, + kind: "DOCUMENT_SUMMARY", + documentId: ask.document.id, + params: runParams(ask, inputMode, budget), + input: text === null ? null : Buffer.from(text, "utf8"), + refundSlot: () => refundDocumentAiSlot(ask.userId), }); - annotate({ - action: { name: "documents.summary.serve" }, - meta: { documentId, mode, inputMode }, - }); - return apiSuccess(result); } /** TEXT mode — summarise / echo in-browser-OCR'd text. */ async function handleTextSummary( request: NextRequest, - userId: string, - document: LoadedDocument, - mode: DocumentSummaryMode, - locale: Locale, - persist: SummaryPersistenceTarget | null, + ask: SummaryRequest, ): Promise { + const { userId, document, mode } = ask; const row = await prisma.user.findUnique({ where: { id: userId }, select: { labsLocalOcrEnabled: true }, @@ -294,6 +272,11 @@ async function handleTextSummary( }); } + if (ask.background && mode !== "text") { + const live = await liveRunResponse(ask, "text"); + if (live) return live; + } + const rl = await checkDocumentAiRateLimit(userId); if (!rl.allowed) return documentAiRateLimited(rl); @@ -319,13 +302,24 @@ async function handleTextSummary( // `mode=text` over posted OCR text is a pure echo — the text IS the // transcription. No provider egress, no consent, no budget: session-only. + // There is nothing to run in the background, so the preference is not + // applied and the answer comes straight back. if (mode === "text") { const result = await transcribeDocument({ provider: {} as never, providerType: "local-ocr", ocrText: parsed.data.text, }); - return finishSummary(request, userId, document.id, "text", mode, result); + await auditLog("documents.inbound.summary", { + userId, + ipAddress: getClientIp(request), + details: { documentId: document.id, mode, inputMode: "text" }, + }); + annotate({ + action: { name: "documents.summary.serve" }, + meta: { documentId: document.id, mode, inputMode: "text" }, + }); + return apiSuccess(result); } let pick: Awaited>; @@ -337,145 +331,65 @@ async function handleTextSummary( throw err; } - const dateKey = buildDateKey(); - const reservation = await reserveBudget( - userId, - budgetFor(mode), - dateKey, - resolveDailyCap([{ providerType: pick.entry.providerType }]), - resolveCostOwner([{ providerType: pick.entry.providerType }]), - "coach", - ); - if (!reservation.allowed) { - await refundDocumentAiSlot(userId); - return apiError("Your AI usage budget for today is reached.", 429, { - errorCode: "documents.inbound.budgetExceeded", - }); + const budget = await reserveFor(ask, pick.entry.providerType); + if (!budget) return budgetExceeded(); + + if (ask.background) { + // The worker picks the provider again and re-checks the wire for it right + // before the text leaves. + return queueSummary(ask, "text", budget, parsed.data.text); } - try { - const result = await describe( - mode, - { - provider: pick.entry.instance, - providerType: pick.providerType, - ocrText: parsed.data.text, - }, - locale, - persist, - ); - await reconcileSpend( - userId, - reservation.reserved, - reservation.reserved, - dateKey, - 0, - { servedBy: pick.entry.providerType, reservedOwner: reservation.owner }, - ); - return finishSummary(request, userId, document.id, "text", mode, result); - } catch (err) { - await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, { - servedBy: null, - reservedOwner: reservation.owner, - }); - return summaryError(err, "text"); - } + return runNow(request, ask, { + mode: "text", + text: parsed.data.text, + pick, + budget, + }); } /** VISION mode — summarise / transcribe the stored original. */ async function handleVisionSummary( request: NextRequest, - userId: string, - document: LoadedDocument, - mode: DocumentSummaryMode, - locale: Locale, - persist: SummaryPersistenceTarget | null, + ask: SummaryRequest, ): Promise { - const pick = await requireDocumentVisionProvider(userId); + const pick = await requireDocumentVisionProvider(ask.userId); - const rl = await checkDocumentAiRateLimit(userId); + if (ask.background) { + const live = await liveRunResponse(ask, "vision"); + if (live) return live; + } + + const rl = await checkDocumentAiRateLimit(ask.userId); if (!rl.allowed) return documentAiRateLimited(rl); - const vision = await prepareVisionInput(document, pick.pdfSupported); - if (!vision.ok) { - // Preparation failed before any provider dispatch — the slot goes back. - await refundDocumentAiSlot(userId); - if (vision.reason === "pdfNeedsAnthropic") { - return apiError( - "PDF scanning needs a Claude vision provider; use local OCR instead.", - 422, - { errorCode: "documents.inbound.pdfNeedsAnthropic" }, - ); - } - if (vision.reason === "fileType") { - return apiError( - "This document can't be scanned. Use local OCR (text mode).", - 422, - { errorCode: "documents.inbound.fileType" }, - ); - } - return apiError("Couldn't read the stored document.", 422, { - errorCode: "documents.inbound.extractFailed", - }); + const budget = await reserveFor(ask, pick.entry.providerType); + if (!budget) return budgetExceeded(); + + if (ask.background) { + // The worker decrypts the stored original itself, after it has picked + // the provider again and re-checked the wire for it. + return queueSummary(ask, "vision", budget, null); } - const dateKey = buildDateKey(); - const reservation = await reserveBudget( - userId, - budgetFor(mode), - dateKey, - resolveDailyCap([{ providerType: pick.entry.providerType }]), - resolveCostOwner([{ providerType: pick.entry.providerType }]), - "coach", + return runNow(request, ask, { mode: "vision", pick, budget }); +} + +/** The synchronous path: run the shared body inside this request. */ +async function runNow( + request: NextRequest, + ask: SummaryRequest, + input: DocumentSummaryInput, +): Promise { + return outcomeResponse( + await executeDocumentSummary({ + userId: ask.userId, + document: ask.document, + input, + output: ask.mode, + locale: ask.locale, + persist: ask.persist, + origin: { ipAddress: getClientIp(request), worker: false }, + }), ); - if (!reservation.allowed) { - await refundDocumentAiSlot(userId); - return apiError("Your AI usage budget for today is reached.", 429, { - errorCode: "documents.inbound.budgetExceeded", - }); - } - - try { - const result = await describe( - mode, - { - provider: pick.entry.instance, - providerType: pick.providerType, - images: vision.images, - documents: vision.documents, - }, - locale, - persist, - ); - await reconcileSpend( - userId, - reservation.reserved, - reservation.reserved, - dateKey, - 0, - { servedBy: pick.entry.providerType, reservedOwner: reservation.owner }, - ); - return finishSummary(request, userId, document.id, "vision", mode, result); - } catch (err) { - await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, { - servedBy: null, - reservedOwner: reservation.owner, - }); - return summaryError(err, "vision"); - } -} - -function summaryError(err: unknown, mode: "vision" | "text"): Response { - if (err instanceof DocumentDescribeError) { - return apiError("Couldn't read the document. Try a clearer copy.", 422, { - errorCode: "documents.inbound.extractFailed", - }); - } - annotate({ - action: { name: "documents.summary.failed" }, - meta: { reason: "provider_error", mode }, - }); - return apiError("Couldn't read the document. Try a clearer copy.", 502, { - errorCode: "documents.inbound.extractFailed", - }); } diff --git a/src/components/documents/__tests__/document-ai-section.test.tsx b/src/components/documents/__tests__/document-ai-section.test.tsx index c0af03212..742859335 100644 --- a/src/components/documents/__tests__/document-ai-section.test.tsx +++ b/src/components/documents/__tests__/document-ai-section.test.tsx @@ -254,3 +254,35 @@ describe(" index outcome (#776)", () => { expect(html).not.toContain('data-slot="document-index-outcome"'); }); }); + +describe(" background reads (v1.40)", () => { + it("a running suggestion or summary says it reads in the background without promising it is kept", () => { + const html = render(base({ assistRunPhase: "background" })); + expect(html).toContain('data-slot="document-assist-run-phase"'); + expect(html).toContain("With a slow model this can take a few minutes."); + expect(html).not.toContain("You can leave this page"); + }); + + it("a read whose result lands on the document says the person may leave", () => { + const html = render( + base({ + indexRunPhase: "background", + assistRunPhase: "background", + }), + ); + expect(html).toContain('data-slot="document-index-run-phase"'); + expect(html).toContain("You can leave this page"); + // One line at a time, never two stacked notes. + expect(html).not.toContain('data-slot="document-assist-run-phase"'); + }); + + it("says when no worker has picked the read up yet", () => { + const html = render(base({ assistRunPhase: "waitingForWorker" })); + expect(html).toContain("Waiting for the background worker."); + }); + + it("renders no note while nothing runs", () => { + const html = render(base()); + expect(html).not.toContain("run-phase"); + }); +}); diff --git a/src/components/documents/__tests__/document-summary-block.test.tsx b/src/components/documents/__tests__/document-summary-block.test.tsx index bdf46c1c3..a782849b9 100644 --- a/src/components/documents/__tests__/document-summary-block.test.tsx +++ b/src/components/documents/__tests__/document-summary-block.test.tsx @@ -163,3 +163,15 @@ describe("DocumentSummaryBlock — the action", () => { ); }); }); + +describe("DocumentSummaryBlock — a background read (v1.40)", () => { + it("says the read runs in the background and is stored with the document", () => { + const html = base({ isGenerating: true, runPhase: "background" }); + expect(html).toContain('data-slot="document-detail-summary-run-phase"'); + expect(html).toContain("You can leave this page"); + }); + + it("renders no note while nothing runs", () => { + expect(base()).not.toContain("run-phase"); + }); +}); diff --git a/src/components/documents/document-ai-panels.tsx b/src/components/documents/document-ai-panels.tsx index 919711f4c..daf342284 100644 --- a/src/components/documents/document-ai-panels.tsx +++ b/src/components/documents/document-ai-panels.tsx @@ -12,6 +12,8 @@ * user applies it (and the title lands in the editable field, not on disk). * - `DocumentSummaryPanel` — the transient summary / extracted-text panel with * the persistent "not saved · not a diagnosis" note. + * - `AiRunPhaseNote` — the calm line under an action while its read runs in + * the background (v1.40), shared by every document AI action. */ import { Sparkles } from "lucide-react"; import { Check, FileText, X } from "lucide-react"; @@ -19,6 +21,7 @@ import Link from "next/link"; import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; +import type { AiRunPhase } from "@/hooks/use-ai-run"; import { useTranslations } from "@/lib/i18n/context"; import { cn } from "@/lib/utils"; import { @@ -28,6 +31,37 @@ import { import type { DocumentDescribeResult } from "./use-document-assist"; +/** + * The line under a document AI action while its read runs in the background, + * or nothing when no read is running. + * + * `outcome` keeps the sentence true: a read whose result lands on the document + * (the index, a stored summary, staged facts) tells the person they may leave; + * a read whose answer lives only on this screen (suggestions, a transient + * summary or transcription) does not promise that, because leaving drops it. + */ +export function AiRunPhaseNote({ + phase, + outcome, + slot, +}: { + phase: AiRunPhase; + outcome: "savedWithDocument" | "shownHere"; + slot: string; +}) { + const { t } = useTranslations(); + if (phase === "idle") return null; + return ( +

+ {phase === "waitingForWorker" + ? t("aiRuns.waitingForWorker") + : outcome === "savedWithDocument" + ? t("aiRuns.backgroundDocument") + : t("aiRuns.backgroundScan")} +

+ ); +} + /** * Calm pointer to the AI settings, shown when assist is unavailable. When the * document is already searchable (auto-indexed locally) the copy stays honest — diff --git a/src/components/documents/document-ai-section.tsx b/src/components/documents/document-ai-section.tsx index 4ec4f024d..36beed24d 100644 --- a/src/components/documents/document-ai-section.tsx +++ b/src/components/documents/document-ai-section.tsx @@ -45,6 +45,7 @@ import { } from "@/lib/validations/inbound-documents"; import { + AiRunPhaseNote, AiUnavailableHint, AssistSuggestionReview, DocumentSummaryPanel, @@ -79,6 +80,7 @@ export function DocumentAiSection({ lastIndexOutcome = null, indexPending, indexRunPhase = "idle", + assistRunPhase = "idle", onIndex, }: { aiEnabled: boolean; @@ -124,6 +126,8 @@ export function DocumentAiSection({ indexPending: boolean; /** v1.40 — where a background read stands, for the calm line under it. */ indexRunPhase?: AiRunPhase; + /** v1.40 — the same for a suggestion or transient summary read. */ + assistRunPhase?: AiRunPhase; onIndex: () => void; }) { const { t } = useTranslations(); @@ -253,16 +257,18 @@ export function DocumentAiSection({ ) : null} {indexRunPhase !== "idle" ? ( -

- {indexRunPhase === "waitingForWorker" - ? t("aiRuns.waitingForWorker") - : t("aiRuns.backgroundDocument")} -

- ) : null} + + ) : ( + + )} {indexOutcomeText ? (

@@ -938,6 +939,9 @@ export function DocumentDetailSheet({ lastIndexOutcome={doc.lastIndexOutcome} indexPending={indexDoc.isPending} indexRunPhase={indexDoc.runPhase} + assistRunPhase={ + suggest.isPending ? suggest.runPhase : summary.runPhase + } onIndex={runIndex} /> )} diff --git a/src/components/documents/document-summary-block.tsx b/src/components/documents/document-summary-block.tsx index 7828f5475..7bf383a49 100644 --- a/src/components/documents/document-summary-block.tsx +++ b/src/components/documents/document-summary-block.tsx @@ -30,9 +30,12 @@ import { FileText } from "lucide-react"; import { Button } from "@/components/ui/button"; +import type { AiRunPhase } from "@/hooks/use-ai-run"; import { useTranslations } from "@/lib/i18n/context"; import type { DocumentSummaryStateValue } from "@/lib/validations/inbound-documents"; +import { AiRunPhaseNote } from "./document-ai-panels"; + /** Copy key for a state with no summary to show. */ function absentCopyKey(state: DocumentSummaryStateValue): string { if (state === "WITHHELD") return "documents.detail.summary.withheld"; @@ -46,6 +49,7 @@ export function DocumentSummaryBlock({ generatedAtLabel, aiEnabled, isGenerating, + runPhase = "idle", actionsDisabled, canGenerate = true, onGenerate, @@ -58,6 +62,11 @@ export function DocumentSummaryBlock({ /** Whether an AI provider is configured; false hides the action, not the state. */ aiEnabled: boolean; isGenerating: boolean; + /** + * v1.40 — where the background read stands while one runs. The summary is + * stored on the document, so the line tells the person they may leave. + */ + runPhase?: AiRunPhase; /** Capability still resolving — the transport mode isn't known yet. */ actionsDisabled: boolean; /** @@ -125,6 +134,11 @@ export function DocumentSummaryBlock({ : t("documents.detail.summary.generate")} ) : null} + ); } diff --git a/src/components/documents/use-document-assist.ts b/src/components/documents/use-document-assist.ts index 98f959400..0c26ecdc0 100644 --- a/src/components/documents/use-document-assist.ts +++ b/src/components/documents/use-document-assist.ts @@ -15,9 +15,10 @@ */ import { useCallback } from "react"; -import { useMutation, useQuery } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useAiProviderState } from "@/hooks/use-ai-capability"; +import { useAiRunPhase } from "@/hooks/use-ai-run"; import { apiGet, ApiError } from "@/lib/api/api-fetch"; import { useTranslations } from "@/lib/i18n/context"; import { queryKeys } from "@/lib/query-keys"; @@ -85,10 +86,15 @@ export function useDocumentsAutoAiRead(enabled: boolean) { * Suggest filing metadata for a stored document. Returns DRAFTS only — the * detail sheet prefills its edit fields and the user saves; this call writes * nothing. + * + * The read runs in the background (v1.40), so no proxy in front of the server + * can cut a slow model; `runPhase` says so while it does. */ export function useSuggestDetails() { + const queryClient = useQueryClient(); const { responseTimeoutMs } = useAiProviderState(); - return useMutation< + const { phase, onQueued, onProgress, reset } = useAiRunPhase(); + const mutation = useMutation< DocumentSuggestionDto, Error, { mode: DocumentAiMode; target: DocumentAiTarget } @@ -100,19 +106,25 @@ export function useSuggestDetails() { target, responseTimeoutMs, modelCalls: 1, + background: { queryClient, onQueued, onProgress }, }); return data.suggestions; }, + onSettled: reset, }); + return { ...mutation, runPhase: mutation.isPending ? phase : "idle" }; } /** * On-demand summary or extracted text. Both stay transient unless the document - * summary block explicitly requests persistence or replacement. + * summary block explicitly requests persistence or replacement. The read runs + * in the background (v1.40); `runPhase` says so while it does. */ export function useDocumentSummary() { + const queryClient = useQueryClient(); const { responseTimeoutMs } = useAiProviderState(); - return useMutation< + const { phase, onQueued, onProgress, reset } = useAiRunPhase(); + const mutation = useMutation< DocumentDescribeResult, Error, { @@ -130,8 +142,11 @@ export function useDocumentSummary() { target, responseTimeoutMs, modelCalls: 1, + background: { queryClient, onQueued, onProgress }, }), + onSettled: reset, }); + return { ...mutation, runPhase: mutation.isPending ? phase : "idle" }; } /** diff --git a/src/lib/documents/ai-runs/run-support.ts b/src/lib/documents/ai-runs/run-support.ts new file mode 100644 index 000000000..340f86b10 --- /dev/null +++ b/src/lib/documents/ai-runs/run-support.ts @@ -0,0 +1,74 @@ +/** + * Pieces the summary, suggest and extract run bodies share: the provider pick + * types, the failure shape, and the answer for a stored original that could + * not be turned into a provider input. + * + * Every failure here carries the status, message and `documents.inbound.*` + * code the synchronous route has always answered with, so the background run + * reports exactly what the request would have. + */ +import type { + requireDocumentTextProvider, + requireDocumentVisionProvider, +} from "@/lib/documents/provider-order"; +import type { VisionInput } from "@/lib/documents/ai-route-support"; + +import type { AiRunOutcome } from "./types"; + +export type DocumentVisionPick = Awaited< + ReturnType +>; +export type DocumentTextPick = Awaited< + ReturnType +>; + +/** Who asked: a request (with its client address) or a background run. */ +export interface RunOrigin { + ipAddress: string | null; + worker: boolean; +} + +/** A failure in the shape the route would answer with. */ +export const runFailure = (f: { + status: number; + message: string; + errorCode: string; +}): AiRunOutcome => ({ ok: false, ...f }); + +/** The provider error every document read answers with (422 parse, 502 wire). */ +export function unreadableDocument(parseError: boolean): AiRunOutcome { + return runFailure({ + status: parseError ? 422 : 502, + message: "Couldn't read the document. Try a clearer copy.", + errorCode: "documents.inbound.extractFailed", + }); +} + +/** + * A stored original that could not become a provider input, as the summary + * and suggest routes have always answered it. + */ +export function visionPreparationFailure( + reason: Extract["reason"], +): AiRunOutcome { + if (reason === "pdfNeedsAnthropic") { + return runFailure({ + status: 422, + message: + "PDF scanning needs a Claude vision provider; use local OCR instead.", + errorCode: "documents.inbound.pdfNeedsAnthropic", + }); + } + if (reason === "fileType") { + return runFailure({ + status: 422, + message: "This document can't be scanned. Use local OCR (text mode).", + errorCode: "documents.inbound.fileType", + }); + } + return runFailure({ + status: 422, + message: "Couldn't read the stored document.", + errorCode: "documents.inbound.extractFailed", + }); +} diff --git a/src/lib/documents/ai-runs/start.ts b/src/lib/documents/ai-runs/start.ts index e53455c18..202c591fd 100644 --- a/src/lib/documents/ai-runs/start.ts +++ b/src/lib/documents/ai-runs/start.ts @@ -23,13 +23,18 @@ import { * The live run of this kind for this document, if one is queued or running: * a second press of "Read with AI" (another tab, a double click) attaches to * it instead of paying for a second read. + * + * `paramsMatch` narrows the attach to a run that will produce the same thing: + * a summary that is to be stored must not attach to a transient transcription + * of the same document. */ export async function findLiveDocumentRun( userId: string, documentId: string, kind: DocumentAiRunKindValue, + paramsMatch?: (params: AiRunParams) => boolean, ): Promise { - const row = await prisma.documentAiRun.findFirst({ + const rows = await prisma.documentAiRun.findMany({ where: { userId, documentId, @@ -37,10 +42,14 @@ export async function findLiveDocumentRun( status: { in: ["QUEUED", "RUNNING"] }, expiresAt: { gt: new Date() }, }, - select: { id: true }, + select: { id: true, paramsJson: true }, orderBy: { createdAt: "desc" }, + take: 10, }); - return row?.id ?? null; + const live = paramsMatch + ? rows.find((row) => paramsMatch(row.paramsJson as unknown as AiRunParams)) + : rows[0]; + return live?.id ?? null; } export async function startAiRun(args: { diff --git a/src/lib/documents/ai-runs/suggest-run.ts b/src/lib/documents/ai-runs/suggest-run.ts new file mode 100644 index 000000000..d6fa7a9db --- /dev/null +++ b/src/lib/documents/ai-runs/suggest-run.ts @@ -0,0 +1,112 @@ +/** + * Filing suggestions for a stored document: the body both the synchronous + * suggest route and the background run execute. + * + * One provider call over the stored original (vision) or text the browser read + * (text) returns a `{ title, kind, documentDate }` draft. Nothing is written to + * the document: the person reviews the draft and saves it on the edit form. + * The reservation the caller took is settled here. + */ +import type { ProviderChainType } from "@/lib/ai/provider-chain"; +import { auditLog } from "@/lib/auth/audit"; +import { + prepareVisionInput, + refundDocumentAiSlot, + type LoadedDocument, +} from "@/lib/documents/ai-route-support"; +import { + DocumentAssistError, + runDocumentAssist, + type DocumentSuggestion, +} from "@/lib/documents/assist"; +import { annotate } from "@/lib/logging/context"; + +import { + unreadableDocument, + visionPreparationFailure, + type DocumentTextPick, + type DocumentVisionPick, + type RunOrigin, +} from "./run-support"; +import { settleAiRunBudget } from "./store"; +import type { + AiRunBudget, + AiRunOutcome, + DocumentSuggestRunResult, +} from "./types"; + +/** One provider call. */ +export const DOCUMENT_SUGGEST_MODEL_CALLS = 1; + +export type DocumentSuggestInput = + | { mode: "text"; text: string; pick: DocumentTextPick; budget: AiRunBudget } + | { mode: "vision"; pick: DocumentVisionPick; budget: AiRunBudget }; + +/** Suggest filing details for one owned document. The caller authorised and charged it. */ +export async function executeDocumentSuggest(args: { + userId: string; + document: LoadedDocument; + input: DocumentSuggestInput; + origin: RunOrigin; +}): Promise> { + const { userId, document, input, origin } = args; + const { pick, budget } = input; + const servedBy = pick.entry.providerType as ProviderChainType; + + let suggestion: DocumentSuggestion; + try { + if (input.mode === "text") { + suggestion = await runDocumentAssist({ + provider: pick.entry.instance, + providerType: pick.providerType, + ocrText: input.text, + }); + } else { + const vision = await prepareVisionInput( + document, + input.pick.pdfSupported, + ); + if (!vision.ok) { + // Preparation failed before any provider dispatch: the reservation + // and the slot go back. + await settleAiRunBudget(userId, budget, 0, null); + await refundDocumentAiSlot(userId); + return visionPreparationFailure(vision.reason); + } + suggestion = await runDocumentAssist({ + provider: pick.entry.instance, + providerType: pick.providerType, + images: vision.images, + documents: vision.documents, + }); + } + } catch (err) { + await settleAiRunBudget(userId, budget, 0, null); + if (!(err instanceof DocumentAssistError)) { + annotate({ + action: { name: "documents.assist.failed" }, + meta: { reason: "provider_error", mode: input.mode }, + }); + } + return unreadableDocument(err instanceof DocumentAssistError); + } + await settleAiRunBudget(userId, budget, budget.reserved, servedBy); + + await auditLog("documents.inbound.suggest", { + userId, + ...(origin.worker ? { actorUserId: userId } : {}), + ipAddress: origin.ipAddress, + details: { documentId: document.id, mode: input.mode }, + }); + annotate({ + action: { name: "documents.assist.suggest" }, + meta: { + documentId: document.id, + mode: input.mode, + hasTitle: suggestion.title !== null, + hasKind: suggestion.kind !== null, + hasDate: suggestion.documentDate !== null, + }, + }); + return { ok: true, data: { suggestions: suggestion } }; +} diff --git a/src/lib/documents/ai-runs/summary-run.ts b/src/lib/documents/ai-runs/summary-run.ts new file mode 100644 index 000000000..898a7e2a3 --- /dev/null +++ b/src/lib/documents/ai-runs/summary-run.ts @@ -0,0 +1,250 @@ +/** + * A document summary or transcription: the body both the synchronous summary + * route and the background run execute, so the two paths cannot drift. + * + * Vision mode decrypts the stored original and makes one provider call; text + * mode sends text the browser read on the device. The call runs against a + * reservation the caller already took, which is settled here. A summary the + * caller asked to keep is stored on the document (or its refusal recorded as + * a state); a transcription and an ordinary summary stay transient. + * + * Every failure comes back as the outcome the route has always answered with. + */ +import type { ProviderChainType } from "@/lib/ai/provider-chain"; +import type { OutboundReason } from "@/lib/ai/safety/outbound-screen"; +import { auditLog } from "@/lib/auth/audit"; +import { prisma } from "@/lib/db"; +import { + prepareVisionInput, + refundDocumentAiSlot, + type LoadedDocument, +} from "@/lib/documents/ai-route-support"; +import { + DocumentDescribeError, + documentSummaryBlockedCopy, + runDocumentSummary, + transcribeDocument, + type DescribeInput, +} from "@/lib/documents/describe"; +import { encryptDocumentSummary } from "@/lib/documents/store"; +import type { Locale } from "@/lib/i18n/config"; +import { annotate } from "@/lib/logging/context"; + +import { + unreadableDocument, + visionPreparationFailure, + type DocumentTextPick, + type DocumentVisionPick, + type RunOrigin, +} from "./run-support"; +import { settleAiRunBudget } from "./store"; +import type { + AiRunBudget, + AiRunOutcome, + DocumentSummaryRunResult, +} from "./types"; + +/** One provider call: the summary, or the transcription. */ +export const DOCUMENT_SUMMARY_MODEL_CALLS = 1; + +export type DocumentSummaryOutput = "summary" | "text"; + +export type DocumentSummaryInput = + | { mode: "text"; text: string; pick: DocumentTextPick; budget: AiRunBudget } + | { mode: "vision"; pick: DocumentVisionPick; budget: AiRunBudget }; + +export interface SummaryPersistence { + /** Replace a stored summary; without it only an empty slot is filled. */ + replaceExisting: boolean; +} + +/** + * Store a screened-clean summary, or record the refusal as a state. + * + * A blocked summary NEVER lands as text: the honest statement the caller shows + * is generated copy for this response, not the model's prose, and persisting it + * would put a refusal where a summary belongs. Only the WITHHELD state is kept, + * and it is not terminal: the person can ask again. + * + * Storing is best-effort. The summary is already in the answer; a failed write + * must not turn that into an error. + */ +async function persistSummary( + userId: string, + documentId: string, + persist: SummaryPersistence, + summary: string, + blocked: OutboundReason | null, +): Promise<"stored" | "withheld" | "failed"> { + try { + if (blocked) { + await prisma.inboundDocument.updateMany({ + where: { + id: documentId, + userId, + deletedAt: null, + summaryState: { not: "READY" }, + }, + // An explicit request is the person reading it on purpose, so an + // import's hold on automatic AI reading ends here. + data: { summaryState: "WITHHELD", aiReadDeferred: false }, + }); + return "withheld"; + } + // Preserve a previously stored summary unless the caller explicitly chose + // the replacement action. Failed or screened attempts never reach this + // write, so the previous clean summary remains available in those cases. + const written = await prisma.inboundDocument.updateMany({ + where: { + id: documentId, + userId, + deletedAt: null, + ...(persist.replaceExisting ? {} : { summaryEncrypted: null }), + }, + data: { + summaryEncrypted: encryptDocumentSummary(summary), + summaryGeneratedAt: new Date(), + summaryState: "READY", + aiReadDeferred: false, + }, + }); + annotate({ + action: { name: "documents.summary.persisted" }, + meta: { documentId, stored: written.count > 0 }, + }); + return written.count > 0 ? "stored" : "failed"; + } catch { + annotate({ + action: { name: "documents.summary.persistFailed" }, + meta: { documentId }, + }); + return blocked ? "withheld" : "failed"; + } +} + +/** + * Mark a summary the background run was asked to store as not produced. Only + * the PENDING state the queuing route set is moved, so a summary that landed + * meanwhile (another tab, the automatic job) is never overwritten. + */ +export async function markQueuedSummaryUnavailable( + userId: string, + documentId: string, +): Promise { + try { + await prisma.inboundDocument.updateMany({ + where: { id: documentId, userId, summaryState: "PENDING" }, + data: { summaryState: "UNAVAILABLE" }, + }); + } catch { + // The run's failure is already decided; the hourly summary reaper moves a + // PENDING state that outlives its job. + } +} + +/** + * Mark a summary as being generated when a run that will store it is queued, + * so the detail view says so across reloads. Never over a stored summary: a + * replacement keeps showing the old one until the new one lands. + */ +export async function markSummaryQueued( + userId: string, + documentId: string, +): Promise { + await prisma.inboundDocument.updateMany({ + where: { + id: documentId, + userId, + deletedAt: null, + summaryEncrypted: null, + }, + data: { summaryState: "PENDING" }, + }); +} + +/** Summarise or transcribe one owned document. The caller authorised and charged it. */ +export async function executeDocumentSummary(args: { + userId: string; + document: LoadedDocument; + input: DocumentSummaryInput; + output: DocumentSummaryOutput; + locale: Locale; + persist: SummaryPersistence | null; + origin: RunOrigin; +}): Promise> { + const { userId, document, input, output, locale, origin } = args; + const { pick, budget } = input; + const servedBy = pick.entry.providerType as ProviderChainType; + + let describeInput: DescribeInput; + if (input.mode === "text") { + describeInput = { + provider: pick.entry.instance, + providerType: pick.providerType, + ocrText: input.text, + }; + } else { + const vision = await prepareVisionInput(document, input.pick.pdfSupported); + if (!vision.ok) { + // Preparation failed before any provider dispatch: the reservation and + // the slot go back. + await settleAiRunBudget(userId, budget, 0, null); + await refundDocumentAiSlot(userId); + return visionPreparationFailure(vision.reason); + } + describeInput = { + provider: pick.entry.instance, + providerType: pick.providerType, + images: vision.images, + documents: vision.documents, + }; + } + + let result: DocumentSummaryRunResult; + try { + if (output === "text") { + result = await transcribeDocument(describeInput); + } else { + const { summary, blocked } = await runDocumentSummary({ + ...describeInput, + locale, + }); + const persistence = args.persist + ? await persistSummary( + userId, + document.id, + args.persist, + summary, + blocked, + ) + : undefined; + result = { + summary: blocked ? documentSummaryBlockedCopy(locale) : summary, + ...(persistence ? { persistence } : {}), + }; + } + } catch (err) { + await settleAiRunBudget(userId, budget, 0, null); + if (!(err instanceof DocumentDescribeError)) { + annotate({ + action: { name: "documents.summary.failed" }, + meta: { reason: "provider_error", mode: input.mode }, + }); + } + return unreadableDocument(err instanceof DocumentDescribeError); + } + await settleAiRunBudget(userId, budget, budget.reserved, servedBy); + + await auditLog("documents.inbound.summary", { + userId, + // A background run has no request left to attribute the action from. + ...(origin.worker ? { actorUserId: userId } : {}), + ipAddress: origin.ipAddress, + details: { documentId: document.id, mode: output, inputMode: input.mode }, + }); + annotate({ + action: { name: "documents.summary.serve" }, + meta: { documentId: document.id, mode: output, inputMode: input.mode }, + }); + return { ok: true, data: result }; +} diff --git a/src/lib/documents/ai-runs/types.ts b/src/lib/documents/ai-runs/types.ts index a8ce2d599..74986eb77 100644 --- a/src/lib/documents/ai-runs/types.ts +++ b/src/lib/documents/ai-runs/types.ts @@ -8,14 +8,33 @@ * `GET /api/ai-runs/{id}` until the run is terminal. The result lives in the * row for an hour after it finishes and then the row is deleted. */ +import type { AiCapabilityKey } from "@/lib/ai/capabilities/types"; +import type { Locale } from "@/lib/i18n/config"; +import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; import type { OcrExtractResponseDto } from "@/lib/validations/labs-ocr"; export const DOCUMENT_AI_RUN_KINDS = [ "DOCUMENT_INDEX", "LABS_OCR_EXTRACT", + "DOCUMENT_SUMMARY", + "DOCUMENT_SUGGEST", ] as const; export type DocumentAiRunKindValue = (typeof DOCUMENT_AI_RUN_KINDS)[number]; +/** + * The capability each kind of run answers to: the one its queuing route asks, + * asked again by the worker for the record before anything leaves. + */ +export const DOCUMENT_AI_RUN_CAPABILITY: Record< + DocumentAiRunKindValue, + AiCapabilityKey +> = { + DOCUMENT_INDEX: "documentAi", + LABS_OCR_EXTRACT: "labsOcr", + DOCUMENT_SUMMARY: "documentAi", + DOCUMENT_SUGGEST: "documentAi", +}; + export const DOCUMENT_AI_RUN_STATUSES = [ "QUEUED", "RUNNING", @@ -64,6 +83,13 @@ export interface AiRunParams { mime?: "image/jpeg" | "image/png" | "image/webp" | "application/pdf"; /** Present when the enqueue reserved budget for a provider call. */ budget?: AiRunBudget; + /** A summary run: what to produce, whether to store it, and in which language. */ + summary?: { + output: "summary" | "text"; + persist: boolean; + replace: boolean; + locale: Locale; + }; } /** "Read with AI": the same body the synchronous index route answers with. */ @@ -74,7 +100,25 @@ export interface DocumentIndexRunResult { labFactsStaged: number; } -export type AiRunResult = DocumentIndexRunResult | OcrExtractResponseDto; +/** A summary run: the summary route's body. */ +export type DocumentSummaryRunResult = + | { summary: string; persistence?: "stored" | "withheld" | "failed" } + | { text: string }; + +/** A suggest run: the suggest route's body (drafts, nothing written). */ +export interface DocumentSuggestRunResult { + suggestions: { + title: string | null; + kind: InboundDocumentKindValue | null; + documentDate: string | null; + }; +} + +export type AiRunResult = + | DocumentIndexRunResult + | OcrExtractResponseDto + | DocumentSummaryRunResult + | DocumentSuggestRunResult; /** * What one run body produced: the success body, or the failure the diff --git a/src/lib/jobs/__tests__/document-ai-run.test.ts b/src/lib/jobs/__tests__/document-ai-run.test.ts index 56b65972c..752157713 100644 --- a/src/lib/jobs/__tests__/document-ai-run.test.ts +++ b/src/lib/jobs/__tests__/document-ai-run.test.ts @@ -13,7 +13,11 @@ * in `assertRunMayEgress` turns "refuses when consent was withdrawn" red; * deleting the `aiCapabilityForJob` check turns "refuses when the operator * turned reading off" red; dropping the `ends > deadlineAt` comparison turns - * "refuses a read that cannot fit" red. + * "refuses a read that cannot fit" red. For the summary and suggest kinds: + * deleting `afterRunFailed` (or its `markQueuedSummaryUnavailable` call) + * turns "a stored summary that fails is marked UNAVAILABLE" red, and deleting + * the `assertRunMayEgress` call in `pickForDocumentRead`'s text branch turns + * "a text suggestion re-checks the wire" red. */ import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -38,9 +42,12 @@ vi.mock("@/lib/ai/capabilities/egress", () => ({ })); const requireDocumentVisionProvider = vi.fn(); +const requireDocumentTextProvider = vi.fn(); vi.mock("@/lib/documents/provider-order", () => ({ requireDocumentVisionProvider: (...a: unknown[]) => requireDocumentVisionProvider(...a), + requireDocumentTextProvider: (...a: unknown[]) => + requireDocumentTextProvider(...a), })); const requireLabsOcrProvider = vi.fn(); vi.mock("@/lib/labs/ocr-capability", () => ({ @@ -58,6 +65,20 @@ vi.mock("@/lib/labs/ocr-run", () => ({ executeOcrExtraction: (...a: unknown[]) => executeOcrExtraction(...a), })); +const executeDocumentSummary = vi.fn(); +const markQueuedSummaryUnavailable = vi.fn(); +vi.mock("@/lib/documents/ai-runs/summary-run", () => ({ + DOCUMENT_SUMMARY_MODEL_CALLS: 1, + executeDocumentSummary: (...a: unknown[]) => executeDocumentSummary(...a), + markQueuedSummaryUnavailable: (...a: unknown[]) => + markQueuedSummaryUnavailable(...a), +})); +const executeDocumentSuggest = vi.fn(); +vi.mock("@/lib/documents/ai-runs/suggest-run", () => ({ + DOCUMENT_SUGGEST_MODEL_CALLS: 1, + executeDocumentSuggest: (...a: unknown[]) => executeDocumentSuggest(...a), +})); + const loadOwnedDocument = vi.fn(); vi.mock("@/lib/documents/ai-route-support", () => ({ loadOwnedDocument: (...a: unknown[]) => loadOwnedDocument(...a), @@ -333,3 +354,203 @@ describe("handleDocumentAiRunJobs", () => { }); }); }); + +describe("summary and suggestion runs", () => { + const summaryRun = (overrides: Record = {}) => + claimed({ + kind: "DOCUMENT_SUMMARY", + params: { + mode: "vision", + budget, + summary: { + output: "summary", + persist: true, + replace: false, + locale: "de", + }, + }, + ...overrides, + }); + + beforeEach(() => { + executeDocumentSummary.mockResolvedValue({ + ok: true, + data: { summary: "Ein Arztbrief.", persistence: "stored" }, + }); + executeDocumentSuggest.mockResolvedValue({ + ok: true, + data: { + suggestions: { title: "Brief", kind: "OTHER", documentDate: null }, + }, + }); + requireDocumentTextProvider.mockResolvedValue({ + entry: { + providerType: "local", + instance: { responseTimeoutSeconds: 90 }, + }, + providerType: "local", + }); + }); + + it("reads a summary after the re-check, with one call's deadline and its options", async () => { + claimAiRun.mockResolvedValue(summaryRun()); + requireDocumentVisionProvider.mockResolvedValue(visionPick(300)); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe( + "succeeded", + ); + expect(aiCapabilityForJob).toHaveBeenCalledWith("u1", "documentAi"); + expect(aiEgressRefusal).toHaveBeenCalledWith("documentAi", "u1", [ + "anthropic", + ]); + expect(updateMany).toHaveBeenCalledWith({ + where: { id: "run1", status: "RUNNING" }, + data: { expiresAt: new Date(NOW + 300_000 + 60_000) }, + }); + expect(executeDocumentSummary).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ mode: "vision", budget }), + output: "summary", + locale: "de", + persist: { replaceExisting: false }, + origin: { ipAddress: null, worker: true }, + }), + ); + expect(completeAiRun).toHaveBeenCalledWith( + "run1", + { summary: "Ein Arztbrief.", persistence: "stored" }, + new Date(NOW), + ); + expect(markQueuedSummaryUnavailable).not.toHaveBeenCalled(); + }); + + it("refuses a summary when consent was withdrawn while it waited, and marks it UNAVAILABLE", async () => { + claimAiRun.mockResolvedValue(summaryRun()); + aiEgressRefusal.mockResolvedValue( + new AiUnavailableError("documentAi", "consent_required"), + ); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed"); + expect(executeDocumentSummary).not.toHaveBeenCalled(); + expect(settleAiRunBudget).toHaveBeenCalledWith("u1", budget, 0, null); + expect(markQueuedSummaryUnavailable).toHaveBeenCalledWith("u1", "d1"); + }); + + it("a stored summary that fails is marked UNAVAILABLE", async () => { + claimAiRun.mockResolvedValue(summaryRun()); + executeDocumentSummary.mockResolvedValue({ + ok: false, + status: 502, + message: "Couldn't read the document. Try a clearer copy.", + errorCode: "documents.inbound.extractFailed", + }); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed"); + expect(markQueuedSummaryUnavailable).toHaveBeenCalledWith("u1", "d1"); + }); + + it("a stored summary that could not be written is marked UNAVAILABLE", async () => { + claimAiRun.mockResolvedValue(summaryRun()); + executeDocumentSummary.mockResolvedValue({ + ok: true, + data: { summary: "Ein Arztbrief.", persistence: "failed" }, + }); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe( + "succeeded", + ); + expect(markQueuedSummaryUnavailable).toHaveBeenCalledWith("u1", "d1"); + }); + + it("a transient summary that fails leaves the document's state alone", async () => { + claimAiRun.mockResolvedValue( + summaryRun({ + params: { + mode: "vision", + budget, + summary: { + output: "text", + persist: false, + replace: false, + locale: "en", + }, + }, + }), + ); + executeDocumentSummary.mockResolvedValue({ + ok: false, + status: 502, + message: "Couldn't read the document. Try a clearer copy.", + errorCode: "documents.inbound.extractFailed", + }); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed"); + expect(markQueuedSummaryUnavailable).not.toHaveBeenCalled(); + }); + + it("refuses a summary that cannot fit the job's expiry", async () => { + claimAiRun.mockResolvedValue(summaryRun()); + requireDocumentVisionProvider.mockResolvedValue(visionPick(600)); + // One call at ten minutes plus the margin does not fit in ten minutes. + expect(await runDocumentAiRun("run1", NOW + 10 * 60_000, now)).toBe( + "failed", + ); + expect(executeDocumentSummary).not.toHaveBeenCalled(); + expect(failAiRun).toHaveBeenCalledWith( + "run1", + expect.objectContaining({ errorCode: "aiRuns.timedOut", status: 504 }), + "RUNNING", + new Date(NOW), + ); + }); + + it("fails a summary whose document is gone, settling the reservation once", async () => { + claimAiRun.mockResolvedValue(summaryRun()); + loadOwnedDocument.mockResolvedValue(null); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed"); + expect(settleAiRunBudget).toHaveBeenCalledOnce(); + expect(failAiRun).toHaveBeenCalledWith( + "run1", + expect.objectContaining({ + status: 404, + errorCode: "documents.inbound.notFound", + }), + "RUNNING", + new Date(NOW), + ); + }); + + it("a text suggestion re-checks the wire for the text provider and reads the sealed text", async () => { + claimAiRun.mockResolvedValue( + claimed({ + kind: "DOCUMENT_SUGGEST", + params: { mode: "text", budget }, + input: Buffer.from("Arbeitsunfähigkeitsbescheinigung"), + }), + ); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe( + "succeeded", + ); + expect(aiEgressRefusal).toHaveBeenCalledWith("documentAi", "u1", ["local"]); + expect(updateMany).toHaveBeenCalledWith({ + where: { id: "run1", status: "RUNNING" }, + data: { expiresAt: new Date(NOW + 90_000 + 60_000) }, + }); + expect(executeDocumentSuggest).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + mode: "text", + text: "Arbeitsunfähigkeitsbescheinigung", + }), + }), + ); + }); + + it("refuses a suggestion when the operator turned reading off", async () => { + claimAiRun.mockResolvedValue( + claimed({ kind: "DOCUMENT_SUGGEST", params: { mode: "vision", budget } }), + ); + aiCapabilityForJob.mockResolvedValue({ + available: false, + reason: "operator_disabled", + }); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed"); + expect(executeDocumentSuggest).not.toHaveBeenCalled(); + expect(markQueuedSummaryUnavailable).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/jobs/document-ai-run.ts b/src/lib/jobs/document-ai-run.ts index 90ecd379c..c8244de7e 100644 --- a/src/lib/jobs/document-ai-run.ts +++ b/src/lib/jobs/document-ai-run.ts @@ -44,6 +44,15 @@ import { DOCUMENT_INDEX_MODEL_CALLS, executeDocumentIndex, } from "@/lib/documents/ai-runs/index-run"; +import { + DOCUMENT_SUGGEST_MODEL_CALLS, + executeDocumentSuggest, +} from "@/lib/documents/ai-runs/suggest-run"; +import { + DOCUMENT_SUMMARY_MODEL_CALLS, + executeDocumentSummary, + markQueuedSummaryUnavailable, +} from "@/lib/documents/ai-runs/summary-run"; import { claimAiRun, completeAiRun, @@ -55,11 +64,17 @@ import { import { AI_RUN_ERROR_CODES, AI_RUN_MARGIN_MS, + DOCUMENT_AI_RUN_CAPABILITY, + type AiRunBudget, type AiRunOutcome, type AiRunResult, type DocumentAiRunKindValue, } from "@/lib/documents/ai-runs/types"; -import { requireDocumentVisionProvider } from "@/lib/documents/provider-order"; +import { + requireDocumentTextProvider, + requireDocumentVisionProvider, +} from "@/lib/documents/provider-order"; +import { coerceLocale } from "@/lib/i18n/config"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { jobDeadline } from "@/lib/jobs/job-budget"; import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; @@ -88,10 +103,7 @@ export interface DocumentAiRunPayload { } /** The capability each kind of run answers to. */ -const RUN_CAPABILITY: Record = { - DOCUMENT_INDEX: "documentAi", - LABS_OCR_EXTRACT: "labsOcr", -}; +const RUN_CAPABILITY = DOCUMENT_AI_RUN_CAPABILITY; /** * Hand a created run to the worker. False when no queue is reachable; the @@ -262,6 +274,155 @@ async function runLabsOcr( }); } +/** The owned document a run reads, or the route's own 404 when it is gone. */ +async function loadRunDocument(run: ClaimedAiRun) { + const document = run.documentId + ? await loadOwnedDocument(run.userId, run.documentId) + : null; + if (document) return document; + // The reservation goes back in `runDocumentAiRun`'s catch. + throw new RunRefused({ + status: 404, + message: "Document not found", + errorCode: "documents.inbound.notFound", + }); +} + +/** The reservation a provider read was queued with. */ +function runBudget(run: ClaimedAiRun): AiRunBudget { + if (!run.params.budget) + throw new Error("document read without a reservation"); + return run.params.budget; +} + +/** + * Pick the provider for a document read the way its route does, re-check the + * capability and the wire for exactly that provider, and arm the deadline for + * the calls the read can make. + */ +async function pickForDocumentRead( + run: ClaimedAiRun, + modelCalls: number, + deadlineAt: number | undefined, + now: () => number, +) { + if (run.params.mode === "text") { + const pick = await requireDocumentTextProvider(run.userId); + await assertRunMayEgress("documentAi", run.userId, [pick.providerType]); + await armDeadline( + run.id, + callTimeoutMs({}, pick.entry.instance.responseTimeoutSeconds), + modelCalls, + deadlineAt, + now, + ); + return { mode: "text" as const, pick }; + } + const pick = await requireDocumentVisionProvider(run.userId); + await assertRunMayEgress("documentAi", run.userId, [pick.providerType]); + await armDeadline( + run.id, + callTimeoutMs({}, pick.entry.instance.responseTimeoutSeconds), + modelCalls, + deadlineAt, + now, + ); + return { mode: "vision" as const, pick }; +} + +/** The text the browser read, sealed into the run by the queuing route. */ +function runText(run: ClaimedAiRun): string { + if (!run.input) throw new Error("text read without an input"); + return run.input.toString("utf8"); +} + +async function runDocumentSummaryRead( + run: ClaimedAiRun, + deadlineAt: number | undefined, + now: () => number, +): Promise> { + const options = run.params.summary; + if (!options) throw new Error("summary run without its options"); + const document = await loadRunDocument(run); + const read = await pickForDocumentRead( + run, + DOCUMENT_SUMMARY_MODEL_CALLS, + deadlineAt, + now, + ); + const budget = runBudget(run); + return executeDocumentSummary({ + userId: run.userId, + document, + input: + read.mode === "text" + ? { mode: "text", text: runText(run), pick: read.pick, budget } + : { mode: "vision", pick: read.pick, budget }, + output: options.output, + locale: coerceLocale(options.locale), + persist: options.persist ? { replaceExisting: options.replace } : null, + origin: { ipAddress: null, worker: true }, + }); +} + +async function runDocumentSuggestRead( + run: ClaimedAiRun, + deadlineAt: number | undefined, + now: () => number, +): Promise> { + const document = await loadRunDocument(run); + const read = await pickForDocumentRead( + run, + DOCUMENT_SUGGEST_MODEL_CALLS, + deadlineAt, + now, + ); + const budget = runBudget(run); + return executeDocumentSuggest({ + userId: run.userId, + document, + input: + read.mode === "text" + ? { mode: "text", text: runText(run), pick: read.pick, budget } + : { mode: "vision", pick: read.pick, budget }, + origin: { ipAddress: null, worker: true }, + }); +} + +type RunDispatcher = ( + run: ClaimedAiRun, + deadlineAt: number | undefined, + now: () => number, +) => Promise>; + +/** + * The body each kind of run executes. Every dispatcher that sends anything to + * a model re-checks the capability and the wire for its provider first + * (`assertRunMayEgress`); `ai-egress-capability-guard.test.ts` holds that. + */ +const RUN_DISPATCH: Record = { + DOCUMENT_INDEX: runDocumentIndex, + LABS_OCR_EXTRACT: runLabsOcr, + DOCUMENT_SUMMARY: runDocumentSummaryRead, + DOCUMENT_SUGGEST: runDocumentSuggestRead, +}; + +/** + * What a failed run leaves behind besides itself. A summary that was to be + * stored had its document marked PENDING at enqueue; a failure says it could + * not be produced rather than leaving "being prepared" until the hourly + * reaper notices. + */ +async function afterRunFailed(run: ClaimedAiRun): Promise { + if ( + run.kind === "DOCUMENT_SUMMARY" && + run.params.summary?.persist && + run.documentId + ) { + await markQueuedSummaryUnavailable(run.userId, run.documentId); + } +} + export type DocumentAiRunResult = "succeeded" | "failed" | "skipped"; /** @@ -281,10 +442,7 @@ export async function runDocumentAiRun( let outcome: AiRunOutcome; try { - outcome = - run.kind === "DOCUMENT_INDEX" - ? await runDocumentIndex(run, deadlineAt, now) - : await runLabsOcr(run, deadlineAt, now); + outcome = await RUN_DISPATCH[run.kind](run, deadlineAt, now); } catch (err) { // Everything that throws here threw before the body took over the // reservation, so it goes back here. @@ -298,6 +456,7 @@ export async function runDocumentAiRun( message: "The background read failed.", }; await failAiRun(run.id, failure, "RUNNING", new Date(now())); + await afterRunFailed(run); annotate({ action: { name: "ai_runs.failed" }, meta: { @@ -311,6 +470,13 @@ export async function runDocumentAiRun( if (outcome.ok) { await completeAiRun(run.id, outcome.data, new Date(now())); + if ( + "persistence" in outcome.data && + outcome.data.persistence === "failed" + ) { + // The summary was read but could not be stored on the document. + await afterRunFailed(run); + } annotate({ action: { name: "ai_runs.succeeded" }, meta: { kind: run.kind }, @@ -327,6 +493,7 @@ export async function runDocumentAiRun( "RUNNING", new Date(now()), ); + await afterRunFailed(run); annotate({ action: { name: "ai_runs.failed" }, meta: { kind: run.kind, errorCode: outcome.errorCode }, diff --git a/src/lib/openapi/routes/ai-runs.ts b/src/lib/openapi/routes/ai-runs.ts index dabd0a709..e65d7c1e0 100644 --- a/src/lib/openapi/routes/ai-runs.ts +++ b/src/lib/openapi/routes/ai-runs.ts @@ -6,8 +6,9 @@ * * Part of the OpenAPI route table; aggregated in `./index.ts`. * - * iOS coordination (v1.40): `POST /api/documents/inbound/{id}/index` keeps - * its 200 body when the request carries no `Prefer` header. A client that + * iOS coordination (v1.40): `POST /api/documents/inbound/{id}/index`, + * `…/summary` and `…/suggest` keep their 200 bodies when the request carries + * no `Prefer` header. A client that * sends `Prefer: respond-async` gets 202 with a run id and polls the run here; * the result is the same body the route would have answered with. * `POST /api/labs/ocr/extract` (web only) always answers 202. @@ -15,24 +16,38 @@ import type { ZodOpenApiObject } from "zod-openapi"; import { z } from "zod/v4"; -import { documentIndexResponse } from "./documents"; +import { + documentIndexResponse, + documentSuggestResponse, + documentSummaryResponse, +} from "./documents"; import { ocrExtractResponse } from "./ocr"; import { dataEnvelope, errorEnvelope, stdResponses } from "./shared"; const aiRun = z .object({ id: z.string(), - kind: z.enum(["DOCUMENT_INDEX", "LABS_OCR_EXTRACT"]), + kind: z.enum([ + "DOCUMENT_INDEX", + "LABS_OCR_EXTRACT", + "DOCUMENT_SUMMARY", + "DOCUMENT_SUGGEST", + ]), documentId: z .string() .nullable() .describe("The document a document run reads; null for a lab scan."), status: z.enum(["QUEUED", "RUNNING", "SUCCEEDED", "FAILED"]), result: z - .union([documentIndexResponse, ocrExtractResponse]) + .union([ + documentIndexResponse, + ocrExtractResponse, + documentSummaryResponse, + documentSuggestResponse, + ]) .nullable() .describe( - "Present once `status` is `SUCCEEDED`: the body the queuing route answers with synchronously. `DOCUMENT_INDEX` carries `DocumentIndexResponse`, `LABS_OCR_EXTRACT` carries `OcrExtractResponse`.", + "Present once `status` is `SUCCEEDED`: the body the queuing route answers with synchronously. `DOCUMENT_INDEX` carries `DocumentIndexResponse`, `LABS_OCR_EXTRACT` carries `OcrExtractResponse`, `DOCUMENT_SUMMARY` carries `DocumentSummaryResponse`, `DOCUMENT_SUGGEST` carries `DocumentSuggestResponse`. Decode by `kind`; treat an unknown `kind` as a run this client did not start.", ), error: z .object({ diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index 9d684e770..bc63f1941 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -287,6 +287,26 @@ const documentUploadEnvelope = dataEnvelope( "DocumentUploadEnvelope", ); +/** The suggest route's body; also a `DOCUMENT_SUGGEST` run's result. */ +export const documentSuggestResponse = z + .object({ + suggestions: z.object({ + title: z.string().nullable(), + kind: kindEnum.nullable(), + documentDate: z.string().nullable(), + }), + }) + .meta({ id: "DocumentSuggestResponse" }); + +/** The summary route's body; also a `DOCUMENT_SUMMARY` run's result. */ +export const documentSummaryResponse = z + .object({ + summary: z.string().optional(), + text: z.string().optional(), + persistence: z.enum(["stored", "withheld", "failed"]).optional(), + }) + .meta({ id: "DocumentSummaryResponse" }); + /** The index route's body; also a `DOCUMENT_INDEX` run's result. */ export const documentIndexResponse = z .object({ @@ -954,9 +974,10 @@ export const inboundDocumentPaths: NonNullable = { tags: ["Documents"], summary: "Suggest filing metadata (drafts only)", description: - 'Optional AI assist on an already-stored document. Runs ONE provider call over the stored original (VISION, empty body) or browser-OCR\'d text (TEXT, `application/json` `{ mode: "text", text }`, opt-in local OCR) and returns a `{ title, kind, documentDate }` DRAFT for the edit form. Answers under the `documentAi` capability (403 with the capability envelope when it is closed), then rate / budget gated (shares the per-user document-AI bucket with extract/summary/index — default 6/hour, operator-tunable via `DOCUMENT_AI_LIMIT_PER_HOUR`; a slot is only consumed when the request reaches the provider, and the 429 carries `Retry-After`, the `X-RateLimit-Limit` / `-Remaining` / `-Reset` triple, and `meta.retryAt`). WRITES NOTHING — never stages facts, never flips status; the user reviews and saves. 422 (`documents.inbound.providerUnsupported`) with no provider configured. Never interprets or diagnoses; the title is a neutral filing label.', + "Optional AI assist on an already-stored document. Runs ONE provider call over the stored original (VISION, empty body) or browser-OCR'd text (TEXT, `application/json` `{ mode: \"text\", text }`, opt-in local OCR) and returns a `{ title, kind, documentDate }` DRAFT for the edit form. Answers under the `documentAi` capability (403 with the capability envelope when it is closed), then rate / budget gated (shares the per-user document-AI bucket with extract/summary/index — default 6/hour, operator-tunable via `DOCUMENT_AI_LIMIT_PER_HOUR`; a slot is only consumed when the request reaches the provider, and the 429 carries `Retry-After`, the `X-RateLimit-Limit` / `-Remaining` / `-Reset` triple, and `meta.retryAt`). WRITES NOTHING — never stages facts, never flips status; the user reviews and saves. 422 (`documents.inbound.providerUnsupported`) with no provider configured. Never interprets or diagnoses; the title is a neutral filing label. With `Prefer: respond-async` (v1.40) the read runs in the background: after the same quick refusals the route answers 202 with a run id, a second request while a suggestion read of this document over the same transport is queued or running answers with that run, and `GET /api/ai-runs/{id}` serves this body as the run's `result`.", parameters: [ { name: "id", in: "path", required: true, schema: { type: "string" } }, + preferRespondAsyncParameter, ], requestBody: { required: false, @@ -976,20 +997,13 @@ export const inboundDocumentPaths: NonNullable = { content: { "application/json": { schema: dataEnvelope( - z - .object({ - suggestions: z.object({ - title: z.string().nullable(), - kind: kindEnum.nullable(), - documentDate: z.string().nullable(), - }), - }) - .meta({ id: "DocumentSuggestResponse" }), + documentSuggestResponse, "DocumentSuggestEnvelope", ), }, }, }, + ...aiRunAccepted, ...aiExtractionRefusals("documentAi"), ...stdResponses, }, @@ -1000,7 +1014,7 @@ export const inboundDocumentPaths: NonNullable = { tags: ["Documents"], summary: "Summarise or transcribe a document", description: - 'On-demand description of a stored document. `?mode=summary` (default) returns a short plain-language summary of WHAT the document is; it remains transient unless `persist=true` fills the empty stored-summary slot, while `persist=true&replace=true` explicitly replaces an existing summary. `?mode=text` returns transient raw transcribed text. Neither result reaches coach memory, snapshots, structured stores, or the search index. The summary is descriptive only and never a diagnosis. Same VISION (empty body) / TEXT (`application/json` `{ mode: "text", text }`, opt-in local OCR) dispatch as extract. Answers under the `documentAi` capability (403 with the capability envelope when it is closed), then rate / budget gated. 422 `documents.inbound.providerUnsupported` with no provider.', + "On-demand description of a stored document. `?mode=summary` (default) returns a short plain-language summary of WHAT the document is; it remains transient unless `persist=true` fills the empty stored-summary slot, while `persist=true&replace=true` explicitly replaces an existing summary. `?mode=text` returns transient raw transcribed text. Neither result reaches coach memory, snapshots, structured stores, or the search index. The summary is descriptive only and never a diagnosis. Same VISION (empty body) / TEXT (`application/json` `{ mode: \"text\", text }`, opt-in local OCR) dispatch as extract. Answers under the `documentAi` capability (403 with the capability envelope when it is closed), then rate / budget gated. 422 `documents.inbound.providerUnsupported` with no provider. With `Prefer: respond-async` (v1.40) the read runs in the background: after the same quick refusals the route answers 202 with a run id, and `GET /api/ai-runs/{id}` serves this body as the run's `result`. A second request while a read with the same `mode`, `persist` and `replace` is queued or running answers with that run. With `persist=true` the document's `summaryState` is PENDING while the run is queued or running (unless a summary is already stored) and becomes UNAVAILABLE when the run fails. `mode=text` over posted text calls no model and is answered synchronously either way (no `Preference-Applied`).", parameters: [ { name: "id", in: "path", required: true, schema: { type: "string" } }, { @@ -1031,6 +1045,7 @@ export const inboundDocumentPaths: NonNullable = { description: "With `persist=true`, explicitly replace an existing stored summary.", }, + preferRespondAsyncParameter, ], requestBody: { required: false, @@ -1051,20 +1066,13 @@ export const inboundDocumentPaths: NonNullable = { content: { "application/json": { schema: dataEnvelope( - z - .object({ - summary: z.string().optional(), - text: z.string().optional(), - persistence: z - .enum(["stored", "withheld", "failed"]) - .optional(), - }) - .meta({ id: "DocumentSummaryResponse" }), + documentSummaryResponse, "DocumentSummaryEnvelope", ), }, }, }, + ...aiRunAccepted, ...aiExtractionRefusals("documentAi"), ...stdResponses, }, diff --git a/tests/integration/document-ai-runs.test.ts b/tests/integration/document-ai-runs.test.ts index c21f5de6d..ee54eb1e3 100644 --- a/tests/integration/document-ai-runs.test.ts +++ b/tests/integration/document-ai-runs.test.ts @@ -16,6 +16,10 @@ * anything reaches a model. * 6. The reaper fails a run no worker took (and hands its reservation back) * and deletes a finished run an hour after it ended. + * 7. Summary and suggestion runs (0370) round-trip the routes' own bodies; a + * summary that is to be stored marks the document PENDING while queued, + * READY when it lands and UNAVAILABLE when the run fails; a transient one + * leaves the state alone and never attaches to a stored one. */ import { Buffer } from "node:buffer"; @@ -68,6 +72,18 @@ vi.mock("@/lib/documents/describe", async (importOriginal) => ({ transcribeDocument: vi.fn(async () => ({ text: "Entlassbrief Leukozyten erhoeht", })), + runDocumentSummary: vi.fn(async () => ({ + summary: "A hospital discharge letter.", + blocked: null, + })), +})); +vi.mock("@/lib/documents/assist", async (importOriginal) => ({ + ...(await importOriginal()), + runDocumentAssist: vi.fn(async () => ({ + title: "Discharge letter", + kind: "DISCHARGE_LETTER", + documentDate: "2027-01-10", + })), })); vi.mock("@/lib/labs/ocr-extract", async (importOriginal) => ({ ...(await importOriginal()), @@ -84,7 +100,11 @@ vi.mock("@/lib/documents/auto-stage-labs", () => ({ })), })); -import { transcribeDocument } from "@/lib/documents/describe"; +import { runDocumentAssist } from "@/lib/documents/assist"; +import { + runDocumentSummary, + transcribeDocument, +} from "@/lib/documents/describe"; import { runOcrExtraction } from "@/lib/labs/ocr-extract"; const PNG = Buffer.from( @@ -186,6 +206,40 @@ async function postLabScanText(text: string) { return { status: res.status, body: (await res.json()) as Envelope }; } +async function postDocumentAi( + documentId: string, + path: string, + prefer: boolean, +) { + const route = path.startsWith("summary") + ? await import("@/app/api/documents/inbound/[id]/summary/route") + : await import("@/app/api/documents/inbound/[id]/suggest/route"); + const { NextRequest } = await import("next/server"); + const res = await route.POST( + new NextRequest( + `http://localhost/api/documents/inbound/${documentId}/${path}`, + { + method: "POST", + headers: prefer ? { prefer: "respond-async" } : {}, + }, + ) as never, + { params: Promise.resolve({ id: documentId }) } as never, + ); + return { + status: res.status, + headers: res.headers, + body: (await res.json()) as Envelope, + }; +} + +async function summaryState(documentId: string) { + const row = await getPrismaClient().inboundDocument.findUniqueOrThrow({ + where: { id: documentId }, + select: { summaryState: true, summaryEncrypted: true }, + }); + return { state: row.summaryState, stored: row.summaryEncrypted !== null }; +} + async function poll(runId: string) { const { GET } = await import("@/app/api/ai-runs/[id]/route"); const res = await GET( @@ -211,6 +265,8 @@ beforeEach(async () => { send.mockClear(); vi.mocked(transcribeDocument).mockClear(); vi.mocked(runOcrExtraction).mockClear(); + vi.mocked(runDocumentSummary).mockClear(); + vi.mocked(runDocumentAssist).mockClear(); }); describe("Read with AI in the background", () => { @@ -449,3 +505,178 @@ describe("the reaper", () => { ).toBeNull(); }); }); + +describe("a summary in the background", () => { + it("marks a stored summary PENDING while queued and READY when it lands", async () => { + const user = await makeUser(); + await grantConsent(user.id); + const document = await storeDocument(user.id); + await signIn(user.id); + + const queued = await postDocumentAi( + document.id, + "summary?mode=summary&persist=true", + true, + ); + expect(queued.status).toBe(202); + expect(queued.headers.get("preference-applied")).toBe("respond-async"); + const { runId } = queued.body.data as { runId: string }; + expect(runDocumentSummary).not.toHaveBeenCalled(); + expect(await summaryState(document.id)).toEqual({ + state: "PENDING", + stored: false, + }); + + // A second press for the same stored summary attaches; a transient + // summary of the same document is its own run. + const again = await postDocumentAi( + document.id, + "summary?mode=summary&persist=true", + true, + ); + expect((again.body.data as { runId: string }).runId).toBe(runId); + const transient = await postDocumentAi( + document.id, + "summary?mode=summary", + true, + ); + expect(transient.status).toBe(202); + expect((transient.body.data as { runId: string }).runId).not.toBe(runId); + + expect(await work(runId)).toBe("succeeded"); + expect(runDocumentSummary).toHaveBeenCalledOnce(); + const done = await poll(runId); + expect(done.body.data).toMatchObject({ + status: "SUCCEEDED", + kind: "DOCUMENT_SUMMARY", + documentId: document.id, + result: { + summary: "A hospital discharge letter.", + persistence: "stored", + }, + }); + expect(await summaryState(document.id)).toEqual({ + state: "READY", + stored: true, + }); + }); + + it("marks a stored summary UNAVAILABLE when its run fails", async () => { + const user = await makeUser(); + await grantConsent(user.id); + const document = await storeDocument(user.id); + await signIn(user.id); + vi.mocked(runDocumentSummary).mockRejectedValueOnce( + new Error("provider down"), + ); + + const queued = await postDocumentAi( + document.id, + "summary?mode=summary&persist=true", + true, + ); + const { runId } = queued.body.data as { runId: string }; + expect(await work(runId)).toBe("failed"); + const done = await poll(runId); + expect(done.body.data).toMatchObject({ + status: "FAILED", + error: { code: "documents.inbound.extractFailed", status: 502 }, + }); + expect(await summaryState(document.id)).toEqual({ + state: "UNAVAILABLE", + stored: false, + }); + }); + + it("leaves the document's summary state alone for a transient summary", async () => { + const user = await makeUser(); + await grantConsent(user.id); + const document = await storeDocument(user.id); + await signIn(user.id); + vi.mocked(runDocumentSummary).mockRejectedValueOnce( + new Error("provider down"), + ); + + const queued = await postDocumentAi( + document.id, + "summary?mode=summary", + true, + ); + const { runId } = queued.body.data as { runId: string }; + expect(await summaryState(document.id)).toEqual({ + state: "NONE", + stored: false, + }); + expect(await work(runId)).toBe("failed"); + expect(await summaryState(document.id)).toEqual({ + state: "NONE", + stored: false, + }); + }); + + it("answers synchronously, exactly as before, without the header", async () => { + const user = await makeUser(); + await grantConsent(user.id); + const document = await storeDocument(user.id); + await signIn(user.id); + + const res = await postDocumentAi(document.id, "summary?mode=text", false); + expect(res.status).toBe(200); + expect(res.headers.get("preference-applied")).toBeNull(); + expect(res.body.data).toEqual({ text: "Entlassbrief Leukozyten erhoeht" }); + expect(send).not.toHaveBeenCalled(); + expect(await getPrismaClient().documentAiRun.count()).toBe(0); + }); +}); + +describe("filing suggestions in the background", () => { + it("queues, reads in the worker, writes nothing to the document", async () => { + const user = await makeUser(); + await grantConsent(user.id); + const document = await storeDocument(user.id); + await signIn(user.id); + + const queued = await postDocumentAi(document.id, "suggest", true); + expect(queued.status).toBe(202); + const { runId } = queued.body.data as { runId: string }; + expect(await work(runId)).toBe("succeeded"); + expect(runDocumentAssist).toHaveBeenCalledOnce(); + + const done = await poll(runId); + expect(done.body.data).toMatchObject({ + status: "SUCCEEDED", + kind: "DOCUMENT_SUGGEST", + result: { + suggestions: { + title: "Discharge letter", + kind: "DISCHARGE_LETTER", + documentDate: "2027-01-10", + }, + }, + }); + const after = await getPrismaClient().inboundDocument.findUniqueOrThrow({ + where: { id: document.id }, + }); + expect(after.kind).toBe("OTHER"); + expect(after.title).toBeNull(); + }); + + it("is a 404 for another account's document and its run", async () => { + const owner = await makeUser(); + await grantConsent(owner.id); + const document = await storeDocument(owner.id); + await signIn(owner.id); + const queued = await postDocumentAi(document.id, "suggest", true); + const { runId } = queued.body.data as { runId: string }; + + const other = await makeUser(); + await grantConsent(other.id); + await signIn(other.id); + const foreignPost = await postDocumentAi(document.id, "suggest", true); + expect(foreignPost.status).toBe(404); + expect(foreignPost.body.meta?.errorCode).toBe("documents.inbound.notFound"); + const foreignPoll = await poll(runId); + expect(foreignPoll.status).toBe(404); + expect(foreignPoll.body.meta?.errorCode).toBe("aiRuns.notFound"); + }); +}); From d6571c4e9c4e7db8a035b33f520d29936fcf8d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 3 Oct 2026 21:13:45 +0200 Subject: [PATCH 04/14] feat(symptoms): add person-defined symptoms and timestamped occurrences A symptom of one's own ("aura", "headache") is a definition with an encrypted name and an icon from the cycle custom-symptom allowlist; each occurrence carries a time, a 0-10 intensity (the pain score's scale), an optional encrypted note and an optional link to one of the record's open illness episodes. Both ride the illness module: every route answers illness.disabled when it is off. - Migration 0369 (idempotent): symptom_definitions and symptom_events, intensity CHECK 0-10, episode link ON DELETE SET NULL. - /api/symptoms/definitions{,/[id]} and /api/symptoms/events{,/[id]}: eight active definitions per record (a hidden one frees its slot, showing it again re-checks), hide or purge, events create/list/edit/ delete, a new link must name an open episode of the same record (422 symptoms.episode.notOpen). Error group symptoms.*. - Sharing: reads at READ, logging an occurrence at WRITE, defining and editing at MANAGE; C1/C3/C4 audit details; frozen lists, the handler matrix and the activity verbs updated. - Both encrypted columns registered and rotated; both models in the wipe plan; both carried at both backup ends with occurrences nested in their definition, plaintext in a portable file and ciphertext in a recovery file, and a link to an episode the file lacks dropped and named (symptomEpisodeReference). - OpenAPI regenerated. --- docs/api/openapi.yaml | 746 +++++++++++++++++- .../migration.sql | 73 ++ prisma/schema.prisma | 66 ++ scripts/rotate-encryption-key.ts | 15 + src/__tests__/sharing-surface-guard.test.ts | 42 +- src/__tests__/v137-wave-zero-fixtures.test.ts | 2 +- .../api/symptoms/definitions/[id]/route.ts | 232 ++++++ src/app/api/symptoms/definitions/route.ts | 139 ++++ src/app/api/symptoms/events/[id]/route.ts | 192 +++++ src/app/api/symptoms/events/route.ts | 166 ++++ src/components/admin/backups-section.tsx | 3 + src/lib/crypto/encrypted-columns.ts | 3 + src/lib/data-wipe/wipe-plan.ts | 3 + .../__tests__/full-backup-payload.test.ts | 1 + .../restore-backup-round-trip.test.ts | 1 + src/lib/export/backup-plan.ts | 11 + src/lib/export/full-backup-payload.ts | 18 +- src/lib/export/restore-backup.ts | 15 + src/lib/export/restore-skips.ts | 9 +- src/lib/export/symptoms-backup.ts | 289 +++++++ src/lib/openapi/error-codes.ts | 7 + src/lib/openapi/routes/index.ts | 4 + src/lib/openapi/routes/symptoms.ts | 371 +++++++++ src/lib/record-activity/activity-verb.ts | 12 + src/lib/symptoms/server.ts | 154 ++++ src/lib/symptoms/shared.ts | 154 ++++ src/lib/validations/backup.ts | 40 + tests/fixtures/v137/sharing-matrix.ts | 48 ++ tests/integration/backup-round-trip.test.ts | 112 +++ .../sharing-manage-handler-matrix.test.ts | 179 ++++- 30 files changed, 3091 insertions(+), 16 deletions(-) create mode 100644 prisma/migrations/0369_symptom_definitions_and_events/migration.sql create mode 100644 src/app/api/symptoms/definitions/[id]/route.ts create mode 100644 src/app/api/symptoms/definitions/route.ts create mode 100644 src/app/api/symptoms/events/[id]/route.ts create mode 100644 src/app/api/symptoms/events/route.ts create mode 100644 src/lib/export/symptoms-backup.ts create mode 100644 src/lib/openapi/routes/symptoms.ts create mode 100644 src/lib/symptoms/server.ts create mode 100644 src/lib/symptoms/shared.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index cf0c25eb9..3f58070c3 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -21908,6 +21908,276 @@ paths: schema: $ref: "#/components/schemas/ErrorEnvelope" "502": *a59 + /api/symptoms/definitions: + get: + tags: + - Symptoms + summary: List your own symptoms (v1.40) + description: The record's active symptom definitions with their 30-day summary; `includeHidden=true` adds the hidden + ones. Rides the illness module. + parameters: + - in: query + name: includeHidden + schema: + type: string + enum: + - "true" + - "false" + responses: + "200": + description: The definitions. + content: + application/json: + schema: + $ref: "#/components/schemas/ListSymptomDefinitionsEnvelope" + "401": *a1 + "403": &a61 + description: >- + `illness.disabled`: the record has the illness module switched off (or the operator turned it off + server-wide); symptoms ride that module. + + + Refused: this request named a record the caller may not act on (`meta.errorCode` = `sharing.access.denied`). + Byte-identical for an account that does not exist, an account that granted nothing, a grant whose sections + do not reach this surface, and a read grant on a request that writes — the response carries no reason, so it + is not an account-enumeration oracle. The reason reaches the record owner's activity feed and the operator's + audit trail instead. On the cookie transport the same code answers one further case: a session that has + entered a shared record and sends a request without its record-context assertion, which is a client older + than the fence — leave the record and reload. A request that names no record but the caller's own never + reaches this response, so a client that never switches and never sends the per-request account selector will + not see it. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a4 + "429": *a5 + post: + tags: + - Symptoms + summary: Define a symptom (v1.40) + description: Creates one definition. Idempotent under `Idempotency-Key`. Audits as `symptoms.definition.create` without + the label. Rate-limited to 30 a minute. + parameters: + - *a6 + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateSymptomDefinitionRequest" + responses: + "201": + description: Definition created. + content: + application/json: + schema: + $ref: "#/components/schemas/CreateSymptomDefinitionEnvelope" + "401": *a1 + "403": *a61 + "409": *a7 + "422": *a4 + "429": *a5 + /api/symptoms/definitions/{id}: + patch: + tags: + - Symptoms + summary: Edit, hide or show a symptom (v1.40) + description: Partial edit. Audits as `symptoms.definition.update`, naming a label change without quoting it. + parameters: + - in: path + name: id + schema: + type: string + required: true + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateSymptomDefinitionRequest" + responses: + "200": + description: Definition updated. + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateSymptomDefinitionEnvelope" + "401": *a1 + "403": *a61 + "404": &a62 + description: "`symptoms.definition.notFound`: no definition with this id on the record." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a4 + "429": *a5 + delete: + tags: + - Symptoms + summary: Hide or delete a symptom (v1.40) + description: Hides the definition and keeps its history. `?purge=true` deletes it and every occurrence of it. + parameters: + - in: path + name: id + schema: + type: string + required: true + - in: query + name: purge + schema: + type: string + enum: + - "true" + - "false" + responses: + "200": + description: Hidden or deleted. + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteSymptomDefinitionEnvelope" + "401": *a1 + "403": *a61 + "404": *a62 + "422": *a4 + "429": *a5 + /api/symptoms/events: + get: + tags: + - Symptoms + summary: List symptom occurrences (v1.40) + description: Occurrences of the record's symptoms, newest first, narrowed by the optional query. + parameters: + - in: query + name: from + schema: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + - in: query + name: to + schema: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + - in: query + name: definitionId + schema: + type: string + minLength: 1 + maxLength: 40 + - in: query + name: episodeId + schema: + type: string + minLength: 1 + maxLength: 40 + - in: query + name: limit + schema: + default: 200 + type: integer + minimum: 1 + maximum: 500 + responses: + "200": + description: A page of occurrences. + content: + application/json: + schema: + $ref: "#/components/schemas/ListSymptomEventsEnvelope" + "401": *a1 + "403": *a61 + "422": *a4 + "429": *a5 + post: + tags: + - Symptoms + summary: Log a symptom occurrence (v1.40) + description: Creates one occurrence. Idempotent under `Idempotency-Key`. A hidden definition can still be logged + against; another record's definition 404s (`symptoms.definition.notFound`). Audits as `symptoms.event.create` + without the note. + parameters: + - *a6 + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateSymptomEventRequest" + responses: + "201": + description: Occurrence logged. + content: + application/json: + schema: + $ref: "#/components/schemas/CreateSymptomEventEnvelope" + "401": *a1 + "403": *a61 + "404": *a62 + "409": *a7 + "422": *a4 + "429": *a5 + /api/symptoms/events/{id}: + patch: + tags: + - Symptoms + summary: Edit a symptom occurrence (v1.40) + description: Partial edit. Audits as `symptoms.event.update`, naming a note change without quoting it. + parameters: + - in: path + name: id + schema: + type: string + required: true + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateSymptomEventRequest" + responses: + "200": + description: Occurrence updated. + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateSymptomEventEnvelope" + "401": *a1 + "403": *a61 + "404": &a63 + description: "`symptoms.event.notFound`: no occurrence with this id on the record." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a4 + "429": *a5 + delete: + tags: + - Symptoms + summary: Delete a symptom occurrence (v1.40) + description: Removes the occurrence. Audits as `symptoms.event.delete`. + parameters: + - in: path + name: id + schema: + type: string + required: true + responses: + "200": + description: Deleted. + content: + application/json: + schema: + $ref: "#/components/schemas/DeleteSymptomEventEnvelope" + "401": *a1 + "403": *a61 + "404": *a63 + "422": *a4 + "429": *a5 /api/bugreport: post: security: [] @@ -31652,6 +31922,97 @@ components: leaves the singleton instance settings — registration, MFA requirement, default locale and timezone, module availability, notification and AI configuration, document cap and quota — exactly as they are. Send `true` only when rebuilding a host from a snapshot, because those settings apply to every account on the installation." + CreateSymptomDefinitionRequest: + type: object + properties: + label: + type: string + minLength: 1 + maxLength: 40 + icon: + anyOf: + - type: string + - type: "null" + required: + - label + description: Define a symptom of your own. `label` (1-40 characters) is encrypted at rest; `icon` is one of the cycle + custom-symptom icon names, null for the default glyph. At most eight active definitions per account (422 + `symptoms.definition.limitReached`). + UpdateSymptomDefinitionRequest: + type: object + properties: + label: + type: string + minLength: 1 + maxLength: 40 + icon: + anyOf: + - type: string + - type: "null" + isActive: + type: boolean + sortOrder: + type: integer + minimum: 0 + maximum: 1000 + description: "Partial edit; an omitted key leaves the column untouched. `isActive: false` hides the symptom (history + kept, slot freed); `isActive: true` shows it again and re-checks the cap." + CreateSymptomEventRequest: + type: object + properties: + definitionId: + type: string + minLength: 1 + maxLength: 40 + occurredAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + intensity: + type: integer + minimum: 0 + maximum: 10 + note: + anyOf: + - type: string + maxLength: 500 + - type: "null" + episodeId: + anyOf: + - type: string + minLength: 1 + maxLength: 40 + - type: "null" + required: + - definitionId + - intensity + description: Log one occurrence. `intensity` is 0-10 (0 none, 10 the worst imaginable). `occurredAt` defaults to now and + may not lie in the future beyond five minutes. `episodeId`, when given, must name one of the record's own + illness episodes that is still open (422 `symptoms.episode.notOpen` otherwise). The note is encrypted at rest. + UpdateSymptomEventRequest: + type: object + properties: + occurredAt: + type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + intensity: + type: integer + minimum: 0 + maximum: 10 + note: + anyOf: + - type: string + maxLength: 500 + - type: "null" + episodeId: + anyOf: + - type: string + minLength: 1 + maxLength: 40 + - type: "null" + description: "Partial edit of one occurrence. `note: null` clears the note, `episodeId: null` drops the link. A changed + `episodeId` must name an open episode; an unchanged link to an episode that has since resolved is kept." MedicationTreatmentClass: type: string enum: @@ -31953,14 +32314,15 @@ components: sharing.renounce.already_ended, sharing.renounce.not_found, sharing.revoke.already_ended, sharing.revoke.not_found, sharing.session.changed, sharing.switch.invalid, sharing.switch.wrong_transport; source-priority — source-priority.body.invalid_json, - source-priority.body.invalid_shape; tokens — tokens.documents.ceiling_reached, - tokens.measurements.ceiling_reached, tokens.workouts.ceiling_reached; vaccination — - vaccination.booster-invalid, vaccination.booster-no-antigen, vaccination.booster-out-of-scope, - vaccination.encounter-not-found, vaccination.identity-required, vaccination.invalid, - vaccination.link.invalid, vaccination.not-found, vaccination.practitioner-not-found; workout — - workout.batch.invalid, workout.batch.payload_too_large, workout.batch.source_not_permitted, - workout.batch.too_large, workout.delete.synced_source; (unprefixed) — about_me_conflict, - ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file, + source-priority.body.invalid_shape; symptoms — symptoms.definition.limitReached, + symptoms.definition.notFound, symptoms.episode.notOpen, symptoms.event.notFound, symptoms.invalid; + tokens — tokens.documents.ceiling_reached, tokens.measurements.ceiling_reached, + tokens.workouts.ceiling_reached; vaccination — vaccination.booster-invalid, + vaccination.booster-no-antigen, vaccination.booster-out-of-scope, vaccination.encounter-not-found, + vaccination.identity-required, vaccination.invalid, vaccination.link.invalid, vaccination.not-found, + vaccination.practitioner-not-found; workout — workout.batch.invalid, workout.batch.payload_too_large, + workout.batch.source_not_permitted, workout.batch.too_large, workout.delete.synced_source; (unprefixed) + — about_me_conflict, ai_response_truncated, coach_prefs_conflict, dashboard_layout_conflict, empty_file, health_score_config_conflict, insights_layout_conflict, invalid_base_updated_at, json_carries_no_intake_time, json_not_an_array, medication_layout_conflict, missing_required_columns, modules_conflict, mood_tag_layout_conflict, not_configured, notification_prefs_conflict, oidc_only, @@ -36314,6 +36676,86 @@ components: - onsetAt - dayLogs additionalProperties: {} + symptomDefinitions: + default: [] + type: array + items: + type: object + properties: + id: + type: string + minLength: 1 + label: + type: string + minLength: 1 + labelEncrypted: + type: string + minLength: 1 + icon: + anyOf: + - type: string + - type: "null" + sortOrder: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + isActive: + type: boolean + createdAt: + type: string + minLength: 1 + updatedAt: + type: string + minLength: 1 + events: + default: [] + type: array + items: + type: object + properties: + id: + type: string + minLength: 1 + occurredAt: + type: string + minLength: 1 + intensity: + type: integer + minimum: 0 + maximum: 10 + note: + anyOf: + - type: string + - type: "null" + noteEncrypted: + anyOf: + - type: string + minLength: 1 + - type: "null" + episodeId: + anyOf: + - type: string + minLength: 1 + - type: "null" + createdAt: + type: string + minLength: 1 + updatedAt: + type: string + minLength: 1 + required: + - occurredAt + - intensity + - createdAt + - updatedAt + additionalProperties: {} + required: + - sortOrder + - isActive + - createdAt + - updatedAt + - events + additionalProperties: {} allergies: default: [] type: array @@ -38828,6 +39270,7 @@ components: - labResults - biomarkers - illnessEpisodes + - symptomDefinitions - allergies - familyHistory - workouts @@ -75632,6 +76075,293 @@ components: - data - error additionalProperties: false + ListSymptomDefinitionsEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/SymptomDefinitionList" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + SymptomDefinitionList: + type: object + properties: + definitions: + type: array + items: + $ref: "#/components/schemas/SymptomDefinition" + limit: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + required: + - definitions + - limit + additionalProperties: false + description: The record's symptom definitions in display order, and the active-definition cap. + SymptomDefinition: + type: object + properties: + id: + type: string + label: + anyOf: + - type: string + - type: "null" + icon: + anyOf: + - type: string + - type: "null" + sortOrder: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + isActive: + type: boolean + recent: + type: object + properties: + count30d: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + maxIntensity30d: + anyOf: + - type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + - type: "null" + lastOccurredAt: + anyOf: + - type: string + - type: "null" + required: + - count30d + - maxIntensity30d + - lastOccurredAt + additionalProperties: false + required: + - id + - label + - icon + - sortOrder + - isActive + - recent + additionalProperties: false + description: "A symptom the person defined. `label` is decrypted (null only on an unreadable value). `recent` is + computed server-side: occurrences and the highest intensity over the last 30 days, and the last occurrence ever. + Each active definition is also the correlation channel `SYMPTOM:`." + CreateSymptomDefinitionEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/SymptomDefinition" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + UpdateSymptomDefinitionEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/SymptomDefinition" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DeleteSymptomDefinitionEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/SymptomDefinitionDeleted" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + SymptomDefinitionDeleted: + type: object + properties: + deleted: + type: boolean + const: true + purged: + type: boolean + eventsDeleted: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + required: + - deleted + - purged + - eventsDeleted + additionalProperties: false + description: "`purged: false` means the definition was hidden and its history kept; `purged: true` means it and + `eventsDeleted` occurrences were removed." + ListSymptomEventsEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/SymptomEventList" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + SymptomEventList: + type: object + properties: + events: + type: array + items: + $ref: "#/components/schemas/SymptomEvent" + limit: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + hasMore: + type: boolean + required: + - events + - limit + - hasMore + additionalProperties: false + description: A page of occurrences, newest first. `hasMore` is true when the page is full; page on with `to` set to the + oldest `occurredAt` seen. + SymptomEvent: + type: object + properties: + id: + type: string + definitionId: + type: string + occurredAt: + type: string + intensity: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + note: + anyOf: + - type: string + - type: "null" + episodeId: + anyOf: + - type: string + - type: "null" + createdAt: + type: string + updatedAt: + type: string + required: + - id + - definitionId + - occurredAt + - intensity + - note + - episodeId + - createdAt + - updatedAt + additionalProperties: false + description: "One occurrence of a defined symptom: when, how strong (0-10), the decrypted note, and the illness episode + it was filed against (null when none)." + CreateSymptomEventEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/SymptomEvent" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + UpdateSymptomEventEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/SymptomEvent" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + DeleteSymptomEventEnvelope: + type: object + properties: + data: + type: object + properties: + deleted: + type: boolean + const: true + required: + - deleted + additionalProperties: false + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false __schema0: anyOf: - type: string diff --git a/prisma/migrations/0369_symptom_definitions_and_events/migration.sql b/prisma/migrations/0369_symptom_definitions_and_events/migration.sql new file mode 100644 index 000000000..a6dc2d55a --- /dev/null +++ b/prisma/migrations/0369_symptom_definitions_and_events/migration.sql @@ -0,0 +1,73 @@ +-- Person-defined symptoms and their timestamped events. A definition is the +-- person's own name for a symptom (encrypted); an event is one occurrence with +-- a 0-10 intensity and an optional link to an illness episode. Idempotent, so +-- a re-run of a half-applied migration converges. +CREATE TABLE IF NOT EXISTS "symptom_definitions" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "label_encrypted" BYTEA NOT NULL, + "icon" TEXT, + "sort_order" INTEGER NOT NULL DEFAULT 0, + "is_active" BOOLEAN NOT NULL DEFAULT true, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + CONSTRAINT "symptom_definitions_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "symptom_definitions_user_id_sort_order_idx" + ON "symptom_definitions" ("user_id", "sort_order"); + +DO $$ BEGIN + ALTER TABLE "symptom_definitions" + ADD CONSTRAINT "symptom_definitions_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +CREATE TABLE IF NOT EXISTS "symptom_events" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "definition_id" TEXT NOT NULL, + "occurred_at" TIMESTAMP(3) NOT NULL, + "intensity" INTEGER NOT NULL, + "note_encrypted" BYTEA, + "episode_id" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + CONSTRAINT "symptom_events_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "symptom_events_user_id_occurred_at_idx" + ON "symptom_events" ("user_id", "occurred_at"); +CREATE INDEX IF NOT EXISTS "symptom_events_definition_id_occurred_at_idx" + ON "symptom_events" ("definition_id", "occurred_at"); +CREATE INDEX IF NOT EXISTS "symptom_events_episode_id_idx" + ON "symptom_events" ("episode_id"); + +DO $$ BEGIN + ALTER TABLE "symptom_events" + ADD CONSTRAINT "symptom_events_intensity_check" + CHECK ("intensity" >= 0 AND "intensity" <= 10); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +DO $$ BEGIN + ALTER TABLE "symptom_events" + ADD CONSTRAINT "symptom_events_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +DO $$ BEGIN + ALTER TABLE "symptom_events" + ADD CONSTRAINT "symptom_events_definition_id_fkey" + FOREIGN KEY ("definition_id") REFERENCES "symptom_definitions"("id") ON DELETE CASCADE ON UPDATE CASCADE; +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +DO $$ BEGIN + ALTER TABLE "symptom_events" + ADD CONSTRAINT "symptom_events_episode_id_fkey" + FOREIGN KEY ("episode_id") REFERENCES "illness_episodes"("id") ON DELETE SET NULL ON UPDATE CASCADE; +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 36dbabbce..b88d38179 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -911,6 +911,8 @@ model User { // per-day timeline mirroring the CycleDayLog shape. illnessEpisodes IllnessEpisode[] illnessDayLogs IllnessDayLog[] + symptomDefinitions SymptomDefinition[] + symptomEvents SymptomEvent[] // v1.25 (W-RECORDS) — structured health-record entries. Allergies map to // FHIR AllergyIntolerance; family-history entries map to FHIR // FamilyMemberHistory. Reference records (not time-series signals); the @@ -7726,6 +7728,8 @@ model IllnessEpisode { dayLogs IllnessDayLog[] documentLinks DocumentConditionLink[] encounterLinks EncounterConditionLink[] + /// Symptom events the person filed against this episode while it was open. + symptomEvents SymptomEvent[] @@index([userId, deletedAt]) @@index([userId, onsetAt]) @@ -7814,6 +7818,68 @@ model IllnessSymptomLink { @@map("illness_symptom_links") } +/// A symptom the person defined for themselves ("aura", "headache"), v1.40. +/// +/// Separate from the seeded `IllnessSymptom` catalogue on the illness day log: +/// that one is a fixed list of cold symptoms rated 1-3 per episode day, this one +/// is the person's own vocabulary, logged as timestamped events with a 0-10 +/// intensity whether or not an episode is open. Rides the `illness` module. +/// At most eight active definitions per account +/// (`MAX_ACTIVE_SYMPTOM_DEFINITIONS`), and each active one is its own +/// correlation channel `SYMPTOM:`, so the cap also bounds the matrix. +model SymptomDefinition { + id String @id @default(cuid()) + userId String @map("user_id") + /// The person's own name for the symptom, AES-256-GCM encrypted at rest. A + /// symptom name is health data in the person's own words. + labelEncrypted Bytes @map("label_encrypted") + /// Lucide icon name from `SYMPTOM_ICON_ALLOWLIST` (the cycle custom-symptom + /// list, which the iOS client already maps). NULL = the default glyph. + icon String? + sortOrder Int @default(0) @map("sort_order") + /// FALSE = hidden: not offered in the quick entry, not a correlation + /// channel, history kept. Does not count toward the cap. + isActive Boolean @default(true) @map("is_active") + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + events SymptomEvent[] + + @@index([userId, sortOrder]) + @@map("symptom_definitions") +} + +/// One occurrence of a person-defined symptom: when, how strong (0-10, the +/// same numeric rating scale the pain score uses), an optional note, and an +/// optional link to the illness episode it happened during. v1.40. +model SymptomEvent { + id String @id @default(cuid()) + userId String @map("user_id") + definitionId String @map("definition_id") + /// The instant it happened (UTC). Day grouping is the reader's concern. + occurredAt DateTime @map("occurred_at") + /// 0-10, CHECK-constrained in the migration. + intensity Int + /// Optional free-text note, AES-256-GCM encrypted at rest. + noteEncrypted Bytes? @map("note_encrypted") + /// The episode the person said this happened during. Only an open episode + /// of the same account can be linked (the route checks); deleting the + /// episode row keeps the event and drops the link. + episodeId String? @map("episode_id") + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + definition SymptomDefinition @relation(fields: [definitionId], references: [id], onDelete: Cascade) + episode IllnessEpisode? @relation(fields: [episodeId], references: [id], onDelete: SetNull) + + @@index([userId, occurredAt]) + @@index([definitionId, occurredAt]) + @@index([episodeId]) + @@map("symptom_events") +} + // ─── Structured health records (v1.25 W-RECORDS) ────────────────── // // Allergies + family history kept as STRUCTURED records (not time-series diff --git a/scripts/rotate-encryption-key.ts b/scripts/rotate-encryption-key.ts index a814f2365..a3cd1742a 100644 --- a/scripts/rotate-encryption-key.ts +++ b/scripts/rotate-encryption-key.ts @@ -594,6 +594,21 @@ async function main() { prisma.illnessDayLog, ), ); + // v1.40 — person-defined symptoms: the name and the per-occurrence note. + results.push( + await rotateBytesColumn( + "SymptomDefinition", + "labelEncrypted", + prisma.symptomDefinition, + ), + ); + results.push( + await rotateBytesColumn( + "SymptomEvent", + "noteEncrypted", + prisma.symptomEvent, + ), + ); // ───── v1.19.0 ECG waveform (Bytes column) ───── // "waveformEncrypted" holds the JSON-encoded micro-volt sample array in the diff --git a/src/__tests__/sharing-surface-guard.test.ts b/src/__tests__/sharing-surface-guard.test.ts index 9a5e17cd0..e13fa70b6 100644 --- a/src/__tests__/sharing-surface-guard.test.ts +++ b/src/__tests__/sharing-surface-guard.test.ts @@ -991,6 +991,22 @@ const DELEGABLE_ROUTES: Record = { domain: "illness", why: "The day logs of one episode. The episode-scoped queries are safe because `loadOwnedEpisode(id, user.id)` runs first and now resolves against the record; the upsert arm keeps `requireAuth()`.", }, + "app/api/symptoms/definitions/route.ts": { + domain: "illness", + why: "The record's own symptom list. Module-gated on the record on both arms and scoped `userId: user.id`; the create arm is MANAGE (see the manage literal), because the vocabulary is the owner's and a delegate who may log an occurrence picks from it.", + }, + "app/api/symptoms/definitions/[id]/route.ts": { + domain: "illness", + why: "One symptom definition of the record, fetch-then-guard against the resolved user. Reached only through this module's MANAGE arm; the argument for admitting it is the manage literal's reason line.", + }, + "app/api/symptoms/events/route.ts": { + domain: "illness", + why: "The record's symptom occurrences. Module-gated on the record; the definition and the optional episode a create names are both re-narrowed to the resolved user before anything is written, so a delegate cannot file one record's occurrence under another's episode. The create arm is a delegable write.", + }, + "app/api/symptoms/events/[id]/route.ts": { + domain: "illness", + why: "One symptom occurrence of the record, fetch-then-guard against the resolved user. Reached only through this module's MANAGE arm.", + }, "app/api/workouts/route.ts": { domain: "measurements", why: "The record's workout list. Verified to carry no AI-written paragraph: the single-workout route beside it does, and stays refused for exactly that reason.", @@ -1427,6 +1443,8 @@ const DELEGABLE_WRITE_ROUTES: Record = { "Adding an analyte to the record's catalogue. A name the record already tracks is the ordinary 409 from the same `(userId, name)` uniqueness the owner would hit themselves.", "app/api/illness/episodes/route.ts": "Opening an illness episode. The module gate runs against the RECORD before the write, so a delegate cannot create an episode inside a record whose owner switched the module off.", + "app/api/symptoms/events/route.ts": + "Logging a symptom occurrence. The definition and the episode the body names are looked up under the RECORD, and an episode must still be open, so a delegate cannot attach an occurrence to an episode the owner closed or to another record's.", "app/api/encounters/route.ts": "Filing a visit, or booking one. Every id the body may carry — the practitioner, the checkup it closes, the three link arrays — is re-narrowed to the resolved record before anything is written, so a delegate cannot attach one record's document to another's visit. Booking mints the appointment reminder under the RECORD, which is correct: it is the owner's appointment and the owner's channels that should be nudged about it.", "app/api/practitioners/route.ts": @@ -1596,6 +1614,21 @@ const DELEGABLE_MANAGE_ROUTES: Record = { conditions: ["C3"], why: "Writing a day of an episode. The upsert revives a tombstoned day and replaces that day's symptom links, so C3 names the date, the previous symptom set, and the revival.", }, + "app/api/symptoms/definitions/route.ts": { + domain: "illness", + conditions: ["C1"], + why: "Defining a symptom on the record. Capped at eight active ones; C1 so a manager burns their own bucket rather than the owner's.", + }, + "app/api/symptoms/definitions/[id]/route.ts": { + domain: "illness", + conditions: ["C1", "C3", "C4"], + why: "Renaming, hiding and deleting a symptom. Hiding keeps the history; the purge is hard and takes the occurrences with it, so C3 names the definition and the count. C4 on the edit, over the plaintext fields only; a rename is named, never quoted. C1 on the bucket.", + }, + "app/api/symptoms/events/[id]/route.ts": { + domain: "illness", + conditions: ["C3", "C4"], + why: "Correcting and removing one occurrence. The delete is hard, so C3 carries its time and intensity; C4 on the edit, the encrypted note named and never quoted.", + }, "app/api/mood-entries/route.ts": { domain: "mind", conditions: ["C6"], @@ -1960,8 +1993,13 @@ const ACTOR_ROUTES: Record = { * v1.39.2 -- the body-site view adds one: a read on the record list in the * visits' domain, with conditions joined only under an illness grant. * 231 -> 232. + * + * v1.40 -- person-defined symptoms add eight: four modules on the record + * list, the occurrence create on the write literal, and the definition create, + * the definition edit/delete and the occurrence edit/delete on the manage + * literal. 232 -> 240. */ -const FROZEN_ENTRY_COUNT = 232; +const FROZEN_ENTRY_COUNT = 240; /** * The two surfaces that authenticate a Bearer token outside `requireAuth` — @@ -2722,7 +2760,7 @@ describe("(g) the MANAGE route set is frozen", () => { it("keeps the admitted mutation inventory complete and discoverable", () => { expect(ADMITTED_MUTATING_HANDLERS.length).toBeGreaterThan(0); - expect(ADMITTED_MUTATING_HANDLERS.length).toBe(77); + expect(ADMITTED_MUTATING_HANDLERS.length).toBe(83); const expected = ADMITTED_MUTATING_HANDLERS.map( ({ handlerModule, action, level }) => diff --git a/src/__tests__/v137-wave-zero-fixtures.test.ts b/src/__tests__/v137-wave-zero-fixtures.test.ts index 34bfae2db..f775c3e5d 100644 --- a/src/__tests__/v137-wave-zero-fixtures.test.ts +++ b/src/__tests__/v137-wave-zero-fixtures.test.ts @@ -26,7 +26,7 @@ describe("sharing and handler inventories", () => { }); it("keeps every admitted mutation uniquely addressable and fully controlled", () => { - expect(ADMITTED_MUTATING_HANDLERS).toHaveLength(77); + expect(ADMITTED_MUTATING_HANDLERS).toHaveLength(83); expect( new Set( ADMITTED_MUTATING_HANDLERS.map( diff --git a/src/app/api/symptoms/definitions/[id]/route.ts b/src/app/api/symptoms/definitions/[id]/route.ts new file mode 100644 index 000000000..e578f4a30 --- /dev/null +++ b/src/app/api/symptoms/definitions/[id]/route.ts @@ -0,0 +1,232 @@ +/** + * One person-defined symptom (v1.40). + * + * PATCH /api/symptoms/definitions/{id} — rename, change the icon, reorder, + * hide or show again. Showing a hidden one re-clears the cap. + * DELETE /api/symptoms/definitions/{id} — hide by default (history kept); + * `?purge=true` deletes the definition and every event of it. + * + * Owner-scoped: another account's id is a 404. Module-gated on the record. + * The label is never logged. + */ +import { NextRequest } from "next/server"; + +import { prisma } from "@/lib/db"; +import { + apiSuccess, + apiError, + getClientIp, + returnAllZodIssues, + safeJson, +} from "@/lib/api-response"; +import { apiHandler, requireRecordAuth } from "@/lib/api-handler"; +import { annotate } from "@/lib/logging/context"; +import { auditLog } from "@/lib/auth/audit"; +import { + destroyedDetails, + overwriteDetails, +} from "@/lib/sharing/audit-details"; +import { invalidateUserHealthContext } from "@/lib/cache/invalidate"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { requireIllnessEnabled } from "@/lib/illness/gate"; +import { + encryptSymptomText, + symptomDefinitionUpdateSchema, + toSymptomDefinitionDTO, + MAX_ACTIVE_SYMPTOM_DEFINITIONS, +} from "@/lib/symptoms/server"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ id: string }> }; + +function notFound(): Response { + return apiError("Symptom not found", 404, { + errorCode: "symptoms.definition.notFound", + }); +} + +export const PATCH = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const { user, actor } = await requireRecordAuth("manage", "illness"); + + const gate = await requireIllnessEnabled(user.id); + if (!gate.enabled) return gate.response; + + const rl = await checkRateLimit( + `symptoms:definition:${actor.id}`, + 30, + 60_000, + ); + if (!rl.allowed) { + return apiError("Too many requests, try again later", 429); + } + + const { id } = await params; + const { data: rawBody, error: jsonError } = await safeJson(request, { + maxBytes: 16 * 1024, + }); + if (jsonError) return jsonError; + const parsed = symptomDefinitionUpdateSchema.safeParse(rawBody); + if (!parsed.success) { + return returnAllZodIssues(parsed.error, 422, { + errorCode: "symptoms.invalid", + }); + } + + const existing = await prisma.symptomDefinition.findFirst({ + where: { id, userId: user.id }, + select: { id: true, icon: true, isActive: true, sortOrder: true }, + }); + if (!existing) return notFound(); + + // A hidden definition does not hold a slot; showing it again takes one. + if (parsed.data.isActive === true && !existing.isActive) { + const activeCount = await prisma.symptomDefinition.count({ + where: { userId: user.id, isActive: true }, + }); + if (activeCount >= MAX_ACTIVE_SYMPTOM_DEFINITIONS) { + return apiError( + `Symptom limit reached (${MAX_ACTIVE_SYMPTOM_DEFINITIONS})`, + 422, + { + errorCode: "symptoms.definition.limitReached", + limit: MAX_ACTIVE_SYMPTOM_DEFINITIONS, + }, + ); + } + } + + const updated = await prisma.symptomDefinition.update({ + where: { id: existing.id }, + data: { + ...(parsed.data.label !== undefined + ? { labelEncrypted: encryptSymptomText(parsed.data.label) } + : {}), + ...(parsed.data.icon !== undefined ? { icon: parsed.data.icon } : {}), + ...(parsed.data.isActive !== undefined + ? { isActive: parsed.data.isActive } + : {}), + ...(parsed.data.sortOrder !== undefined + ? { sortOrder: parsed.data.sortOrder } + : {}), + }, + select: { + id: true, + labelEncrypted: true, + icon: true, + sortOrder: true, + isActive: true, + }, + }); + + await auditLog("symptoms.definition.update", { + userId: user.id, + ipAddress: getClientIp(request), + details: { + definitionId: id, + ...overwriteDetails({ + before: { + icon: existing.icon, + isActive: existing.isActive, + sortOrder: existing.sortOrder, + }, + after: { + icon: updated.icon, + isActive: updated.isActive, + sortOrder: updated.sortOrder, + }, + redacted: parsed.data.label !== undefined ? ["label"] : [], + }), + }, + }); + + // Hiding or showing one changes the correlation channel set and the + // Coach's symptom block. + if (parsed.data.isActive !== undefined) + invalidateUserHealthContext(user.id); + + annotate({ + action: { + name: "symptoms.definition.update", + entity_type: "symptom_definition", + entity_id: id, + }, + }); + + return apiSuccess(toSymptomDefinitionDTO(updated)); + }, +); + +export const DELETE = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const { user, actor } = await requireRecordAuth("manage", "illness"); + + const gate = await requireIllnessEnabled(user.id); + if (!gate.enabled) return gate.response; + + const rl = await checkRateLimit( + `symptoms:definition:${actor.id}`, + 30, + 60_000, + ); + if (!rl.allowed) { + return apiError("Too many requests, try again later", 429); + } + + const { id } = await params; + const existing = await prisma.symptomDefinition.findFirst({ + where: { id, userId: user.id }, + select: { + id: true, + createdAt: true, + _count: { select: { events: true } }, + }, + }); + if (!existing) return notFound(); + + const purge = request.nextUrl.searchParams.get("purge") === "true"; + if (purge) { + // The FK cascade removes the events with it. + await prisma.symptomDefinition.delete({ where: { id: existing.id } }); + } else { + await prisma.symptomDefinition.update({ + where: { id: existing.id }, + data: { isActive: false }, + }); + } + + await auditLog("symptoms.definition.delete", { + userId: user.id, + ipAddress: getClientIp(request), + // C3 on the purge arm: the definition and its events are gone, so the + // audit row names what went. The label stays out (the person's words). + details: purge + ? destroyedDetails({ + model: "SymptomDefinition", + id, + label: null, + effectiveAt: existing.createdAt, + extra: { eventsDeleted: existing._count.events }, + }) + : { definitionId: id, purge }, + }); + + invalidateUserHealthContext(user.id); + + annotate({ + action: { + name: "symptoms.definition.delete", + entity_type: "symptom_definition", + entity_id: id, + }, + meta: { purge }, + }); + + return apiSuccess({ + deleted: true, + purged: purge, + eventsDeleted: purge ? existing._count.events : 0, + }); + }, +); diff --git a/src/app/api/symptoms/definitions/route.ts b/src/app/api/symptoms/definitions/route.ts new file mode 100644 index 000000000..ddcc7df04 --- /dev/null +++ b/src/app/api/symptoms/definitions/route.ts @@ -0,0 +1,139 @@ +/** + * Person-defined symptoms (v1.40). + * + * GET /api/symptoms/definitions — the record's own symptom list with a + * 30-day summary per symptom. `?includeHidden=true` adds hidden ones. + * POST /api/symptoms/definitions — define one (`{ label, icon? }`). At most + * eight active definitions; the label is encrypted at rest. + * + * Rides the illness module: an account (or instance) with it off answers 403 + * `illness.disabled` even to a valid Bearer token. The label is the person's + * own words for a health complaint, so it never reaches an audit row or a + * wide event. + */ +import { NextRequest } from "next/server"; + +import { prisma } from "@/lib/db"; +import { + apiSuccess, + apiError, + getClientIp, + returnAllZodIssues, + safeJson, +} from "@/lib/api-response"; +import { apiHandler, requireRecordAuth } from "@/lib/api-handler"; +import { annotate } from "@/lib/logging/context"; +import { withIdempotency } from "@/lib/idempotency"; +import { auditLog } from "@/lib/auth/audit"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { requireIllnessEnabled } from "@/lib/illness/gate"; +import { + encryptSymptomText, + listSymptomDefinitions, + symptomDefinitionCreateSchema, + toSymptomDefinitionDTO, + MAX_ACTIVE_SYMPTOM_DEFINITIONS, +} from "@/lib/symptoms/server"; + +export const dynamic = "force-dynamic"; + +export const GET = apiHandler(async (request: NextRequest) => { + const { user } = await requireRecordAuth("read", "illness"); + + const gate = await requireIllnessEnabled(user.id); + if (!gate.enabled) return gate.response; + + const includeHidden = + request.nextUrl.searchParams.get("includeHidden") === "true"; + const definitions = await listSymptomDefinitions(user.id, { includeHidden }); + + annotate({ + action: { name: "symptoms.definition.list" }, + meta: { count: definitions.length, includeHidden }, + }); + + return apiSuccess({ definitions, limit: MAX_ACTIVE_SYMPTOM_DEFINITIONS }); +}); + +/** + * Wrapped in `withIdempotency`: the label is ciphertext under a fresh IV, so + * no unique index can catch a replayed create, and a retry after a lost + * response would otherwise mint a second "aura" and burn a slot of eight. + */ +export const POST = apiHandler(withIdempotency<[NextRequest]>(postDefinition)); + +async function postDefinition(request: NextRequest): Promise { + // MANAGE, like the cycle module's own symptom list: the vocabulary of a + // record is the owner's, a delegate who may log an occurrence picks from it. + const { user, actor } = await requireRecordAuth("manage", "illness"); + + const gate = await requireIllnessEnabled(user.id); + if (!gate.enabled) return gate.response; + + const rl = await checkRateLimit( + `symptoms:definition:${actor.id}`, + 30, + 60_000, + ); + if (!rl.allowed) { + return apiError("Too many requests, try again later", 429); + } + + const { data: rawBody, error: jsonError } = await safeJson(request, { + maxBytes: 16 * 1024, + }); + if (jsonError) return jsonError; + const parsed = symptomDefinitionCreateSchema.safeParse(rawBody); + if (!parsed.success) { + return returnAllZodIssues(parsed.error, 422, { + errorCode: "symptoms.invalid", + }); + } + + const activeCount = await prisma.symptomDefinition.count({ + where: { userId: user.id, isActive: true }, + }); + if (activeCount >= MAX_ACTIVE_SYMPTOM_DEFINITIONS) { + return apiError( + `Symptom limit reached (${MAX_ACTIVE_SYMPTOM_DEFINITIONS})`, + 422, + { + errorCode: "symptoms.definition.limitReached", + limit: MAX_ACTIVE_SYMPTOM_DEFINITIONS, + }, + ); + } + + const created = await prisma.symptomDefinition.create({ + data: { + userId: user.id, + labelEncrypted: encryptSymptomText(parsed.data.label), + icon: parsed.data.icon ?? null, + sortOrder: activeCount, + }, + select: { + id: true, + labelEncrypted: true, + icon: true, + sortOrder: true, + isActive: true, + }, + }); + + await auditLog("symptoms.definition.create", { + userId: user.id, + ipAddress: getClientIp(request), + details: { definitionId: created.id, icon: created.icon }, + }); + + annotate({ + action: { + name: "symptoms.definition.create", + entity_type: "symptom_definition", + entity_id: created.id, + }, + meta: { icon: created.icon }, + }); + + return apiSuccess(toSymptomDefinitionDTO(created), 201); +} diff --git a/src/app/api/symptoms/events/[id]/route.ts b/src/app/api/symptoms/events/[id]/route.ts new file mode 100644 index 000000000..82928b38a --- /dev/null +++ b/src/app/api/symptoms/events/[id]/route.ts @@ -0,0 +1,192 @@ +/** + * One occurrence of a person-defined symptom (v1.40). + * + * PATCH /api/symptoms/events/{id} — change the time, intensity, note or + * episode link. A new link must name an open episode of the record; + * an unchanged link to an episode that has since resolved is kept. + * DELETE /api/symptoms/events/{id} — remove it. + * + * Owner-scoped (another account's id is a 404), module-gated on the record, + * MANAGE on a shared record. The note is never logged. + */ +import { NextRequest } from "next/server"; + +import { prisma } from "@/lib/db"; +import { + apiSuccess, + apiError, + getClientIp, + returnAllZodIssues, + safeJson, +} from "@/lib/api-response"; +import { apiHandler, requireRecordAuth } from "@/lib/api-handler"; +import { annotate } from "@/lib/logging/context"; +import { auditLog } from "@/lib/auth/audit"; +import { + destroyedDetails, + overwriteDetails, +} from "@/lib/sharing/audit-details"; +import { invalidateUserHealthContext } from "@/lib/cache/invalidate"; +import { requireIllnessEnabled } from "@/lib/illness/gate"; +import { + encryptSymptomText, + isLinkableEpisode, + symptomEventUpdateSchema, + toSymptomEventDTO, +} from "@/lib/symptoms/server"; + +export const dynamic = "force-dynamic"; + +type RouteParams = { params: Promise<{ id: string }> }; + +function notFound(): Response { + return apiError("Symptom event not found", 404, { + errorCode: "symptoms.event.notFound", + }); +} + +export const PATCH = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const { user } = await requireRecordAuth("manage", "illness"); + + const gate = await requireIllnessEnabled(user.id); + if (!gate.enabled) return gate.response; + + const { id } = await params; + const { data: rawBody, error: jsonError } = await safeJson(request, { + maxBytes: 16 * 1024, + }); + if (jsonError) return jsonError; + const parsed = symptomEventUpdateSchema.safeParse(rawBody); + if (!parsed.success) { + return returnAllZodIssues(parsed.error, 422, { + errorCode: "symptoms.invalid", + }); + } + const entry = parsed.data; + + const existing = await prisma.symptomEvent.findFirst({ + where: { id, userId: user.id }, + select: { + id: true, + occurredAt: true, + intensity: true, + episodeId: true, + }, + }); + if (!existing) return notFound(); + + if ( + entry.episodeId && + entry.episodeId !== existing.episodeId && + !(await isLinkableEpisode(user.id, entry.episodeId)) + ) { + return apiError("Episode is not open", 422, { + errorCode: "symptoms.episode.notOpen", + }); + } + + const updated = await prisma.symptomEvent.update({ + where: { id: existing.id }, + data: { + ...(entry.occurredAt !== undefined + ? { occurredAt: entry.occurredAt } + : {}), + ...(entry.intensity !== undefined + ? { intensity: entry.intensity } + : {}), + ...(entry.note !== undefined + ? { + noteEncrypted: entry.note ? encryptSymptomText(entry.note) : null, + } + : {}), + ...(entry.episodeId !== undefined + ? { episodeId: entry.episodeId ?? null } + : {}), + }, + }); + + await auditLog("symptoms.event.update", { + userId: user.id, + ipAddress: getClientIp(request), + details: { + eventId: id, + ...overwriteDetails({ + before: { + occurredAt: existing.occurredAt, + intensity: existing.intensity, + episodeId: existing.episodeId, + }, + after: { + occurredAt: updated.occurredAt, + intensity: updated.intensity, + episodeId: updated.episodeId, + }, + redacted: entry.note !== undefined ? ["note"] : [], + }), + }, + }); + + invalidateUserHealthContext(user.id); + + annotate({ + action: { + name: "symptoms.event.update", + entity_type: "symptom_event", + entity_id: id, + }, + }); + + return apiSuccess(toSymptomEventDTO(updated)); + }, +); + +export const DELETE = apiHandler( + async (request: NextRequest, { params }: RouteParams) => { + const { user } = await requireRecordAuth("manage", "illness"); + + const gate = await requireIllnessEnabled(user.id); + if (!gate.enabled) return gate.response; + + const { id } = await params; + const existing = await prisma.symptomEvent.findFirst({ + where: { id, userId: user.id }, + select: { + id: true, + definitionId: true, + occurredAt: true, + intensity: true, + }, + }); + if (!existing) return notFound(); + + await prisma.symptomEvent.delete({ where: { id: existing.id } }); + + await auditLog("symptoms.event.delete", { + userId: user.id, + ipAddress: getClientIp(request), + // C3: the row is gone after this; the audit row says what it was. The + // symptom's name is the person's own words and stays out of it, so the + // label is the definition id. + details: destroyedDetails({ + model: "SymptomEvent", + id, + label: existing.definitionId, + effectiveAt: existing.occurredAt, + extra: { intensity: existing.intensity }, + }), + }); + + invalidateUserHealthContext(user.id); + + annotate({ + action: { + name: "symptoms.event.delete", + entity_type: "symptom_event", + entity_id: id, + }, + }); + + return apiSuccess({ deleted: true }); + }, +); diff --git a/src/app/api/symptoms/events/route.ts b/src/app/api/symptoms/events/route.ts new file mode 100644 index 000000000..447502a0b --- /dev/null +++ b/src/app/api/symptoms/events/route.ts @@ -0,0 +1,166 @@ +/** + * Occurrences of person-defined symptoms (v1.40). + * + * GET /api/symptoms/events — newest first, optionally narrowed by + * `from` / `to` (ISO instants), `definitionId` and `episodeId`; at most + * 500 per page. + * POST /api/symptoms/events — log one (`{ definitionId, intensity, + * occurredAt?, note?, episodeId? }`). `occurredAt` defaults to now and + * may not lie in the future beyond the shared five-minute skew; + * `episodeId` must name one of the record's own episodes that is still + * open. The note is encrypted at rest. + * + * Module-gated on the record (`illness.disabled`). `userId` comes from auth + * and is never a body field. + */ +import { NextRequest } from "next/server"; + +import { prisma } from "@/lib/db"; +import { + apiSuccess, + apiError, + getClientIp, + returnAllZodIssues, + safeJson, +} from "@/lib/api-response"; +import { apiHandler, requireRecordAuth } from "@/lib/api-handler"; +import { annotate } from "@/lib/logging/context"; +import { withIdempotency } from "@/lib/idempotency"; +import { auditLog } from "@/lib/auth/audit"; +import { invalidateUserHealthContext } from "@/lib/cache/invalidate"; +import { requireIllnessEnabled } from "@/lib/illness/gate"; +import type { Prisma } from "@/generated/prisma/client"; +import { + encryptSymptomText, + isLinkableEpisode, + symptomEventCreateSchema, + symptomEventListQuerySchema, + toSymptomEventDTO, +} from "@/lib/symptoms/server"; + +export const dynamic = "force-dynamic"; + +export const GET = apiHandler(async (request: NextRequest) => { + const { user } = await requireRecordAuth("read", "illness"); + + const gate = await requireIllnessEnabled(user.id); + if (!gate.enabled) return gate.response; + + const sp = request.nextUrl.searchParams; + const parsed = symptomEventListQuerySchema.safeParse({ + from: sp.get("from") ?? undefined, + to: sp.get("to") ?? undefined, + definitionId: sp.get("definitionId") ?? undefined, + episodeId: sp.get("episodeId") ?? undefined, + limit: sp.get("limit") ?? undefined, + }); + if (!parsed.success) { + return returnAllZodIssues(parsed.error, 422, { + errorCode: "symptoms.invalid", + }); + } + const q = parsed.data; + + const occurredAt: Prisma.DateTimeFilter = {}; + if (q.from) occurredAt.gte = new Date(q.from); + if (q.to) occurredAt.lte = new Date(q.to); + + const rows = await prisma.symptomEvent.findMany({ + where: { + userId: user.id, + ...(q.from || q.to ? { occurredAt } : {}), + ...(q.definitionId ? { definitionId: q.definitionId } : {}), + ...(q.episodeId ? { episodeId: q.episodeId } : {}), + }, + orderBy: [{ occurredAt: "desc" }, { id: "desc" }], + take: q.limit, + }); + + annotate({ + action: { name: "symptoms.event.list" }, + meta: { count: rows.length, limit: q.limit }, + }); + + return apiSuccess({ + events: rows.map(toSymptomEventDTO), + limit: q.limit, + hasMore: rows.length === q.limit, + }); +}); + +// A retry from a flaky connection re-sends the same `Idempotency-Key` and gets +// the first answer back instead of a second occurrence at the same minute. +export const POST = apiHandler(withIdempotency<[NextRequest]>(postEvent)); + +async function postEvent(request: NextRequest): Promise { + // WRITE, like opening an episode or adding a reading: logging what happened + // is an admitted delegated write; editing or removing it is MANAGE. + const { user } = await requireRecordAuth("write", "illness"); + + const gate = await requireIllnessEnabled(user.id); + if (!gate.enabled) return gate.response; + + const { data: rawBody, error: jsonError } = await safeJson(request, { + maxBytes: 16 * 1024, + }); + if (jsonError) return jsonError; + const parsed = symptomEventCreateSchema.safeParse(rawBody); + if (!parsed.success) { + return returnAllZodIssues(parsed.error, 422, { + errorCode: "symptoms.invalid", + }); + } + const entry = parsed.data; + + // A hidden definition stays loggable from an older client's cached list; + // only another account's id (or none) is refused. + const definition = await prisma.symptomDefinition.findFirst({ + where: { id: entry.definitionId, userId: user.id }, + select: { id: true }, + }); + if (!definition) { + return apiError("Symptom not found", 404, { + errorCode: "symptoms.definition.notFound", + }); + } + + if (entry.episodeId && !(await isLinkableEpisode(user.id, entry.episodeId))) { + return apiError("Episode is not open", 422, { + errorCode: "symptoms.episode.notOpen", + }); + } + + const created = await prisma.symptomEvent.create({ + data: { + userId: user.id, + definitionId: definition.id, + occurredAt: entry.occurredAt ?? new Date(), + intensity: entry.intensity, + noteEncrypted: entry.note ? encryptSymptomText(entry.note) : null, + episodeId: entry.episodeId ?? null, + }, + }); + + await auditLog("symptoms.event.create", { + userId: user.id, + ipAddress: getClientIp(request), + details: { + eventId: created.id, + definitionId: created.definitionId, + linked: created.episodeId !== null, + }, + }); + + invalidateUserHealthContext(user.id); + + annotate({ + action: { + name: "symptoms.event.create", + entity_type: "symptom_event", + entity_id: created.id, + }, + meta: { linked: created.episodeId !== null }, + }); + + return apiSuccess(toSymptomEventDTO(created), 201); +} diff --git a/src/components/admin/backups-section.tsx b/src/components/admin/backups-section.tsx index bcd8cca24..3a0550cc5 100644 --- a/src/components/admin/backups-section.tsx +++ b/src/components/admin/backups-section.tsx @@ -461,6 +461,9 @@ function catalogueLabel( if (catalogue === "ecgReference") { return t("admin.section.backups.restoreSkippedEcgReference"); } + if (catalogue === "symptomEpisodeReference") { + return t("admin.section.backups.restoreSkippedSymptomEpisodeReference"); + } if (catalogue === "medicationTarget") { return t("admin.section.backups.restoreSkippedMedicationTarget"); } diff --git a/src/lib/crypto/encrypted-columns.ts b/src/lib/crypto/encrypted-columns.ts index 67d37b102..21deb72c0 100644 --- a/src/lib/crypto/encrypted-columns.ts +++ b/src/lib/crypto/encrypted-columns.ts @@ -310,6 +310,9 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [ // (`laterality`) stays plaintext, as on `Encounter`. { model: "IllnessEpisode", field: "bodySiteEncrypted", kind: "bytes" }, { model: "IllnessDayLog", field: "noteEncrypted", kind: "bytes" }, + // v1.40 — the person's own symptom names and the note on one occurrence. + { model: "SymptomDefinition", field: "labelEncrypted", kind: "bytes" }, + { model: "SymptomEvent", field: "noteEncrypted", kind: "bytes" }, // ───── v1.19.0 ECG waveform (Bytes column) ───── { model: "EcgRecording", field: "waveformEncrypted", kind: "bytes" }, diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 60512c4cf..c9b2926c3 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -123,6 +123,8 @@ export const WIPE_MODELS = [ "Allergy", "FamilyHistoryEntry", "IllnessDayLog", + // Before its definition and before the episode it may point at (SetNull). + "SymptomEvent", "EcgRecording", "NutrientIntakeDay", @@ -251,6 +253,7 @@ export const WIPE_MODELS = [ "MoodTagCategory", "CustomMetric", "IllnessEpisode", + "SymptomDefinition", "Workout", "InboundDocument", // Encounters point here with SetNull, so the address book outlives the diff --git a/src/lib/export/__tests__/full-backup-payload.test.ts b/src/lib/export/__tests__/full-backup-payload.test.ts index c4a65b1ec..d1a242631 100644 --- a/src/lib/export/__tests__/full-backup-payload.test.ts +++ b/src/lib/export/__tests__/full-backup-payload.test.ts @@ -426,6 +426,7 @@ function makePrisma() { // an account whose watch has never taken one is the ordinary case the // builder must still answer for. ecgRecording: { findMany: vi.fn().mockResolvedValue([]) }, + symptomDefinition: { findMany: vi.fn().mockResolvedValue([]) }, // Left unmocked on purpose: `buildProfileBackupSection` runs for real // against these, so the assertions below exercise the builder rather than // a stand-in that would agree with whatever the payload happened to do. diff --git a/src/lib/export/__tests__/restore-backup-round-trip.test.ts b/src/lib/export/__tests__/restore-backup-round-trip.test.ts index a988ddfed..116fdf6d6 100644 --- a/src/lib/export/__tests__/restore-backup-round-trip.test.ts +++ b/src/lib/export/__tests__/restore-backup-round-trip.test.ts @@ -312,6 +312,7 @@ function sourceClient() { documentConditionLink: { findMany: vi.fn().mockResolvedValue([]) }, extractedFact: { findMany: vi.fn().mockResolvedValue([]) }, ecgRecording: { findMany: vi.fn().mockResolvedValue([]) }, + symptomDefinition: { findMany: vi.fn().mockResolvedValue([]) }, customMetric: { findMany: vi.fn().mockResolvedValue([ { diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index 6af2a3d66..14118bde3 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -95,6 +95,10 @@ export const BACKED_UP_MODELS = [ "IllnessEpisode", "IllnessDayLog", "IllnessSymptomLink", + // v1.40 — the person's own symptoms and every occurrence of them. Nothing + // else holds either; a restore without them loses the history outright. + "SymptomDefinition", + "SymptomEvent", "EcgRecording", "UserHealthProfile", "HealthProfileFactRevision", @@ -250,6 +254,10 @@ export const BACKUP_WRITER_FILES: readonly string[] = [ // because that reference is a pointer between two independently carried // tables and not a parent-child ride. "src/lib/export/ecg-backup.ts", + // The person's own symptoms, both ends beside each other; the occurrences + // ride inside their definition (`include: { events }` out, `events: { + // create }` back), the way a medication's side effects do. + "src/lib/export/symptoms-backup.ts", "src/lib/cycle/backup.ts", ]; @@ -268,6 +276,7 @@ export const BACKUP_RESTORE_FILES: readonly string[] = [ "src/lib/export/environment-backup.ts", "src/lib/export/onboarding-backup.ts", "src/lib/export/ecg-backup.ts", + "src/lib/export/symptoms-backup.ts", "src/lib/cycle/backup.ts", ]; @@ -308,6 +317,8 @@ export const TWO_ENDED_MODELS = [ "IllnessEpisode", "IllnessDayLog", "IllnessSymptomLink", + "SymptomDefinition", + "SymptomEvent", "UserHealthProfile", "HealthProfileFactRevision", "CycleProfile", diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index 5f5e35fed..a57822647 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -106,6 +106,12 @@ import { type EcgBackupCounts, type EcgBackupSection, } from "@/lib/export/ecg-backup"; +import { + buildSymptomsBackupSection, + countSymptomsBackupSection, + type SymptomsBackupCounts, + type SymptomsBackupSection, +} from "@/lib/export/symptoms-backup"; import { buildIntradayProfileBackupSection, countIntradayProfileBackupSection, @@ -140,7 +146,8 @@ export interface FullBackupCounts AwardsBackupCounts, EnvironmentBackupCounts, OnboardingBackupCounts, - EcgBackupCounts { + EcgBackupCounts, + SymptomsBackupCounts { measurements: number; medications: number; intakeEvents: number; @@ -710,6 +717,7 @@ export async function buildFullBackupPayload( environment, onboardingRecord, ecg, + symptoms, nutrientDays, accountSettings, ] = await Promise.all([ @@ -886,6 +894,11 @@ export async function buildFullBackupPayload( buildEcgBackupSection(prisma, userId, { purpose: disasterRecovery ? "disaster-recovery" : "portable-export", }), + // The person's own symptoms with their occurrences nested inside. Both + // ends live in `src/lib/export/symptoms-backup.ts`. + buildSymptomsBackupSection(prisma, userId, { + purpose: disasterRecovery ? "disaster-recovery" : "portable-export", + }), // Nutrient day totals were absent from every export path, which // contradicted the schema's own reason for denormalising the unit column // ("rows stay self-describing in exports even if the catalog ever drifts"). @@ -939,6 +952,7 @@ export async function buildFullBackupPayload( const awardsSection: AwardsBackupSection = awards; const environmentSection: EnvironmentBackupSection = environment; const ecgSection: EcgBackupSection = ecg; + const symptomsSection: SymptomsBackupSection = symptoms; const accountSettingsSection: AccountSettingsBackupSection = accountSettings; // The three unbounded tables, either read into arrays or left as markers @@ -1306,6 +1320,7 @@ export async function buildFullBackupPayload( ...awardsSection, ...environmentSection, ...ecgSection, + ...symptomsSection, nutrientDays: nutrientDays.map((n) => ({ day: n.day, nutrient: n.nutrient, @@ -1373,6 +1388,7 @@ export async function buildFullBackupPayload( ...countAwardsBackupSection(awards), ...countEnvironmentBackupSection(environment), ...countEcgBackupSection(ecg), + ...countSymptomsBackupSection(symptoms), }, }; } diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index 8b80622d1..53e6608b9 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -106,6 +106,7 @@ import { restoreDocumentFilingData } from "@/lib/export/document-filing-backup"; import { restoreAwardsData } from "@/lib/export/awards-backup"; import { restoreEnvironmentData } from "@/lib/export/environment-backup"; import { restoreEcgData } from "@/lib/export/ecg-backup"; +import { restoreSymptomsData } from "@/lib/export/symptoms-backup"; import { restoredMedicationCreatedAt } from "@/lib/export/medication-created-at"; import { ensureMedicationCategoryTable, @@ -179,6 +180,7 @@ export interface RestoreResponse { environmentContexts: number; environmentTravelLocations: number; ecgRecordings: number; + symptomDefinitions: number; }; } @@ -2079,6 +2081,18 @@ export async function restoreBackup( } } + // The person's own symptoms and their occurrences. After the episodes, + // because an occurrence may point at one; the id set is the one this + // branch just wrote. Both ends live in + // `src/lib/export/symptoms-backup.ts`. + const symptomsCleared = await restoreSymptomsData( + tx, + ownerId, + payload, + episodeIds, + skips, + ); + reportSection("allergies"); for (const allergy of payload.allergies) { await tx.allergy.create({ @@ -2460,6 +2474,7 @@ export async function restoreBackup( environmentTravelLocations: environmentCleared.environmentTravelLocations, ecgRecordings: ecgCleared.ecgRecordings, + symptomDefinitions: symptomsCleared.symptomDefinitions, }; assertNoNewForeignReferences( foreignBefore, diff --git a/src/lib/export/restore-skips.ts b/src/lib/export/restore-skips.ts index 5e98d1b4b..d79d09bbd 100644 --- a/src/lib/export/restore-skips.ts +++ b/src/lib/export/restore-skips.ts @@ -146,6 +146,12 @@ * know, a threshold out of range, an avatar the upload would refuse. Its key * is the column (or `column.part` for one band or one site), never the value. * The account keeps what it had for that setting; everything else comes back. + * + * The fifteenth, `symptomEpisodeReference`, names the illness episode a + * symptom occurrence was filed against that the restore did not put back. A + * real foreign key, so the POINTER is dropped and the occurrence restores + * unlinked. A portable export omits soft-deleted episodes, which is the + * ordinary way an occurrence ends up naming one the file does not carry. */ export type SkippedCatalogue = | "cycleSymptom" @@ -165,7 +171,8 @@ export type SkippedCatalogue = | "medicationTarget" | "scheduleRevisionLink" | "checkupClosure" - | "accountSetting"; + | "accountSetting" + | "symptomEpisodeReference"; /** One key this instance does not know, and the links it cost. */ export interface SkippedCatalogueKey { diff --git a/src/lib/export/symptoms-backup.ts b/src/lib/export/symptoms-backup.ts new file mode 100644 index 000000000..c38f8a80c --- /dev/null +++ b/src/lib/export/symptoms-backup.ts @@ -0,0 +1,289 @@ +/** + * Person-defined symptoms, with both backup ends in one file (v1.40). + * + * Same arrangement as `ecg-backup.ts` and `reminders-backup.ts`: a reader + * asking "is this carried at both ends?" answers it here, and a reader who + * greps only the restore route gets a false negative because the route + * delegates. + * + * ## Events ride inside their definition + * + * An occurrence means nothing without the symptom it is an occurrence of, so + * the file nests each definition's events under it, the way a medication's + * side effects ride inside the medication. A nested create binds the child to + * whatever id the parent row actually got, so there is no definition reference + * to resolve and none that can dangle. + * + * ## Free text follows the note contract + * + * A disaster-recovery payload carries the label and note ciphertext verbatim + * as base64 (the same instance's key reads it back unchanged). A portable + * export decrypts both, because a portable file exists to be readable by the + * person who owns it and to restore under another instance's key. A label this + * instance cannot decrypt travels as the visible unreadable marker rather than + * as nothing, the rule `records-backup.ts` states for notes: the definition is + * still the person's, and an empty name would read as one they never gave. + * + * ## The one reference that needs care + * + * `episodeId` addresses an illness episode and is a real foreign key. A + * portable export omits soft-deleted episodes, so an occurrence filed against + * an episode the person has since deleted names a row the file does not carry. + * The restore resolves the reference against the episodes it actually wrote + * and NULLS a miss (reported as `symptomEpisodeReference`) rather than writing + * it and rolling the whole restore back. The occurrence is the record; the + * episode link is a cross-reference. + */ +import { Buffer } from "node:buffer"; + +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; + +import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec"; +import { getEvent } from "@/lib/logging/context"; +import { + recordUnknownKeys, + type RestoreSkipLog, +} from "@/lib/export/restore-skips"; + +import { UNREADABLE_EXPORT_MARKER } from "./unreadable-marker"; + +export interface SymptomsBackupOptions { + purpose?: "portable-export" | "disaster-recovery"; +} + +/** One occurrence, nested under its definition. */ +export interface SymptomEventBackupEntry { + /** Present in a disaster-recovery payload. */ + id?: string; + occurredAt: string; + intensity: number; + /** Plaintext; present on a portable payload. */ + note?: string | null; + /** Ciphertext as base64; present on a disaster-recovery payload. */ + noteEncrypted?: string | null; + episodeId: string | null; + createdAt: string; + updatedAt: string; +} + +/** One definition and every occurrence of it. */ +export interface SymptomDefinitionBackupEntry { + /** Present in a disaster-recovery payload. */ + id?: string; + /** Plaintext; present on a portable payload. */ + label?: string; + /** Ciphertext as base64; present on a disaster-recovery payload. */ + labelEncrypted?: string; + icon: string | null; + sortOrder: number; + isActive: boolean; + createdAt: string; + updatedAt: string; + events: SymptomEventBackupEntry[]; +} + +export interface SymptomsBackupSection { + symptomDefinitions: SymptomDefinitionBackupEntry[]; +} + +export interface SymptomsBackupCounts { + symptomDefinitions: number; + symptomEvents: number; +} + +function base64(buf: Uint8Array): string { + return Buffer.from(buf).toString("base64"); +} + +function readTextForExport(buf: Uint8Array, field: string): string { + try { + return decryptFromBytes(buf); + } catch (err) { + getEvent()?.addWarning( + `symptom ${field} decrypt failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return UNREADABLE_EXPORT_MARKER; + } +} + +/** + * Build the symptoms slice of a user's full backup. The tables carry no + * tombstone, so both purposes read the same rows and differ only in how the + * free text travels. + */ +export async function buildSymptomsBackupSection( + prisma: Pick, + userId: string, + options: SymptomsBackupOptions = {}, +): Promise { + const disasterRecovery = options.purpose === "disaster-recovery"; + const rows = await prisma.symptomDefinition.findMany({ + where: { userId }, + orderBy: [{ sortOrder: "asc" }, { createdAt: "asc" }, { id: "asc" }], + include: { + events: { orderBy: [{ occurredAt: "asc" }, { id: "asc" }] }, + }, + }); + + return { + symptomDefinitions: rows.map((row) => ({ + ...(disasterRecovery + ? { id: row.id, labelEncrypted: base64(row.labelEncrypted) } + : { label: readTextForExport(row.labelEncrypted, "label") }), + icon: row.icon, + sortOrder: row.sortOrder, + isActive: row.isActive, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + events: row.events.map((event) => ({ + ...(disasterRecovery + ? { + id: event.id, + noteEncrypted: event.noteEncrypted + ? base64(event.noteEncrypted) + : null, + } + : { + note: event.noteEncrypted + ? readTextForExport(event.noteEncrypted, "note") + : null, + }), + occurredAt: event.occurredAt.toISOString(), + intensity: event.intensity, + episodeId: event.episodeId, + createdAt: event.createdAt.toISOString(), + updatedAt: event.updatedAt.toISOString(), + })), + })), + }; +} + +export function countSymptomsBackupSection( + section: SymptomsBackupSection, +): SymptomsBackupCounts { + return { + symptomDefinitions: section.symptomDefinitions.length, + symptomEvents: section.symptomDefinitions.reduce( + (sum, d) => sum + d.events.length, + 0, + ), + }; +} + +/** Counts the symptoms restore wiped, for the audit trail. */ +export interface SymptomsRestoreCleared { + symptomDefinitions: number; +} + +/** + * What the parser hands over, which is looser than what the builder writes: + * an older file carries no key at all, and optional columns arrive as + * `undefined` rather than `null`. + */ +export interface SymptomsRestoreInput { + symptomDefinitions: Array<{ + id?: string | undefined; + label?: string | undefined; + labelEncrypted?: string | undefined; + icon?: string | null | undefined; + sortOrder: number; + isActive: boolean; + createdAt: string; + updatedAt: string; + events: Array<{ + id?: string | undefined; + occurredAt: string; + intensity: number; + note?: string | null | undefined; + noteEncrypted?: string | null | undefined; + episodeId?: string | null | undefined; + createdAt: string; + updatedAt: string; + }>; + }>; +} + +function decodeBytes(encoded: string): Uint8Array { + const decoded = Buffer.from(encoded, "base64"); + const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength)); + bytes.set(decoded); + return bytes; +} + +/** + * Re-create the account's symptom definitions and their occurrences. + * + * Delete-then-recreate inside the caller's transaction, like every other + * section; deleting a definition cascades its events. MUST run AFTER the + * illness episodes are restored: `episodeIds` is the set the restore actually + * wrote, passed in rather than re-queried so this stays a pure function of the + * transaction it was handed. + */ +export async function restoreSymptomsData( + tx: Prisma.TransactionClient, + ownerId: string, + payload: SymptomsRestoreInput, + episodeIds: ReadonlySet, + skips: RestoreSkipLog, +): Promise { + const cleared = await tx.symptomDefinition.deleteMany({ + where: { userId: ownerId }, + }); + + const droppedEpisodeRefs: string[] = []; + for (const definition of payload.symptomDefinitions) { + const labelEncrypted = + definition.labelEncrypted !== undefined + ? decodeBytes(definition.labelEncrypted) + : encryptToBytes(definition.label ?? ""); + await tx.symptomDefinition.create({ + data: { + ...(definition.id ? { id: definition.id } : {}), + userId: ownerId, + labelEncrypted, + icon: definition.icon ?? null, + sortOrder: definition.sortOrder, + isActive: definition.isActive, + createdAt: new Date(definition.createdAt), + updatedAt: new Date(definition.updatedAt), + events: { + create: definition.events.map((event) => { + let episodeId = event.episodeId ?? null; + if (episodeId && !episodeIds.has(episodeId)) { + droppedEpisodeRefs.push(episodeId); + episodeId = null; + } + return { + ...(event.id ? { id: event.id } : {}), + userId: ownerId, + occurredAt: new Date(event.occurredAt), + intensity: event.intensity, + noteEncrypted: + event.noteEncrypted !== undefined + ? event.noteEncrypted === null + ? null + : decodeBytes(event.noteEncrypted) + : event.note + ? encryptToBytes(event.note) + : null, + episodeId, + createdAt: new Date(event.createdAt), + updatedAt: new Date(event.updatedAt), + }; + }), + }, + }, + }); + } + + recordUnknownKeys( + skips, + "symptomEpisodeReference", + [...new Set(droppedEpisodeRefs)], + droppedEpisodeRefs, + ); + + return { symptomDefinitions: cleared.count }; +} diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 9f4d83bf7..b157a9918 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -346,6 +346,13 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "source-priority.body.invalid_json", "source-priority.body.invalid_shape", ], + symptoms: [ + "symptoms.definition.limitReached", + "symptoms.definition.notFound", + "symptoms.episode.notOpen", + "symptoms.event.notFound", + "symptoms.invalid", + ], tokens: [ "tokens.documents.ceiling_reached", "tokens.measurements.ceiling_reached", diff --git a/src/lib/openapi/routes/index.ts b/src/lib/openapi/routes/index.ts index 195210048..350bc02ba 100644 --- a/src/lib/openapi/routes/index.ts +++ b/src/lib/openapi/routes/index.ts @@ -84,6 +84,7 @@ import { profilePaths } from "./profile"; import { recordSettingsPaths } from "./record-settings"; import { retiredPaths } from "./retired"; import { settingsPaths } from "./settings"; +import { symptomPaths } from "./symptoms"; import { syncPaths } from "./sync"; import { workoutPaths } from "./workouts"; import { coachPrefsSchema } from "./shared"; @@ -177,6 +178,9 @@ export const openApiPaths: NonNullable = { // Papra, searching it and importing picked documents; cookie-only // (appended, spread order is load-bearing). ...documentSourcePaths, + // v1.40 — person-defined symptoms and their occurrences, behind the illness + // module (appended, spread order is load-bearing). + ...symptomPaths, // The environmental-context overview. Its own module because nothing else // owns the surface (appended, spread order is load-bearing). // diff --git a/src/lib/openapi/routes/symptoms.ts b/src/lib/openapi/routes/symptoms.ts new file mode 100644 index 000000000..8f3df7bf4 --- /dev/null +++ b/src/lib/openapi/routes/symptoms.ts @@ -0,0 +1,371 @@ +/** + * OpenAPI route table for person-defined symptoms (`/api/symptoms`, v1.40). + * + * Part of the OpenAPI route table; aggregated in `./index.ts`. Request bodies + * and queries reuse the runtime Zod schemas from `@/lib/symptoms/shared` so the + * wire contract stays single-source; response shapes mirror the DTOs the + * routes serialise (decrypted label and note, the 30-day summary). + * + * Every route rides the illness module: an account that turned it off (or an + * operator-disabled instance) answers 403 `illness.disabled` even to a valid + * Bearer token. + */ +import type { ZodOpenApiObject } from "zod-openapi"; +import { z } from "zod/v4"; + +import { + symptomDefinitionCreateSchema, + symptomDefinitionUpdateSchema, + symptomEventCreateSchema, + symptomEventListQuerySchema, + symptomEventUpdateSchema, +} from "@/lib/symptoms/shared"; + +import { + dataEnvelope, + errorEnvelope, + idempotencyKeyParameter, + idempotentWrite, + recordRefusal, + stdResponses, +} from "./shared"; + +const MODULE_OFF = + "`illness.disabled`: the record has the illness module switched off (or the operator turned it off server-wide); symptoms ride that module."; + +const createSymptomDefinitionRequest = symptomDefinitionCreateSchema.meta({ + id: "CreateSymptomDefinitionRequest", + description: + "Define a symptom of your own. `label` (1-40 characters) is encrypted at rest; `icon` is one of the cycle custom-symptom icon names, null for the default glyph. At most eight active definitions per account (422 `symptoms.definition.limitReached`).", +}); + +const updateSymptomDefinitionRequest = symptomDefinitionUpdateSchema.meta({ + id: "UpdateSymptomDefinitionRequest", + description: + "Partial edit; an omitted key leaves the column untouched. `isActive: false` hides the symptom (history kept, slot freed); `isActive: true` shows it again and re-checks the cap.", +}); + +const createSymptomEventRequest = symptomEventCreateSchema.meta({ + id: "CreateSymptomEventRequest", + description: + "Log one occurrence. `intensity` is 0-10 (0 none, 10 the worst imaginable). `occurredAt` defaults to now and may not lie in the future beyond five minutes. `episodeId`, when given, must name one of the record's own illness episodes that is still open (422 `symptoms.episode.notOpen` otherwise). The note is encrypted at rest.", +}); + +const updateSymptomEventRequest = symptomEventUpdateSchema.meta({ + id: "UpdateSymptomEventRequest", + description: + "Partial edit of one occurrence. `note: null` clears the note, `episodeId: null` drops the link. A changed `episodeId` must name an open episode; an unchanged link to an episode that has since resolved is kept.", +}); + +const listSymptomEventsQuery = symptomEventListQuerySchema.meta({ + id: "ListSymptomEventsQuery", + description: + "Optional `from` / `to` (ISO instants, inclusive), `definitionId`, `episodeId`, and `limit` (1-500, default 200). Newest first.", +}); + +const symptomDefinition = z + .object({ + id: z.string(), + label: z.string().nullable(), + icon: z.string().nullable(), + sortOrder: z.number().int(), + isActive: z.boolean(), + recent: z.object({ + count30d: z.number().int(), + maxIntensity30d: z.number().int().nullable(), + lastOccurredAt: z.string().nullable(), + }), + }) + .meta({ + id: "SymptomDefinition", + description: + "A symptom the person defined. `label` is decrypted (null only on an unreadable value). `recent` is computed server-side: occurrences and the highest intensity over the last 30 days, and the last occurrence ever. Each active definition is also the correlation channel `SYMPTOM:`.", + }); + +const symptomDefinitionList = z + .object({ + definitions: z.array(symptomDefinition), + limit: z.number().int(), + }) + .meta({ + id: "SymptomDefinitionList", + description: + "The record's symptom definitions in display order, and the active-definition cap.", + }); + +const symptomEvent = z + .object({ + id: z.string(), + definitionId: z.string(), + occurredAt: z.string(), + intensity: z.number().int(), + note: z.string().nullable(), + episodeId: z.string().nullable(), + createdAt: z.string(), + updatedAt: z.string(), + }) + .meta({ + id: "SymptomEvent", + description: + "One occurrence of a defined symptom: when, how strong (0-10), the decrypted note, and the illness episode it was filed against (null when none).", + }); + +const symptomEventList = z + .object({ + events: z.array(symptomEvent), + limit: z.number().int(), + hasMore: z.boolean(), + }) + .meta({ + id: "SymptomEventList", + description: + "A page of occurrences, newest first. `hasMore` is true when the page is full; page on with `to` set to the oldest `occurredAt` seen.", + }); + +const deletedResponse = z + .object({ + deleted: z.literal(true), + purged: z.boolean(), + eventsDeleted: z.number().int(), + }) + .meta({ + id: "SymptomDefinitionDeleted", + description: + "`purged: false` means the definition was hidden and its history kept; `purged: true` means it and `eventsDeleted` occurrences were removed.", + }); + +const pathId = { path: z.object({ id: z.string() }) }; + +const definitionNotFound = { + "404": { + description: + "`symptoms.definition.notFound`: no definition with this id on the record.", + content: { "application/json": { schema: errorEnvelope } }, + }, +} as const; + +const eventNotFound = { + "404": { + description: + "`symptoms.event.notFound`: no occurrence with this id on the record.", + content: { "application/json": { schema: errorEnvelope } }, + }, +} as const; + +export const symptomPaths: NonNullable = { + "/api/symptoms/definitions": { + get: { + tags: ["Symptoms"], + summary: "List your own symptoms (v1.40)", + description: + "The record's active symptom definitions with their 30-day summary; `includeHidden=true` adds the hidden ones. Rides the illness module.", + requestParams: { + query: z.object({ + includeHidden: z.enum(["true", "false"]).optional(), + }), + }, + responses: { + ...recordRefusal(MODULE_OFF), + "200": { + description: "The definitions.", + content: { + "application/json": { + schema: dataEnvelope( + symptomDefinitionList, + "ListSymptomDefinitionsEnvelope", + ), + }, + }, + }, + ...stdResponses, + }, + }, + post: { + tags: ["Symptoms"], + summary: "Define a symptom (v1.40)", + description: + "Creates one definition. Idempotent under `Idempotency-Key`. Audits as `symptoms.definition.create` without the label. Rate-limited to 30 a minute.", + parameters: [idempotencyKeyParameter], + requestBody: { + required: true, + content: { + "application/json": { schema: createSymptomDefinitionRequest }, + }, + }, + responses: { + ...idempotentWrite(), + ...recordRefusal(MODULE_OFF), + "201": { + description: "Definition created.", + content: { + "application/json": { + schema: dataEnvelope( + symptomDefinition, + "CreateSymptomDefinitionEnvelope", + ), + }, + }, + }, + ...stdResponses, + }, + }, + }, + "/api/symptoms/definitions/{id}": { + patch: { + tags: ["Symptoms"], + summary: "Edit, hide or show a symptom (v1.40)", + description: + "Partial edit. Audits as `symptoms.definition.update`, naming a label change without quoting it.", + requestParams: pathId, + requestBody: { + required: true, + content: { + "application/json": { schema: updateSymptomDefinitionRequest }, + }, + }, + responses: { + ...recordRefusal(MODULE_OFF), + "200": { + description: "Definition updated.", + content: { + "application/json": { + schema: dataEnvelope( + symptomDefinition, + "UpdateSymptomDefinitionEnvelope", + ), + }, + }, + }, + ...definitionNotFound, + ...stdResponses, + }, + }, + delete: { + tags: ["Symptoms"], + summary: "Hide or delete a symptom (v1.40)", + description: + "Hides the definition and keeps its history. `?purge=true` deletes it and every occurrence of it.", + requestParams: { + ...pathId, + query: z.object({ purge: z.enum(["true", "false"]).optional() }), + }, + responses: { + ...recordRefusal(MODULE_OFF), + "200": { + description: "Hidden or deleted.", + content: { + "application/json": { + schema: dataEnvelope( + deletedResponse, + "DeleteSymptomDefinitionEnvelope", + ), + }, + }, + }, + ...definitionNotFound, + ...stdResponses, + }, + }, + }, + "/api/symptoms/events": { + get: { + tags: ["Symptoms"], + summary: "List symptom occurrences (v1.40)", + description: + "Occurrences of the record's symptoms, newest first, narrowed by the optional query.", + requestParams: { query: listSymptomEventsQuery }, + responses: { + ...recordRefusal(MODULE_OFF), + "200": { + description: "A page of occurrences.", + content: { + "application/json": { + schema: dataEnvelope( + symptomEventList, + "ListSymptomEventsEnvelope", + ), + }, + }, + }, + ...stdResponses, + }, + }, + post: { + tags: ["Symptoms"], + summary: "Log a symptom occurrence (v1.40)", + description: + "Creates one occurrence. Idempotent under `Idempotency-Key`. A hidden definition can still be logged against; another record's definition 404s (`symptoms.definition.notFound`). Audits as `symptoms.event.create` without the note.", + parameters: [idempotencyKeyParameter], + requestBody: { + required: true, + content: { + "application/json": { schema: createSymptomEventRequest }, + }, + }, + responses: { + ...idempotentWrite(), + ...recordRefusal(MODULE_OFF), + "201": { + description: "Occurrence logged.", + content: { + "application/json": { + schema: dataEnvelope(symptomEvent, "CreateSymptomEventEnvelope"), + }, + }, + }, + ...definitionNotFound, + ...stdResponses, + }, + }, + }, + "/api/symptoms/events/{id}": { + patch: { + tags: ["Symptoms"], + summary: "Edit a symptom occurrence (v1.40)", + description: + "Partial edit. Audits as `symptoms.event.update`, naming a note change without quoting it.", + requestParams: pathId, + requestBody: { + required: true, + content: { + "application/json": { schema: updateSymptomEventRequest }, + }, + }, + responses: { + ...recordRefusal(MODULE_OFF), + "200": { + description: "Occurrence updated.", + content: { + "application/json": { + schema: dataEnvelope(symptomEvent, "UpdateSymptomEventEnvelope"), + }, + }, + }, + ...eventNotFound, + ...stdResponses, + }, + }, + delete: { + tags: ["Symptoms"], + summary: "Delete a symptom occurrence (v1.40)", + description: "Removes the occurrence. Audits as `symptoms.event.delete`.", + requestParams: pathId, + responses: { + ...recordRefusal(MODULE_OFF), + "200": { + description: "Deleted.", + content: { + "application/json": { + schema: dataEnvelope( + z.object({ deleted: z.literal(true) }), + "DeleteSymptomEventEnvelope", + ), + }, + }, + }, + ...eventNotFound, + ...stdResponses, + }, + }, + }, +}; diff --git a/src/lib/record-activity/activity-verb.ts b/src/lib/record-activity/activity-verb.ts index b9e7a128d..66781e05b 100644 --- a/src/lib/record-activity/activity-verb.ts +++ b/src/lib/record-activity/activity-verb.ts @@ -72,6 +72,8 @@ export function recordActivityVerbLine( return t("recordSharing.activityVerb.familyHistoryCreate", { name }); case "illness.episode.create": return t("recordSharing.activityVerb.illnessEpisodeCreate", { name }); + case "symptoms.event.create": + return t("recordSharing.activityVerb.symptomEventCreate", { name }); case "encounter.visit.create": return t("recordSharing.activityVerb.encounterVisitCreate", { name }); case "vaccination.record.create": @@ -193,6 +195,16 @@ export function recordActivityVerbLine( return t("recordSharing.activityVerb.illnessEpisodeRestore", { name }); case "illness.day-log.upsert": return t("recordSharing.activityVerb.illnessDayLogUpsert", { name }); + case "symptoms.definition.create": + return t("recordSharing.activityVerb.symptomDefinitionCreate", { name }); + case "symptoms.definition.update": + return t("recordSharing.activityVerb.symptomDefinitionUpdate", { name }); + case "symptoms.definition.delete": + return t("recordSharing.activityVerb.symptomDefinitionDelete", { name }); + case "symptoms.event.update": + return t("recordSharing.activityVerb.symptomEventUpdate", { name }); + case "symptoms.event.delete": + return t("recordSharing.activityVerb.symptomEventDelete", { name }); case "encounter.visit.update": return t("recordSharing.activityVerb.encounterVisitUpdate", { name }); diff --git a/src/lib/symptoms/server.ts b/src/lib/symptoms/server.ts new file mode 100644 index 000000000..fbf46feae --- /dev/null +++ b/src/lib/symptoms/server.ts @@ -0,0 +1,154 @@ +/** + * Person-defined symptoms (v1.40): the server-only half. Label and note + * crypto, the row → wire mappers, the 30-day header summary and the one check + * that decides whether an event may be filed against an episode. + * + * Free text (the symptom's name, the note on one occurrence) is held + * AES-256-GCM encrypted in Bytes columns through the shared codec, the same + * codec the illness episode note uses, and decrypted fail-soft on read: a key + * gap on one row reads `null` rather than taking the page down. + */ +import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec"; +import { prisma } from "@/lib/db"; +import { getEvent } from "@/lib/logging/context"; +import type { + SymptomDefinition, + SymptomEvent, +} from "@/generated/prisma/client"; + +import type { SymptomDefinitionDTO, SymptomEventDTO } from "./shared"; + +export * from "./shared"; + +/** Encrypt a symptom label or event note for storage. */ +export function encryptSymptomText(plaintext: string): Uint8Array { + return encryptToBytes(plaintext); +} + +/** Decrypt a stored label or note; null on a missing or unreadable value. */ +export function decryptSymptomText( + buf: Uint8Array | null, + field: "label" | "note", +): string | null { + if (!buf || buf.byteLength === 0) return null; + try { + return decryptFromBytes(buf); + } catch (err) { + getEvent()?.addWarning( + `symptom ${field} decrypt failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return null; + } +} + +/** Header summary of one definition over the trailing window. */ +export interface SymptomRecentSummary { + count30d: number; + maxIntensity30d: number | null; + lastOccurredAt: string | null; +} + +const EMPTY_RECENT: SymptomRecentSummary = { + count30d: 0, + maxIntensity30d: null, + lastOccurredAt: null, +}; + +export function toSymptomDefinitionDTO( + row: Pick< + SymptomDefinition, + "id" | "labelEncrypted" | "icon" | "sortOrder" | "isActive" + >, + recent: SymptomRecentSummary = EMPTY_RECENT, +): SymptomDefinitionDTO { + return { + id: row.id, + label: decryptSymptomText(row.labelEncrypted, "label"), + icon: row.icon, + sortOrder: row.sortOrder, + isActive: row.isActive, + recent, + }; +} + +export function toSymptomEventDTO(row: SymptomEvent): SymptomEventDTO { + return { + id: row.id, + definitionId: row.definitionId, + occurredAt: row.occurredAt.toISOString(), + intensity: row.intensity, + note: decryptSymptomText(row.noteEncrypted, "note"), + episodeId: row.episodeId, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + }; +} + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** + * Every definition of the account with its 30-day summary, ordered for the + * chip row. One grouped aggregate for the counts and maxima, one for the last + * occurrence over all time (a symptom last felt two months ago still says so). + */ +export async function listSymptomDefinitions( + userId: string, + options: { includeHidden: boolean; now?: Date }, +): Promise { + const now = options.now ?? new Date(); + const since = new Date(now.getTime() - 30 * DAY_MS); + const [rows, recent, last] = await Promise.all([ + prisma.symptomDefinition.findMany({ + where: { userId, ...(options.includeHidden ? {} : { isActive: true }) }, + orderBy: [{ sortOrder: "asc" }, { createdAt: "asc" }, { id: "asc" }], + select: { + id: true, + labelEncrypted: true, + icon: true, + sortOrder: true, + isActive: true, + }, + }), + prisma.symptomEvent.groupBy({ + by: ["definitionId"], + where: { userId, occurredAt: { gte: since, lte: now } }, + _count: { _all: true }, + _max: { intensity: true }, + }), + prisma.symptomEvent.groupBy({ + by: ["definitionId"], + where: { userId }, + _max: { occurredAt: true }, + }), + ]); + const recentById = new Map(recent.map((r) => [r.definitionId, r])); + const lastById = new Map(last.map((r) => [r.definitionId, r._max])); + return rows.map((row) => { + const r = recentById.get(row.id); + const lastAt = lastById.get(row.id)?.occurredAt ?? null; + return toSymptomDefinitionDTO(row, { + count30d: r?._count._all ?? 0, + maxIntensity30d: r?._max.intensity ?? null, + lastOccurredAt: lastAt ? lastAt.toISOString() : null, + }); + }); +} + +/** + * Whether `episodeId` names an episode an event of `userId` may be filed + * against: the caller's own, not deleted, and still open. A resolved episode + * is history; filing a new occurrence into it would rewrite what the person + * recorded about how it ended. + */ +export async function isLinkableEpisode( + userId: string, + episodeId: string, +): Promise { + const episode = await prisma.illnessEpisode.findFirst({ + where: { id: episodeId, userId, deletedAt: null, resolvedAt: null }, + select: { id: true }, + }); + return episode !== null; +} diff --git a/src/lib/symptoms/shared.ts b/src/lib/symptoms/shared.ts new file mode 100644 index 000000000..60e9b17b7 --- /dev/null +++ b/src/lib/symptoms/shared.ts @@ -0,0 +1,154 @@ +import { z } from "zod/v4"; + +import { CUSTOM_SYMPTOM_ICON_ALLOWLIST } from "@/lib/cycle/custom-symptoms-shared"; +import { validateEntryInstant } from "@/lib/validations/entry-instant"; + +/** + * Person-defined symptoms (v1.40): client-safe constants, wire types and the + * request schemas. Server-only helpers (label crypto, DTO mappers) live in + * `./server.ts`, so the quick-entry sheet can import this file without pulling + * `node:crypto` into the browser bundle. + * + * A definition is the person's own name for a symptom ("aura", "headache"); an + * event is one occurrence of it with a 0-10 intensity, the same numeric rating + * scale the pain score uses. Both ride the `illness` module. + */ + +/** + * Active definitions per account. The same eight the custom-metric channels + * are capped at, and for the same reason: every active definition is its own + * correlation channel, so the cap bounds the discovery matrix as well as the + * chip row. A hidden definition does not count. + */ +export const MAX_ACTIVE_SYMPTOM_DEFINITIONS = 8; + +/** The intensity scale: 0 none, 10 the worst imaginable (NRS). */ +export const SYMPTOM_INTENSITY_MIN = 0; +export const SYMPTOM_INTENSITY_MAX = 10; + +/** Upper bound on one events page. */ +export const MAX_SYMPTOM_EVENTS_PAGE = 500; + +/** + * Icon names a definition may carry: the cycle custom-symptom allowlist, which + * the iOS client already maps to SF Symbols. + */ +export const SYMPTOM_ICON_ALLOWLIST = CUSTOM_SYMPTOM_ICON_ALLOWLIST; +export type SymptomIconName = (typeof SYMPTOM_ICON_ALLOWLIST)[number]; + +const ICON_SET = new Set(SYMPTOM_ICON_ALLOWLIST); + +/** Correlation channel key prefix: `SYMPTOM:`. */ +export const SYMPTOM_CHANNEL_PREFIX = "SYMPTOM:"; + +/** Wire shape of one definition. */ +export interface SymptomDefinitionDTO { + id: string; + /** Decrypted; null only when the stored label cannot be read. */ + label: string | null; + icon: string | null; + sortOrder: number; + isActive: boolean; + /** Server-computed over the last 30 days, for the section header line. */ + recent: { + count30d: number; + maxIntensity30d: number | null; + lastOccurredAt: string | null; + }; +} + +/** Wire shape of one event. */ +export interface SymptomEventDTO { + id: string; + definitionId: string; + occurredAt: string; + intensity: number; + note: string | null; + episodeId: string | null; + createdAt: string; + updatedAt: string; +} + +const labelSchema = z + .string() + .trim() + .min(1, "Label must not be empty") + .max(40, "Label must be at most 40 characters"); + +const iconSchema = z + .string() + .refine((v) => ICON_SET.has(v), "Unknown icon") + .nullish(); + +export const symptomDefinitionCreateSchema = z.object({ + label: labelSchema, + icon: iconSchema, +}); + +export const symptomDefinitionUpdateSchema = z + .object({ + label: labelSchema.optional(), + icon: iconSchema, + isActive: z.boolean().optional(), + sortOrder: z.number().int().min(0).max(1000).optional(), + }) + .refine( + (v) => + v.label !== undefined || + v.icon !== undefined || + v.isActive !== undefined || + v.sortOrder !== undefined, + "At least one field is required", + ); + +const intensitySchema = z + .number() + .int() + .min(SYMPTOM_INTENSITY_MIN) + .max(SYMPTOM_INTENSITY_MAX); + +const occurredAtSchema = validateEntryInstant( + z.iso + .datetime({ offset: true }) + .transform((s) => new Date(s)) + .pipe(z.date()), +); + +export const symptomEventCreateSchema = z.object({ + definitionId: z.string().min(1).max(40), + /** Defaults to now. Not in the future beyond the shared 5-minute skew. */ + occurredAt: occurredAtSchema.optional(), + intensity: intensitySchema, + note: z.string().trim().max(500).nullish(), + /** Must be one of the caller's own episodes that is still open. */ + episodeId: z.string().min(1).max(40).nullish(), +}); + +export const symptomEventUpdateSchema = z + .object({ + occurredAt: occurredAtSchema.optional(), + intensity: intensitySchema.optional(), + note: z.string().trim().max(500).nullish(), + episodeId: z.string().min(1).max(40).nullish(), + }) + .refine( + (v) => + v.occurredAt !== undefined || + v.intensity !== undefined || + v.note !== undefined || + v.episodeId !== undefined, + "At least one field is required", + ); + +export const symptomEventListQuerySchema = z.object({ + from: z.iso.datetime({ offset: true }).optional(), + to: z.iso.datetime({ offset: true }).optional(), + definitionId: z.string().min(1).max(40).optional(), + episodeId: z.string().min(1).max(40).optional(), + limit: z.coerce + .number() + .int() + .min(1) + .max(MAX_SYMPTOM_EVENTS_PAGE) + .default(200), +}); diff --git a/src/lib/validations/backup.ts b/src/lib/validations/backup.ts index 0613a290d..166373083 100644 --- a/src/lib/validations/backup.ts +++ b/src/lib/validations/backup.ts @@ -1102,6 +1102,44 @@ const illnessEpisodeBackupSchema = z }) .passthrough(); +/** + * A person-defined symptom (v1.40) with every occurrence of it nested inside, + * the way a medication carries its side effects. `label` / `labelEncrypted` + * and `note` / `noteEncrypted` are the two ends of the free-text contract: a + * portable file carries the plaintext, a disaster-recovery file the ciphertext. + * An occurrence's `episodeId` is the exported episode's own id; the restore + * nulls one that names an episode the file does not carry. + */ +const symptomEventBackupSchema = z + .object({ + id: z.string().min(1).optional(), + occurredAt: isoDateTime, + intensity: z.number().int().min(0).max(10), + note: z.string().nullable().optional(), + noteEncrypted: base64BytesSchema.nullable().optional(), + episodeId: z.string().min(1).nullable().optional(), + createdAt: isoDateTime, + updatedAt: isoDateTime, + }) + .passthrough(); + +const symptomDefinitionBackupSchema = z + .object({ + id: z.string().min(1).optional(), + label: z.string().min(1).optional(), + labelEncrypted: base64BytesSchema.optional(), + icon: z.string().nullable().optional(), + sortOrder: z.number().int(), + isActive: z.boolean(), + createdAt: isoDateTime, + updatedAt: isoDateTime, + events: z.array(symptomEventBackupSchema).default([]), + }) + .passthrough() + .refine((d) => d.label !== undefined || d.labelEncrypted !== undefined, { + message: "A symptom definition needs a label or its ciphertext", + }); + /** * A practitioner, an encounter, and the edges between an encounter and the * things it produced. @@ -1853,6 +1891,8 @@ export const backupPayloadSchema = z labResults: z.array(labResultBackupSchema).default([]), biomarkers: z.array(biomarkerBackupSchema).default([]), illnessEpisodes: z.array(illnessEpisodeBackupSchema).default([]), + // Defaulted: a file written before v1.40 carries no key. + symptomDefinitions: z.array(symptomDefinitionBackupSchema).default([]), allergies: z.array(allergyBackupSchema).default([]), familyHistory: z.array(familyHistoryBackupSchema).default([]), workouts: z.array(workoutBackupSchema).default([]), diff --git a/tests/fixtures/v137/sharing-matrix.ts b/tests/fixtures/v137/sharing-matrix.ts index e90d4748e..635a5c4aa 100644 --- a/tests/fixtures/v137/sharing-matrix.ts +++ b/tests/fixtures/v137/sharing-matrix.ts @@ -705,4 +705,52 @@ export const ADMITTED_MUTATING_HANDLERS = [ "manage", "export", ), + handler( + "app/api/symptoms/definitions/route.ts", + "/api/symptoms/definitions", + "POST", + "illness", + "manage", + "create", + ), + handler( + "app/api/symptoms/definitions/[id]/route.ts", + "/api/symptoms/definitions/[id]", + "PATCH", + "illness", + "manage", + "update", + ), + handler( + "app/api/symptoms/definitions/[id]/route.ts", + "/api/symptoms/definitions/[id]", + "DELETE", + "illness", + "manage", + "delete", + ), + handler( + "app/api/symptoms/events/route.ts", + "/api/symptoms/events", + "POST", + "illness", + "write", + "create", + ), + handler( + "app/api/symptoms/events/[id]/route.ts", + "/api/symptoms/events/[id]", + "PATCH", + "illness", + "manage", + "update", + ), + handler( + "app/api/symptoms/events/[id]/route.ts", + "/api/symptoms/events/[id]", + "DELETE", + "illness", + "manage", + "delete", + ), ] as const satisfies readonly MutatingHandlerContract[]; diff --git a/tests/integration/backup-round-trip.test.ts b/tests/integration/backup-round-trip.test.ts index 2ae7458d8..58d0e9500 100644 --- a/tests/integration/backup-round-trip.test.ts +++ b/tests/integration/backup-round-trip.test.ts @@ -232,6 +232,9 @@ const COUNT_BACK: Record< p.familyHistoryEntry.count({ where: { userId } }), IllnessEpisode: (p, userId) => p.illnessEpisode.count({ where: { userId } }), IllnessDayLog: (p, userId) => p.illnessDayLog.count({ where: { userId } }), + SymptomDefinition: (p, userId) => + p.symptomDefinition.count({ where: { userId } }), + SymptomEvent: (p, userId) => p.symptomEvent.count({ where: { userId } }), IllnessSymptomLink: (p, userId) => p.illnessSymptomLink.count({ where: { dayLog: { userId } } }), UserHealthProfile: (p, userId) => @@ -750,6 +753,52 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise { }, }); + // v1.40 — two of the person's own symptoms and three occurrences: one filed + // against the open cold, one with a note, one on its own in the gap. + const aura = await prisma.symptomDefinition.create({ + data: { + userId: OWNER_ID, + labelEncrypted: encryptToBytes("Aura"), + icon: "Zap", + sortOrder: 0, + }, + }); + const headache = await prisma.symptomDefinition.create({ + data: { + userId: OWNER_ID, + labelEncrypted: encryptToBytes("Headache"), + icon: "Brain", + sortOrder: 1, + isActive: false, + }, + }); + await prisma.symptomEvent.create({ + data: { + userId: OWNER_ID, + definitionId: aura.id, + occurredAt: AT("2026-06-21T09:00:00.000Z"), + intensity: 4, + episodeId: episode.id, + }, + }); + await prisma.symptomEvent.create({ + data: { + userId: OWNER_ID, + definitionId: headache.id, + occurredAt: AT("2026-06-21T11:00:00.000Z"), + intensity: 8, + noteEncrypted: encryptToBytes("left side, behind the eye"), + }, + }); + await prisma.symptomEvent.create({ + data: { + userId: OWNER_ID, + definitionId: aura.id, + occurredAt: AT("2026-07-10T18:30:00.000Z"), + intensity: 2, + }, + }); + await prisma.userHealthProfile.create({ data: { userId: OWNER_ID, @@ -2108,6 +2157,69 @@ describe("every model the plan claims two-ended survives a real restore", () => "a condition's body site and side must survive the round trip", ).toEqual({ bodySite: "Stomach", laterality: "BOTH" }); + // v1.40 — the person's own symptoms: the names decrypt to what was typed, + // each occurrence keeps its time, intensity and note, and the one filed + // against the cold still points at the cold. + const restoredSymptoms = await prisma.symptomDefinition.findMany({ + where: { userId: OWNER_ID }, + orderBy: { sortOrder: "asc" }, + include: { + events: { + orderBy: { occurredAt: "asc" }, + include: { episode: { select: { label: true } } }, + }, + }, + }); + expect( + restoredSymptoms.map((definition) => ({ + label: decryptFromBytes(definition.labelEncrypted), + icon: definition.icon, + isActive: definition.isActive, + events: definition.events.map((event) => ({ + at: event.occurredAt.toISOString(), + intensity: event.intensity, + note: event.noteEncrypted + ? decryptFromBytes(event.noteEncrypted) + : null, + episode: event.episode?.label ?? null, + })), + })), + "symptom definitions and their occurrences must survive the round trip", + ).toEqual([ + { + label: "Aura", + icon: "Zap", + isActive: true, + events: [ + { + at: "2026-06-21T09:00:00.000Z", + intensity: 4, + note: null, + episode: "Cold", + }, + { + at: "2026-07-10T18:30:00.000Z", + intensity: 2, + note: null, + episode: null, + }, + ], + }, + { + label: "Headache", + icon: "Brain", + isActive: false, + events: [ + { + at: "2026-06-21T11:00:00.000Z", + intensity: 8, + note: "left side, behind the eye", + episode: null, + }, + ], + }, + ]); + // The staged fact, and the decision on it. // // This is the assertion the count cannot make. `status`, `needsReview`, diff --git a/tests/integration/sharing-manage-handler-matrix.test.ts b/tests/integration/sharing-manage-handler-matrix.test.ts index cb4fea195..24ac022ef 100644 --- a/tests/integration/sharing-manage-handler-matrix.test.ts +++ b/tests/integration/sharing-manage-handler-matrix.test.ts @@ -37,6 +37,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { NextRequest } from "next/server"; +import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { ADMITTED_MUTATING_HANDLERS, type MutatingHandlerContract, @@ -281,7 +282,7 @@ const ROUTE_IMPORTS = import.meta.glob>( describe("the complete MANAGE handler matrix", () => { it("starts with a non-empty, exact handler inventory", () => { - expect(ADMITTED_MUTATING_HANDLERS.length).toBe(77); + expect(ADMITTED_MUTATING_HANDLERS.length).toBe(83); }); }); @@ -1331,6 +1332,178 @@ manageContract("POST /api/illness/episodes/[id]/restore", { )?.deletedAt === null, }); +async function makeSymptomDefinition(userId: string) { + return getPrismaClient().symptomDefinition.create({ + data: { + userId, + labelEncrypted: encryptToBytes("Aura"), + icon: "Zap", + }, + }); +} + +async function makeSymptomEvent(userId: string) { + const definition = await makeSymptomDefinition(userId); + return getPrismaClient().symptomEvent.create({ + data: { + userId, + definitionId: definition.id, + occurredAt: new Date(ISO), + intensity: 6, + }, + }); +} + +manageContract("POST /api/symptoms/definitions", { + contract: ADMITTED_MUTATING_HANDLERS.find( + (entry) => + entry.route === "/api/symptoms/definitions" && entry.action === "POST", + )!, + prepare: async (ownerId) => ({ ownerId }), + act: async () => { + const { POST } = await import("@/app/api/symptoms/definitions/route"); + return call(POST as Handler, "POST", "/api/symptoms/definitions", { + label: "Aura", + icon: "Zap", + }); + }, + ok: 201, + auditAction: "symptoms.definition.create", + applied: async ({ ownerId }) => + (await getPrismaClient().symptomDefinition.count({ + where: { userId: ownerId }, + })) === 1, +}); + +manageContract("PATCH /api/symptoms/definitions/[id]", { + contract: ADMITTED_MUTATING_HANDLERS.find( + (entry) => + entry.route === "/api/symptoms/definitions/[id]" && + entry.action === "PATCH", + )!, + prepare: (ownerId) => makeSymptomDefinition(ownerId), + act: async (row) => { + const { PATCH } = await import("@/app/api/symptoms/definitions/[id]/route"); + return call( + PATCH as Handler, + "PATCH", + `/api/symptoms/definitions/${row.id}`, + { icon: "Brain" }, + { id: row.id }, + ); + }, + ok: 200, + auditAction: "symptoms.definition.update", + applied: async (row) => + ( + await getPrismaClient().symptomDefinition.findUnique({ + where: { id: row.id }, + }) + )?.icon === "Brain", +}); + +manageContract("DELETE /api/symptoms/definitions/[id]", { + contract: ADMITTED_MUTATING_HANDLERS.find( + (entry) => + entry.route === "/api/symptoms/definitions/[id]" && + entry.action === "DELETE", + )!, + prepare: (ownerId) => makeSymptomDefinition(ownerId), + act: async (row) => { + const { DELETE } = + await import("@/app/api/symptoms/definitions/[id]/route"); + return call( + DELETE as Handler, + "DELETE", + `/api/symptoms/definitions/${row.id}`, + undefined, + { id: row.id }, + ); + }, + ok: 200, + auditAction: "symptoms.definition.delete", + applied: async (row) => + ( + await getPrismaClient().symptomDefinition.findUnique({ + where: { id: row.id }, + }) + )?.isActive === false, +}); + +writeEffectContract("POST /api/symptoms/events", { + contract: ADMITTED_MUTATING_HANDLERS.find( + (entry) => + entry.route === "/api/symptoms/events" && entry.action === "POST", + )!, + prepare: async (ownerId) => ({ + ownerId, + definition: await makeSymptomDefinition(ownerId), + }), + act: async ({ definition }) => { + const { POST } = await import("@/app/api/symptoms/events/route"); + return call(POST as Handler, "POST", "/api/symptoms/events", { + definitionId: definition.id, + intensity: 7, + }); + }, + ok: 201, + auditAction: "symptoms.event.create", + applied: async ({ ownerId }) => + (await getPrismaClient().symptomEvent.count({ + where: { userId: ownerId }, + })) === 1, +}); + +manageContract("PATCH /api/symptoms/events/[id]", { + contract: ADMITTED_MUTATING_HANDLERS.find( + (entry) => + entry.route === "/api/symptoms/events/[id]" && entry.action === "PATCH", + )!, + prepare: (ownerId) => makeSymptomEvent(ownerId), + act: async (row) => { + const { PATCH } = await import("@/app/api/symptoms/events/[id]/route"); + return call( + PATCH as Handler, + "PATCH", + `/api/symptoms/events/${row.id}`, + { intensity: 9 }, + { id: row.id }, + ); + }, + ok: 200, + auditAction: "symptoms.event.update", + applied: async (row) => + ( + await getPrismaClient().symptomEvent.findUnique({ + where: { id: row.id }, + }) + )?.intensity === 9, +}); + +manageContract("DELETE /api/symptoms/events/[id]", { + contract: ADMITTED_MUTATING_HANDLERS.find( + (entry) => + entry.route === "/api/symptoms/events/[id]" && entry.action === "DELETE", + )!, + prepare: (ownerId) => makeSymptomEvent(ownerId), + act: async (row) => { + const { DELETE } = await import("@/app/api/symptoms/events/[id]/route"); + return call( + DELETE as Handler, + "DELETE", + `/api/symptoms/events/${row.id}`, + undefined, + { id: row.id }, + ); + }, + ok: 200, + auditAction: "symptoms.event.delete", + applied: async (row) => + (await getPrismaClient().symptomEvent.findUnique({ + where: { id: row.id }, + })) === null, +}); + writeEffectContract("POST /api/encounters", { contract: ADMITTED_MUTATING_HANDLERS.find( (entry) => entry.route === "/api/encounters" && entry.action === "POST", @@ -2580,13 +2753,13 @@ describe("the conditions the admissions were granted on", () => { describe("the complete MANAGE handler matrix", () => { it("registers one strict actor-and-effect driver for every admission", () => { const expected = ADMITTED_MUTATING_HANDLERS.map(matrixKey).sort(); - expect(STRICT_DRIVER_KEYS.size).toBe(77); + expect(STRICT_DRIVER_KEYS.size).toBe(83); expect([...STRICT_DRIVER_KEYS].sort()).toEqual(expected); }); it("executes every registered driver through its owned effect and actor audit", () => { const expected = ADMITTED_MUTATING_HANDLERS.map(matrixKey).sort(); - expect(REAL_EFFECT_KEYS.size).toBe(77); + expect(REAL_EFFECT_KEYS.size).toBe(83); expect([...REAL_EFFECT_KEYS].sort()).toEqual(expected); }); }); From 9d9e1f153ca2a080e472b2150906458f8b9279a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 3 Oct 2026 21:14:08 +0200 Subject: [PATCH 05/14] feat(insights): make each person-defined symptom its own correlation channel Every active symptom definition enters pattern discovery as an outcome channel SYMPTOM:, labelled with its decrypted name (the data label the interpretation already prefers over the key). The daily value is the day's highest intensity in the person's zone; days without an occurrence count as 0 only between the first and the last logged day. At most eight, the active-definition cap. The SYMPTOM: prefix resolves to the illness module in the surface map, so the family leaves the matrix with the module, and the discovery guard lists the new fetcher. --- src/__tests__/discovery-matrix-guard.test.ts | 1 + .../correlations/__tests__/route.test.ts | 2 + src/app/api/insights/correlations/route.ts | 1 + .../coach/__tests__/correlations-read.test.ts | 1 + .../discovery-matrix-modules.test.ts | 5 + .../metric-correlation-context.test.ts | 1 + .../__tests__/symptom-event-series.test.ts | 72 +++ .../insights/correlation-channel-series.ts | 60 ++ .../insights/correlation-series-builders.ts | 51 ++ .../__tests__/coach-read-locale.test.ts | 1 + src/lib/insights/discovery-matrix.ts | 11 + .../__tests__/period-narrative-db.test.ts | 1 + .../__tests__/surface-module-registry.test.ts | 7 + src/lib/modules/surface.ts | 11 +- tests/integration/symptoms.test.ts | 578 ++++++++++++++++++ 15 files changed, 799 insertions(+), 4 deletions(-) create mode 100644 src/lib/insights/__tests__/symptom-event-series.test.ts create mode 100644 tests/integration/symptoms.test.ts diff --git a/src/__tests__/discovery-matrix-guard.test.ts b/src/__tests__/discovery-matrix-guard.test.ts index ca4d3c9f2..128b5e8ac 100644 --- a/src/__tests__/discovery-matrix-guard.test.ts +++ b/src/__tests__/discovery-matrix-guard.test.ts @@ -91,6 +91,7 @@ describe("T1 — only the assembler fetches discovery channels", () => { "fetchSymptomSeries", "fetchEnvironmentSeries", "fetchCustomMetricBehaviourSeries", + "fetchSymptomEventSeries", "fetchMoodWindowSeries", "fetchMoodFactorWindowSeries", "fetchMeasurementDailySeriesTiered", diff --git a/src/app/api/insights/correlations/__tests__/route.test.ts b/src/app/api/insights/correlations/__tests__/route.test.ts index 47fa6a4a2..42413f24f 100644 --- a/src/app/api/insights/correlations/__tests__/route.test.ts +++ b/src/app/api/insights/correlations/__tests__/route.test.ts @@ -31,6 +31,7 @@ vi.mock("@/lib/db", () => ({ // channel produces a point. environmentContext: { findMany: vi.fn().mockResolvedValue([]) }, customMetric: { findMany: vi.fn().mockResolvedValue([]) }, + symptomDefinition: { findMany: vi.fn().mockResolvedValue([]) }, }, })); @@ -153,6 +154,7 @@ beforeEach(() => { prisma.environmentContext.findMany as ReturnType ).mockResolvedValue([]); vi.mocked(prisma.customMetric.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.symptomDefinition.findMany).mockResolvedValue([] as never); }); const callGet = GET as unknown as (req: NextRequest) => Promise; diff --git a/src/app/api/insights/correlations/route.ts b/src/app/api/insights/correlations/route.ts index f93c5262f..c94021409 100644 --- a/src/app/api/insights/correlations/route.ts +++ b/src/app/api/insights/correlations/route.ts @@ -236,6 +236,7 @@ async function buildCorrelationsResponse( // available for the scan. environment_days: diagnostics.environmentDays, custom_metric_channels: diagnostics.customMetricChannels, + symptom_event_channels: diagnostics.symptomEventChannels, custom_metric_days: diagnostics.customMetricDays, }, }); diff --git a/src/lib/ai/coach/__tests__/correlations-read.test.ts b/src/lib/ai/coach/__tests__/correlations-read.test.ts index f020b98c1..088f0677e 100644 --- a/src/lib/ai/coach/__tests__/correlations-read.test.ts +++ b/src/lib/ai/coach/__tests__/correlations-read.test.ts @@ -39,6 +39,7 @@ vi.mock("@/lib/db", () => ({ findMany: (a: unknown) => environmentContextFindMany(a), }, customMetric: { findMany: (a: unknown) => customMetricFindMany(a) }, + symptomDefinition: { findMany: async () => [] }, }, })); diff --git a/src/lib/insights/__tests__/discovery-matrix-modules.test.ts b/src/lib/insights/__tests__/discovery-matrix-modules.test.ts index e13d77393..930b52b27 100644 --- a/src/lib/insights/__tests__/discovery-matrix-modules.test.ts +++ b/src/lib/insights/__tests__/discovery-matrix-modules.test.ts @@ -42,6 +42,9 @@ vi.mock("@/lib/insights/correlation-channel-series", () => ({ fetchCustomMetricBehaviourSeries: vi.fn(async () => [ { key: "CUSTOM_METRIC:abc", role: "behaviour", points: pts }, ]), + fetchSymptomEventSeries: vi.fn(async () => [ + { key: "SYMPTOM:def1", label: "Aura", role: "outcome", points: pts }, + ]), })); vi.mock("@/lib/rollups/measurement-read", () => ({ loadUserSourcePriority: vi.fn(), @@ -77,6 +80,7 @@ describe("assembleDiscoveryMatrix — switched-off modules", () => { "SYMPTOM_SEVERITY", "ENV_TEMP_MEAN", "CUSTOM_METRIC:abc", + "SYMPTOM:def1", "WEIGHT", ]) { expect(keys).toContain(key); @@ -103,6 +107,7 @@ describe("assembleDiscoveryMatrix — switched-off modules", () => { "BLOOD_GLUCOSE", "MEDICATION_COMPLIANCE", "SYMPTOM_SEVERITY", + "SYMPTOM:def1", "ENV_TEMP_MEAN", ]) { expect(keys, gone).not.toContain(gone); diff --git a/src/lib/insights/__tests__/metric-correlation-context.test.ts b/src/lib/insights/__tests__/metric-correlation-context.test.ts index f77052076..07927a927 100644 --- a/src/lib/insights/__tests__/metric-correlation-context.test.ts +++ b/src/lib/insights/__tests__/metric-correlation-context.test.ts @@ -11,6 +11,7 @@ vi.mock("@/lib/db", () => ({ illnessDayLog: { findMany: vi.fn() }, environmentContext: { findMany: vi.fn() }, customMetric: { findMany: vi.fn() }, + symptomDefinition: { findMany: vi.fn(async () => []) }, }, })); diff --git a/src/lib/insights/__tests__/symptom-event-series.test.ts b/src/lib/insights/__tests__/symptom-event-series.test.ts new file mode 100644 index 000000000..17c7b9f3a --- /dev/null +++ b/src/lib/insights/__tests__/symptom-event-series.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; + +import { buildSymptomEventDailySeries } from "@/lib/insights/correlation-series-builders"; +import { maskSeriesByModules } from "@/lib/insights/discovery-matrix"; + +const at = (iso: string) => new Date(iso); + +describe("buildSymptomEventDailySeries", () => { + it("takes the day's highest intensity and zero-fills only between the first and last day", () => { + const series = buildSymptomEventDailySeries({ + key: "SYMPTOM:def1", + label: "Aura", + tz: "Europe/Berlin", + events: [ + { at: at("2026-07-01T07:00:00.000Z"), intensity: 3 }, + { at: at("2026-07-01T15:00:00.000Z"), intensity: 7 }, + { at: at("2026-07-04T10:00:00.000Z"), intensity: 2 }, + ], + }); + expect(series).toEqual({ + key: "SYMPTOM:def1", + label: "Aura", + role: "outcome", + points: [ + { day: "2026-07-01", value: 7 }, + { day: "2026-07-02", value: 0 }, + { day: "2026-07-03", value: 0 }, + { day: "2026-07-04", value: 2 }, + ], + }); + }); + + it("keys a late-evening occurrence to the person's own day", () => { + const series = buildSymptomEventDailySeries({ + key: "SYMPTOM:def1", + label: "Aura", + tz: "Europe/Berlin", + // 23:30 UTC on the 1st is 01:30 on the 2nd in Berlin (summer time). + events: [{ at: at("2026-07-01T23:30:00.000Z"), intensity: 5 }], + }); + expect(series.points).toEqual([{ day: "2026-07-02", value: 5 }]); + }); + + it("is empty with no occurrences, so the channel drops out", () => { + const series = buildSymptomEventDailySeries({ + key: "SYMPTOM:def1", + label: "Aura", + tz: "UTC", + events: [], + }); + expect(series.points).toEqual([]); + }); +}); + +describe("the SYMPTOM: channel family follows the illness module", () => { + const series = [ + { key: "SYMPTOM:def1", role: "outcome" as const, points: [] }, + { key: "SLEEP_DURATION", role: "behaviour" as const, points: [] }, + ]; + + it("is masked out when the illness module is off", () => { + expect( + maskSeriesByModules(series, { illness: false }).map((s) => s.key), + ).toEqual(["SLEEP_DURATION"]); + }); + + it("stays when the module is on", () => { + expect( + maskSeriesByModules(series, { illness: true }).map((s) => s.key), + ).toEqual(["SYMPTOM:def1", "SLEEP_DURATION"]); + }); +}); diff --git a/src/lib/insights/correlation-channel-series.ts b/src/lib/insights/correlation-channel-series.ts index aaed68cfa..b0278790d 100644 --- a/src/lib/insights/correlation-channel-series.ts +++ b/src/lib/insights/correlation-channel-series.ts @@ -39,6 +39,7 @@ import { wallClockInTz } from "@/lib/tz/wall-clock"; import { ENVIRONMENT_FIELDS } from "@/lib/environment/fields"; import { buildComplianceDailySeries, + buildSymptomEventDailySeries, buildSymptomSeverityDailySeries, type SymptomDayLogRow, type SymptomEpisodeSpan, @@ -61,6 +62,11 @@ import type { } from "@/generated/prisma/client"; import { TRACKED_INTAKE_WHERE } from "@/lib/medications/intake-tracking"; import { dateOnlyKey } from "@/lib/tz/date-only"; +import { decryptSymptomText } from "@/lib/symptoms/server"; +import { + MAX_ACTIVE_SYMPTOM_DEFINITIONS, + SYMPTOM_CHANNEL_PREFIX, +} from "@/lib/symptoms/shared"; /** * v1.21.0 (FDREXTEND) — build the user's MEDICATION_COMPLIANCE daily series. @@ -315,6 +321,60 @@ export async function fetchCustomMetricBehaviourSeries( })); } +/** + * Ceiling on `SYMPTOM:` channels. The definition cap already holds the + * active set at eight; restated here so the matrix bound does not depend on a + * route check staying where it is. + */ +export const MAX_SYMPTOM_CORRELATION_CHANNELS = MAX_ACTIVE_SYMPTOM_DEFINITIONS; + +/** + * v1.40 — one OUTCOME series per active person-defined symptom, keyed + * `SYMPTOM:` and labelled with the decrypted name (the data + * label `interpret()` prefers over the key). Daily value = that day's highest + * intensity; see `buildSymptomEventDailySeries` for the zero-fill rule. Hidden + * definitions are not channels. A symptom whose name cannot be decrypted is + * left out rather than shown under its database id. + */ +export async function fetchSymptomEventSeries( + userId: string, + tz: string, + since: Date, +): Promise { + const definitions = await prisma.symptomDefinition.findMany({ + where: { userId, isActive: true }, + orderBy: [{ sortOrder: "asc" }, { createdAt: "asc" }, { id: "asc" }], + take: MAX_SYMPTOM_CORRELATION_CHANNELS, + select: { + id: true, + labelEncrypted: true, + events: { + where: { occurredAt: { gte: since } }, + orderBy: [{ occurredAt: "desc" }, { id: "desc" }], + take: 5000, + select: { occurredAt: true, intensity: true }, + }, + }, + }); + const series: NamedSeries[] = []; + for (const definition of definitions) { + const label = decryptSymptomText(definition.labelEncrypted, "label"); + if (label === null) continue; + series.push( + buildSymptomEventDailySeries({ + key: `${SYMPTOM_CHANNEL_PREFIX}${definition.id}`, + label, + events: definition.events.map((event) => ({ + at: event.occurredAt, + intensity: event.intensity, + })), + tz, + }), + ); + } + return series; +} + /** Day key (YYYY-MM-DD) for an instant in the user's display timezone. */ function tzDayKey(at: Date, tz: string): string { const { year, month, day } = wallClockInTz(at, tz); diff --git a/src/lib/insights/correlation-series-builders.ts b/src/lib/insights/correlation-series-builders.ts index b387fa643..7da33628e 100644 --- a/src/lib/insights/correlation-series-builders.ts +++ b/src/lib/insights/correlation-series-builders.ts @@ -35,6 +35,7 @@ * mirroring how the engine itself is pure over `NamedSeries[]`. */ import { dayKeyForUserTz } from "@/lib/measurements/consolidation-tz"; +import { shiftDateKey } from "@/lib/tz/format"; import type { DoseHistoryRow } from "@/lib/medications/scheduling/dose-history"; import { MEDICATION_COMPLIANCE_CHANNEL_KEY, @@ -191,3 +192,53 @@ export function buildSymptomSeverityDailySeries(args: { return { key: SYMPTOM_SEVERITY_CHANNEL_KEY, role, points }; } + +/** One occurrence of a person-defined symptom, as the builder reads it. */ +export interface SymptomEventRow { + at: Date; + intensity: number; +} + +/** + * Build one `SYMPTOM:` outcome series from that symptom's + * occurrences (v1.40). + * + * The daily value is the day's HIGHEST intensity (0-10) in the person's zone: + * two attacks in a day are as bad as the worse one. Days with no occurrence + * are 0, but only between the first and the last logged day: inside that span + * the person was tracking the symptom and its absence is a real zero, outside + * it nothing says whether they were tracking at all. One logged day yields a + * one-point series, which the n >= 20 floor then drops. Pure. + */ +export function buildSymptomEventDailySeries(args: { + key: string; + label: string | undefined; + events: SymptomEventRow[]; + tz: string; +}): NamedSeries { + const { key, label, events, tz } = args; + const byDay = new Map(); + for (const event of events) { + if (!Number.isFinite(event.intensity)) continue; + const day = dayKey(event.at, tz); + const prev = byDay.get(day); + byDay.set( + day, + prev == null ? event.intensity : Math.max(prev, event.intensity), + ); + } + const keys = [...byDay.keys()].sort(); + const points: DailySeriesPoint[] = []; + if (keys.length > 0) { + const last = keys[keys.length - 1]; + for (let day = keys[0]; day <= last; day = shiftDateKey(day, 1)) { + points.push({ day, value: byDay.get(day) ?? 0 }); + } + } + return { + key, + ...(label !== undefined ? { label } : {}), + role: "outcome", + points, + }; +} diff --git a/src/lib/insights/derived/__tests__/coach-read-locale.test.ts b/src/lib/insights/derived/__tests__/coach-read-locale.test.ts index 1c2208411..ebaa47e75 100644 --- a/src/lib/insights/derived/__tests__/coach-read-locale.test.ts +++ b/src/lib/insights/derived/__tests__/coach-read-locale.test.ts @@ -52,6 +52,7 @@ vi.mock("@/lib/db", () => ({ findMany: (a: unknown) => environmentContextFindMany(a), }, customMetric: { findMany: (a: unknown) => customMetricFindMany(a) }, + symptomDefinition: { findMany: async () => [] }, }, })); diff --git a/src/lib/insights/discovery-matrix.ts b/src/lib/insights/discovery-matrix.ts index 7e86c64ea..56a207a9c 100644 --- a/src/lib/insights/discovery-matrix.ts +++ b/src/lib/insights/discovery-matrix.ts @@ -48,6 +48,7 @@ import { buildMeasurementDailySeries, fetchComplianceSeries, fetchCustomMetricBehaviourSeries, + fetchSymptomEventSeries, fetchEnvironmentSeries, fetchMeasurementDailySeriesTiered, fetchMeasurementWindowSeries, @@ -154,6 +155,8 @@ export interface DiscoveryMatrixDiagnostics { customMetricDays: number; /** `FACTOR:*` channels admitted (0 unless `includeMoodFactors`). */ moodFactorChannels: number; + /** `SYMPTOM:*` channels read (before the module mask). */ + symptomEventChannels: number; } export interface DiscoveryMatrix { @@ -294,6 +297,7 @@ export async function assembleDiscoveryMatrix( symptomSeries, environmentSeries, customMetricSeries, + symptomEventSeries, ] = await Promise.all([ readMeasurementDaily(userId, tz, since, measurementTypes, fetchMode), readMood(userId, tz, since, opts.includeMoodFactors === true), @@ -301,6 +305,7 @@ export async function assembleDiscoveryMatrix( fetchSymptomSeries(userId, tz, since), fetchEnvironmentSeries(userId, since), fetchCustomMetricBehaviourSeries(userId, tz, since), + fetchSymptomEventSeries(userId, tz, since), ]); const { factorSeries } = mood; @@ -331,6 +336,11 @@ export async function assembleDiscoveryMatrix( // (D → D+1) against every outcome above, under the unchanged n ≥ 20 / FDR / // effect-size gates, so a thin weather or custom series degrades to absent. series.push(...environmentSeries, ...customMetricSeries); + // The person's own symptoms are OUTCOMES: "short sleep, then an aura the + // next day" is the question they log them to answer. Each is its own + // channel and its own family, capped at eight; the illness module owns them + // through the `SYMPTOM:` prefix rule in the surface map. + series.push(...symptomEventSeries); // A factor is plausibly both a lag source ("rated work today → next-day // sleep") and a lag target ("steps today → next-day rated energy"), so it // enters as both roles. The engine skips self-pairs — every factor shares @@ -371,6 +381,7 @@ export async function assembleDiscoveryMatrix( 0, ), moodFactorChannels: factorSeries.size, + symptomEventChannels: symptomEventSeries.length, }, }; } diff --git a/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts b/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts index 29462480f..a2240617d 100644 --- a/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts +++ b/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts @@ -49,6 +49,7 @@ vi.mock("@/lib/db", () => ({ findMany: (a: unknown) => environmentContextFindMany(a), }, customMetric: { findMany: (a: unknown) => customMetricFindMany(a) }, + symptomDefinition: { findMany: async () => [] }, }, })); diff --git a/src/lib/modules/__tests__/surface-module-registry.test.ts b/src/lib/modules/__tests__/surface-module-registry.test.ts index e96b1325d..a48be16df 100644 --- a/src/lib/modules/__tests__/surface-module-registry.test.ts +++ b/src/lib/modules/__tests__/surface-module-registry.test.ts @@ -66,9 +66,11 @@ import { TYPE_TO_SUB_PAGE_SLUG, } from "@/lib/insights/sub-page-metric"; import { TREND_CHART_CONFIG } from "@/lib/insights/trend-chart-select"; +import { SYMPTOM_CHANNEL_PREFIX } from "@/lib/symptoms/shared"; import { SURFACE_KINDS, SURFACE_MODULE, + correlationChannelSurfaceId, surfaceModule, surfaceModulesOfKind, type SurfaceKind, @@ -167,6 +169,11 @@ describe("every surface id names a surface that exists", () => { // `correlationChannelSurfaceId` folds rated factors onto mood by this // prefix; it has to be the prefix the engine actually writes. expect(FACTOR_CHANNEL_PREFIX).toBe("FACTOR:"); + // Person-defined symptoms fold onto the illness symptom channel by theirs. + expect(SYMPTOM_CHANNEL_PREFIX).toBe("SYMPTOM:"); + expect(correlationChannelSurfaceId(`${SYMPTOM_CHANNEL_PREFIX}abc`)).toBe( + "correlation:SYMPTOM_SEVERITY", + ); }); }); diff --git a/src/lib/modules/surface.ts b/src/lib/modules/surface.ts index 4e39ab30f..6fdb69f58 100644 --- a/src/lib/modules/surface.ts +++ b/src/lib/modules/surface.ts @@ -274,10 +274,13 @@ export function surfaceModulesOfKind( /** * The surface id of a correlation channel. Rated mood factors carry their own * `FACTOR:` channel names but are mood entries all the same, so they - * resolve to the mood channel. + * resolve to the mood channel; person-defined symptoms (`SYMPTOM:`) resolve + * to the illness symptom channel the same way. */ export function correlationChannelSurfaceId(channelKey: string): string { - return channelKey.startsWith("FACTOR:") - ? "correlation:MOOD" - : `correlation:${channelKey}`; + if (channelKey.startsWith("FACTOR:")) return "correlation:MOOD"; + // A person-defined symptom (`SYMPTOM:`, v1.40) rides the illness + // module, like the symptom-burden channel it sits beside. + if (channelKey.startsWith("SYMPTOM:")) return "correlation:SYMPTOM_SEVERITY"; + return `correlation:${channelKey}`; } diff --git a/tests/integration/symptoms.test.ts b/tests/integration/symptoms.test.ts new file mode 100644 index 000000000..330d0dfbe --- /dev/null +++ b/tests/integration/symptoms.test.ts @@ -0,0 +1,578 @@ +/** + * Person-defined symptoms against real Postgres and through the real routes + * (v1.40, migration 0369). + * + * What only a real database can answer: + * + * - the two tables exist, the label and the note are ciphertext on disk, and + * the intensity CHECK holds; + * - the eight-definition cap counts active definitions only, and showing a + * hidden one again re-checks it; + * - an occurrence can be filed against the record's own OPEN episode and + * against nothing else: not a resolved one, not another account's; + * - each active definition reaches the discovery matrix as its own + * `SYMPTOM:` channel, labelled with its name, and leaves it when the + * illness module is off; + * - a portable export carries the names readable and restores them under + * the instance's key, dropping (and naming) a link to an episode the file + * does not carry. + * + * Mutation checks (each run, each seen red): + * - drop `resolvedAt: null` from `isLinkableEpisode` → "refuses a resolved + * episode" goes red (the occurrence is linked to a closed episode); + * - drop `userId` from the same lookup → "refuses another account's episode" + * goes red; + * - remove the cap check in the definitions POST → "holds eight active + * definitions" goes red. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { NextRequest } from "next/server"; + +import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec"; +import { assembleDiscoveryMatrix } from "@/lib/insights/discovery-matrix"; +import { buildSymptomsBackupSection } from "@/lib/export/symptoms-backup"; +import { restoreSymptomsData } from "@/lib/export/symptoms-backup"; +import type { RestoreSkipLog } from "@/lib/export/restore-skips"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const OWNER_ID = "symptoms-owner"; +const OTHER_ID = "symptoms-other"; + +type Handler = ( + req: NextRequest, + ctx: { params: Promise> }, +) => Promise; + +async function call( + handler: unknown, + method: string, + path: string, + body?: unknown, + params: Record = {}, +): Promise { + const request = new NextRequest(`http://localhost${path}`, { + method, + headers: { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + return (handler as Handler)(request, { params: Promise.resolve(params) }); +} + +async function json(response: Response): Promise { + return (await response.json()) as T; +} + +interface Envelope { + data: T; + error: string | null; + meta?: { errorCode?: string }; +} + +async function seedUsers() { + const prisma = getPrismaClient(); + for (const id of [OWNER_ID, OTHER_ID]) { + await prisma.user.create({ + data: { + id, + username: id, + email: `${id}@example.test`, + timezone: "Europe/Berlin", + locale: "en", + }, + }); + } +} + +async function signIn(userId: string) { + const session = await getPrismaClient().session.create({ + data: { + userId, + expiresAt: new Date(Date.now() + 86_400_000), + mfaVerifiedAt: new Date(), + }, + }); + cookieJar.set("healthlog_session", session.id); +} + +async function defineSymptom(label: string): Promise { + const { POST } = await import("@/app/api/symptoms/definitions/route"); + const response = await call(POST, "POST", "/api/symptoms/definitions", { + label, + icon: "Zap", + }); + expect(response.status, label).toBe(201); + return (await json>(response)).data.id; +} + +async function logEvent(body: Record): Promise { + const { POST } = await import("@/app/api/symptoms/events/route"); + return call(POST, "POST", "/api/symptoms/events", body); +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + await seedUsers(); + await signIn(OWNER_ID); +}); + +describe("symptom definitions", () => { + it("stores the name as ciphertext and reads it back with its summary", async () => { + const id = await defineSymptom("Aura"); + const row = await getPrismaClient().symptomDefinition.findUniqueOrThrow({ + where: { id }, + }); + expect(Buffer.from(row.labelEncrypted).toString("utf8")).not.toContain( + "Aura", + ); + expect(decryptFromBytes(row.labelEncrypted)).toBe("Aura"); + + await logEvent({ definitionId: id, intensity: 3 }); + await logEvent({ definitionId: id, intensity: 7 }); + + const { GET } = await import("@/app/api/symptoms/definitions/route"); + const listed = await json< + Envelope<{ + definitions: Array<{ + id: string; + label: string; + recent: { count30d: number; maxIntensity30d: number | null }; + }>; + limit: number; + }> + >(await call(GET, "GET", "/api/symptoms/definitions")); + expect(listed.data.limit).toBe(8); + expect(listed.data.definitions).toMatchObject([ + { id, label: "Aura", recent: { count30d: 2, maxIntensity30d: 7 } }, + ]); + }); + + it("holds eight active definitions, and a hidden one frees its slot", async () => { + const ids: string[] = []; + for (let i = 0; i < 8; i += 1) ids.push(await defineSymptom(`S${i}`)); + + const { POST } = await import("@/app/api/symptoms/definitions/route"); + const ninth = await call(POST, "POST", "/api/symptoms/definitions", { + label: "Ninth", + }); + expect(ninth.status).toBe(422); + expect((await json>(ninth)).meta?.errorCode).toBe( + "symptoms.definition.limitReached", + ); + + const { DELETE, PATCH } = + await import("@/app/api/symptoms/definitions/[id]/route"); + const hidden = await call( + DELETE, + "DELETE", + `/api/symptoms/definitions/${ids[0]}`, + undefined, + { id: ids[0] }, + ); + expect(hidden.status).toBe(200); + expect(await defineSymptom("Ninth")).toBeTruthy(); + + // Showing the hidden one again would make nine. + const reshown = await call( + PATCH, + "PATCH", + `/api/symptoms/definitions/${ids[0]}`, + { isActive: true }, + { id: ids[0] }, + ); + expect(reshown.status).toBe(422); + expect( + ( + await getPrismaClient().symptomDefinition.findUniqueOrThrow({ + where: { id: ids[0] }, + }) + ).isActive, + ).toBe(false); + }); + + it("purges a definition with every occurrence of it", async () => { + const id = await defineSymptom("Aura"); + await logEvent({ definitionId: id, intensity: 4 }); + const { DELETE } = + await import("@/app/api/symptoms/definitions/[id]/route"); + const response = await call( + DELETE, + "DELETE", + `/api/symptoms/definitions/${id}?purge=true`, + undefined, + { id }, + ); + expect(response.status).toBe(200); + expect( + (await json>(response)).data + .eventsDeleted, + ).toBe(1); + expect( + await getPrismaClient().symptomEvent.count({ + where: { userId: OWNER_ID }, + }), + ).toBe(0); + }); + + it("is a 404 for another account's definition", async () => { + const id = await defineSymptom("Aura"); + await signIn(OTHER_ID); + const { PATCH } = await import("@/app/api/symptoms/definitions/[id]/route"); + const response = await call( + PATCH, + "PATCH", + `/api/symptoms/definitions/${id}`, + { label: "Mine now" }, + { id }, + ); + expect(response.status).toBe(404); + expect( + ( + await json>( + await logEvent({ definitionId: id, intensity: 2 }), + ) + ).meta?.errorCode, + ).toBe("symptoms.definition.notFound"); + }); +}); + +describe("symptom occurrences", () => { + it("logs, lists, edits and deletes one, with the note encrypted", async () => { + const id = await defineSymptom("Headache"); + const created = await logEvent({ + definitionId: id, + intensity: 8, + occurredAt: "2026-09-01T09:30:00.000Z", + note: "behind the left eye", + }); + expect(created.status).toBe(201); + const event = (await json>(created)) + .data; + expect(event.note).toBe("behind the left eye"); + const row = await getPrismaClient().symptomEvent.findUniqueOrThrow({ + where: { id: event.id }, + }); + expect(decryptFromBytes(row.noteEncrypted!)).toBe("behind the left eye"); + + const { GET } = await import("@/app/api/symptoms/events/route"); + const listed = await json }>>( + await call(GET, "GET", `/api/symptoms/events?definitionId=${id}`), + ); + expect(listed.data.events.map((e) => e.id)).toEqual([event.id]); + + const { PATCH, DELETE } = + await import("@/app/api/symptoms/events/[id]/route"); + const patched = await call( + PATCH, + "PATCH", + `/api/symptoms/events/${event.id}`, + { intensity: 5, note: null }, + { id: event.id }, + ); + expect(patched.status).toBe(200); + const after = await getPrismaClient().symptomEvent.findUniqueOrThrow({ + where: { id: event.id }, + }); + expect(after.intensity).toBe(5); + expect(after.noteEncrypted).toBeNull(); + + const deleted = await call( + DELETE, + "DELETE", + `/api/symptoms/events/${event.id}`, + undefined, + { id: event.id }, + ); + expect(deleted.status).toBe(200); + expect( + await getPrismaClient().symptomEvent.count({ where: { id: event.id } }), + ).toBe(0); + }); + + it("refuses an intensity outside 0-10 and a time in the future", async () => { + const id = await defineSymptom("Aura"); + expect((await logEvent({ definitionId: id, intensity: 11 })).status).toBe( + 422, + ); + expect( + ( + await logEvent({ + definitionId: id, + intensity: 3, + occurredAt: new Date(Date.now() + 3_600_000).toISOString(), + }) + ).status, + ).toBe(422); + }); + + it("holds the 0-10 bound in the database too", async () => { + const id = await defineSymptom("Aura"); + await expect( + getPrismaClient().symptomEvent.create({ + data: { + userId: OWNER_ID, + definitionId: id, + occurredAt: new Date(), + intensity: 12, + }, + }), + ).rejects.toThrow(); + }); + + it("links an occurrence to the record's own open episode", async () => { + const id = await defineSymptom("Aura"); + const episode = await getPrismaClient().illnessEpisode.create({ + data: { + userId: OWNER_ID, + label: "Migraine", + type: "CHRONIC", + lifecycle: "CHRONIC_ONGOING", + onsetAt: new Date("2026-01-01T00:00:00.000Z"), + }, + }); + const response = await logEvent({ + definitionId: id, + intensity: 6, + episodeId: episode.id, + }); + expect(response.status).toBe(201); + expect( + (await json>(response)).data.episodeId, + ).toBe(episode.id); + }); + + it("refuses a resolved episode, and nothing is written", async () => { + const id = await defineSymptom("Aura"); + const episode = await getPrismaClient().illnessEpisode.create({ + data: { + userId: OWNER_ID, + label: "Cold", + type: "INFECTION", + onsetAt: new Date("2026-08-01T00:00:00.000Z"), + resolvedAt: new Date("2026-08-08T00:00:00.000Z"), + }, + }); + const response = await logEvent({ + definitionId: id, + intensity: 6, + episodeId: episode.id, + }); + expect(response.status).toBe(422); + expect((await json>(response)).meta?.errorCode).toBe( + "symptoms.episode.notOpen", + ); + expect( + await getPrismaClient().symptomEvent.count({ + where: { episodeId: episode.id }, + }), + ).toBe(0); + }); + + it("refuses another account's episode, and nothing is written", async () => { + const id = await defineSymptom("Aura"); + const foreign = await getPrismaClient().illnessEpisode.create({ + data: { + userId: OTHER_ID, + label: "Flu", + type: "INFECTION", + onsetAt: new Date("2026-09-01T00:00:00.000Z"), + }, + }); + const response = await logEvent({ + definitionId: id, + intensity: 6, + episodeId: foreign.id, + }); + expect(response.status).toBe(422); + expect( + await getPrismaClient().symptomEvent.count({ + where: { episodeId: foreign.id }, + }), + ).toBe(0); + }); + + it("answers illness.disabled with the module off", async () => { + await getPrismaClient().user.update({ + where: { id: OWNER_ID }, + data: { modulePreferencesJson: { illness: false } }, + }); + const { GET } = await import("@/app/api/symptoms/definitions/route"); + const response = await call(GET, "GET", "/api/symptoms/definitions"); + expect(response.status).toBe(403); + expect((await json>(response)).meta?.errorCode).toBe( + "illness.disabled", + ); + }); +}); + +describe("the SYMPTOM: correlation channel", () => { + it("reaches the matrix labelled with its name, and leaves it with the module off", async () => { + const prisma = getPrismaClient(); + const definition = await prisma.symptomDefinition.create({ + data: { userId: OWNER_ID, labelEncrypted: encryptToBytes("Aura") }, + }); + const hidden = await prisma.symptomDefinition.create({ + data: { + userId: OWNER_ID, + labelEncrypted: encryptToBytes("Old"), + isActive: false, + }, + }); + for (const [id, at, intensity] of [ + [definition.id, "2026-09-01T08:00:00.000Z", 3], + [definition.id, "2026-09-01T18:00:00.000Z", 6], + [definition.id, "2026-09-03T08:00:00.000Z", 2], + [hidden.id, "2026-09-02T08:00:00.000Z", 9], + ] as const) { + await prisma.symptomEvent.create({ + data: { + userId: OWNER_ID, + definitionId: id, + occurredAt: new Date(at), + intensity, + }, + }); + } + + const options = { + tz: "Europe/Berlin", + since: new Date("2026-08-01T00:00:00.000Z"), + fetchMode: "raw" as const, + }; + const on = await assembleDiscoveryMatrix(OWNER_ID, { + ...options, + modules: {}, + }); + const channels = on.series.filter((s) => s.key.startsWith("SYMPTOM:")); + expect(channels).toEqual([ + { + key: `SYMPTOM:${definition.id}`, + label: "Aura", + role: "outcome", + points: [ + { day: "2026-09-01", value: 6 }, + { day: "2026-09-02", value: 0 }, + { day: "2026-09-03", value: 2 }, + ], + }, + ]); + + const off = await assembleDiscoveryMatrix(OWNER_ID, { + ...options, + modules: { illness: false }, + }); + expect(off.series.some((s) => s.key.startsWith("SYMPTOM:"))).toBe(false); + }); +}); + +describe("the portable backup", () => { + it("carries the names readable and restores them, dropping a link to an episode the file lacks", async () => { + const prisma = getPrismaClient(); + const kept = await prisma.illnessEpisode.create({ + data: { + userId: OWNER_ID, + label: "Migraine", + type: "CHRONIC", + onsetAt: new Date("2026-01-01T00:00:00.000Z"), + }, + }); + const gone = await prisma.illnessEpisode.create({ + data: { + userId: OWNER_ID, + label: "Cold", + type: "INFECTION", + onsetAt: new Date("2026-07-01T00:00:00.000Z"), + }, + }); + const definition = await prisma.symptomDefinition.create({ + data: { + userId: OWNER_ID, + labelEncrypted: encryptToBytes("Aura"), + icon: "Zap", + }, + }); + for (const [episodeId, intensity] of [ + [kept.id, 4], + [gone.id, 6], + [null, 2], + ] as const) { + await prisma.symptomEvent.create({ + data: { + userId: OWNER_ID, + definitionId: definition.id, + occurredAt: new Date(`2026-07-0${intensity}T08:00:00.000Z`), + intensity, + episodeId, + noteEncrypted: intensity === 2 ? encryptToBytes("mild") : null, + }, + }); + } + + const section = await buildSymptomsBackupSection(prisma, OWNER_ID); + const text = JSON.stringify(section); + expect(text).toContain('"label":"Aura"'); + expect(text).toContain('"note":"mild"'); + expect(text).not.toContain("labelEncrypted"); + + // The restore puts back only the kept episode. + const skips: RestoreSkipLog = []; + await prisma.$transaction((tx) => + restoreSymptomsData( + tx, + OWNER_ID, + JSON.parse(text), + new Set([kept.id]), + skips, + ), + ); + + const restored = await prisma.symptomDefinition.findMany({ + where: { userId: OWNER_ID }, + include: { events: { orderBy: { intensity: "asc" } } }, + }); + expect(restored).toHaveLength(1); + expect(decryptFromBytes(restored[0].labelEncrypted)).toBe("Aura"); + expect( + restored[0].events.map((e) => ({ + intensity: e.intensity, + episodeId: e.episodeId, + note: e.noteEncrypted ? decryptFromBytes(e.noteEncrypted) : null, + })), + ).toEqual([ + { intensity: 2, episodeId: null, note: "mild" }, + { intensity: 4, episodeId: kept.id, note: null }, + { intensity: 6, episodeId: null, note: null }, + ]); + expect(skips).toEqual([ + { catalogue: "symptomEpisodeReference", key: gone.id, links: 1 }, + ]); + }); +}); From bb58b40b2aa79d81b13cd9c22d92a2a55bf4d90c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 3 Oct 2026 21:14:43 +0200 Subject: [PATCH 06/14] Run fact extraction in the background, re-checking under a row lock The extract route accepts `Prefer: respond-async` for all three inputs (text, stored, vision) and queues a DOCUMENT_EXTRACT run (added to migration 0370). The run's result is the count of staged facts and the document status; the facts themselves are read from the document. Without the header the route answers with the document detail as before; both paths run one shared body. A background read can take minutes, and a review the person finishes in the meantime must not have its approved facts replaced. Staging now locks the document row and asks again whether it may stage, refusing with the route's own alreadyConfirmed / alreadyPartlyConfirmed codes. The synchronous path gets the same re-check. The vault's stored-text extract uses the background path and shows the shared waiting line in place of its hint while the read runs. --- docs/api/openapi.yaml | 95 ++-- .../migration.sql | 5 +- prisma/schema.prisma | 3 + .../ai-capability-route-inventory.test.ts | 1 + .../[id]/extract/__tests__/route.test.ts | 32 +- .../documents/inbound/[id]/extract/route.ts | 462 +++++++----------- .../__tests__/document-facts-review.test.tsx | 2 + .../documents/document-facts-review.tsx | 25 +- .../documents/use-document-facts.ts | 39 +- src/lib/documents/ai-runs/extract-run.ts | 287 +++++++++++ src/lib/documents/ai-runs/types.ts | 23 +- .../jobs/__tests__/document-ai-run.test.ts | 116 +++++ src/lib/jobs/document-ai-run.ts | 62 +++ src/lib/openapi/routes/ai-runs.ts | 14 +- src/lib/openapi/routes/documents.ts | 21 +- tests/integration/document-ai-runs.test.ts | 168 +++++++ 16 files changed, 1003 insertions(+), 352 deletions(-) create mode 100644 src/lib/documents/ai-runs/extract-run.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 35847ef8a..46f4d3b65 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -16701,13 +16701,32 @@ paths: structures the document's own stored extracted text (its content index) — the manual recovery for a skipped/failed automatic staging run, 422 `documents.inbound.notIndexed` when no stored text exists. Extraction reproduces what the document states — it never interprets. Nothing reaches the structured stores here; the - confirm route is the only write path." + confirm route is the only write path. With `Prefer: respond-async` (v1.40) the extraction runs in the + background: after the same quick refusals the route answers 202 with a run id, a second request while an + extraction of this document from the same input is queued or running answers with that run, and `GET + /api/ai-runs/{id}` serves `DocumentExtractRunResult` (`{ documentId, factsStaged, status }`, not the document + detail) as the run's `result`. Staging re-checks the document under a row lock, so a review finished while the + read ran fails the run with `documents.inbound.alreadyPartlyConfirmed` (409) or + `documents.inbound.alreadyConfirmed` (422) and leaves the review untouched; the synchronous form answers the + same in that race." parameters: - name: id in: path required: true schema: type: string + - &a37 + name: Prefer + in: header + required: false + schema: + type: string + example: respond-async + description: "Send `respond-async` (RFC 7240) to run the read in the background: the route answers 202 with a run id + once every quick refusal has passed, and `GET /api/ai-runs/{id}` serves the result. Without it the route + answers synchronously with the result, as it always has; that form is retained for clients that have not + adopted runs, and its retirement will be announced one minor release ahead. A preference the route cannot + apply is ignored and the response then carries no `Preference-Applied`." requestBody: required: false content: @@ -16721,8 +16740,9 @@ paths: application/json: schema: $ref: "#/components/schemas/InboundExtractEnvelope" + "202": *a36 "401": *a1 - "403": &a37 + "403": &a38 description: "AI refusal: the `documentAi` capability is unavailable for this record. The envelope carries `meta.errorCode`, `meta.capability` and `meta.reason` (one of `AiUnavailableReason`). `assistant.disabled.documentAi` — the operator turned off reading documents, or every AI feature (the master @@ -16746,7 +16766,7 @@ paths: $ref: "#/components/schemas/ErrorEnvelope" "422": *a4 "429": *a5 - "503": &a38 + "503": &a39 description: "`ai.unavailable` — the inputs behind the AI capability could not be read, and the answer failed closed. Nothing was sent. Retry later." content: @@ -16776,18 +16796,7 @@ paths: required: true schema: type: string - - &a39 - name: Prefer - in: header - required: false - schema: - type: string - example: respond-async - description: "Send `respond-async` (RFC 7240) to run the read in the background: the route answers 202 with a run id - once every quick refusal has passed, and `GET /api/ai-runs/{id}` serves the result. Without it the route - answers synchronously with the result, as it always has; that form is retained for clients that have not - adopted runs, and its retirement will be announced one minor release ahead. A preference the route cannot - apply is ignored and the response then carries no `Preference-Applied`." + - *a37 requestBody: required: false content: @@ -16824,10 +16833,10 @@ paths: $ref: "#/components/schemas/DocumentSuggestEnvelope" "202": *a36 "401": *a1 - "403": *a37 + "403": *a38 "422": *a4 "429": *a5 - "503": *a38 + "503": *a39 /api/documents/inbound/{id}/summary: post: tags: @@ -16877,7 +16886,7 @@ paths: type: boolean default: false description: With `persist=true`, explicitly replace an existing stored summary. - - *a39 + - *a37 requestBody: required: false content: @@ -16914,10 +16923,10 @@ paths: $ref: "#/components/schemas/DocumentSummaryEnvelope" "202": *a36 "401": *a1 - "403": *a37 + "403": *a38 "422": *a4 "429": *a5 - "503": *a38 + "503": *a39 /api/documents/inbound/{id}/index: post: tags: @@ -16943,7 +16952,7 @@ paths: required: true schema: type: string - - *a39 + - *a37 requestBody: required: false content: @@ -16980,10 +16989,10 @@ paths: $ref: "#/components/schemas/DocumentIndexEnvelope" "202": *a36 "401": *a1 - "403": *a37 + "403": *a38 "422": *a4 "429": *a5 - "503": *a38 + "503": *a39 /api/documents/inbound/{id}/chat: post: tags: @@ -17023,10 +17032,10 @@ paths: type: string description: "SSE frames: `data: \\n\\n`. See the operation description for the per-`type` frame shapes." "401": *a1 - "403": *a37 + "403": *a38 "422": *a4 "429": *a5 - "503": *a38 + "503": *a39 get: tags: - Documents @@ -17089,10 +17098,10 @@ paths: schema: $ref: "#/components/schemas/DocumentReindexEnvelope" "401": *a1 - "403": *a37 + "403": *a38 "422": *a4 "429": *a5 - "503": *a38 + "503": *a39 /api/documents/inbound/{id}/facts/{factId}: patch: tags: @@ -63669,6 +63678,7 @@ components: - LABS_OCR_EXTRACT - DOCUMENT_SUMMARY - DOCUMENT_SUGGEST + - DOCUMENT_EXTRACT documentId: anyOf: - type: string @@ -63688,11 +63698,13 @@ components: - $ref: "#/components/schemas/OcrExtractResponse" - $ref: "#/components/schemas/DocumentSummaryResponse" - $ref: "#/components/schemas/DocumentSuggestResponse" + - $ref: "#/components/schemas/DocumentExtractRunResult" - type: "null" description: "Present once `status` is `SUCCEEDED`: the body the queuing route answers with synchronously. `DOCUMENT_INDEX` carries `DocumentIndexResponse`, `LABS_OCR_EXTRACT` carries `OcrExtractResponse`, - `DOCUMENT_SUMMARY` carries `DocumentSummaryResponse`, `DOCUMENT_SUGGEST` carries `DocumentSuggestResponse`. - Decode by `kind`; treat an unknown `kind` as a run this client did not start." + `DOCUMENT_SUMMARY` carries `DocumentSummaryResponse`, `DOCUMENT_SUGGEST` carries `DocumentSuggestResponse`, + `DOCUMENT_EXTRACT` carries `DocumentExtractRunResult` (not the extract route's detail body). Decode by + `kind`; treat an unknown `kind` as a run this client did not start." error: anyOf: - type: object @@ -63951,6 +63963,31 @@ components: required: - suggestions additionalProperties: false + DocumentExtractRunResult: + type: object + properties: + documentId: + type: string + factsStaged: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: How many facts were staged PENDING for review. Read them from `GET /api/documents/inbound/{id}`; the + confirm route stays the only write into the record. + status: + type: string + enum: + - STORED + - EXTRACTING + - EXTRACTED + - FAILED + - CONFIRMED + - DISCARDED + required: + - documentId + - factsStaged + - status + additionalProperties: false ListBiomarkersEnvelope: type: object properties: diff --git a/prisma/migrations/0370_document_ai_run_kinds_summary_suggest_extract/migration.sql b/prisma/migrations/0370_document_ai_run_kinds_summary_suggest_extract/migration.sql index 6d1ede60a..a64f219f2 100644 --- a/prisma/migrations/0370_document_ai_run_kinds_summary_suggest_extract/migration.sql +++ b/prisma/migrations/0370_document_ai_run_kinds_summary_suggest_extract/migration.sql @@ -1,6 +1,6 @@ --- Two more kinds of background document AI run. +-- Three more kinds of background document AI run. -- --- Summaries and filing suggestions can now run in the same background worker +-- Summaries, filing suggestions and fact extraction can now run in the same background worker -- as "Read with AI" and the lab scan (0366): the request queues the read and -- answers 202, and the result is polled from the run. The synchronous routes -- are unchanged; the new kinds only name the run. @@ -13,3 +13,4 @@ -- (inert with no rows that carry them). ALTER TYPE "document_ai_run_kind" ADD VALUE IF NOT EXISTS 'DOCUMENT_SUMMARY'; ALTER TYPE "document_ai_run_kind" ADD VALUE IF NOT EXISTS 'DOCUMENT_SUGGEST'; +ALTER TYPE "document_ai_run_kind" ADD VALUE IF NOT EXISTS 'DOCUMENT_EXTRACT'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 473f7c987..69cd6245f 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -9042,6 +9042,9 @@ enum DocumentAiRunKind { /// Filing suggestions (title, kind, date) for a stored document. Drafts /// only: nothing is written to the document. DOCUMENT_SUGGEST + /// Structured facts read out of a stored document, staged PENDING for the + /// review step. The confirm route stays the only write into the record. + DOCUMENT_EXTRACT @@map("document_ai_run_kind") } diff --git a/src/app/api/__tests__/ai-capability-route-inventory.test.ts b/src/app/api/__tests__/ai-capability-route-inventory.test.ts index 91c3c29c1..2bb7afd8c 100644 --- a/src/app/api/__tests__/ai-capability-route-inventory.test.ts +++ b/src/app/api/__tests__/ai-capability-route-inventory.test.ts @@ -171,6 +171,7 @@ const RUN_QUEUED_BY: Record = { LABS_OCR_EXTRACT: "src/app/api/labs/ocr/extract/route.ts", DOCUMENT_SUMMARY: "src/app/api/documents/inbound/[id]/summary/route.ts", DOCUMENT_SUGGEST: "src/app/api/documents/inbound/[id]/suggest/route.ts", + DOCUMENT_EXTRACT: "src/app/api/documents/inbound/[id]/extract/route.ts", }; describe("background run kinds", () => { diff --git a/src/app/api/documents/inbound/[id]/extract/__tests__/route.test.ts b/src/app/api/documents/inbound/[id]/extract/__tests__/route.test.ts index ccb39338b..a03e75fe6 100644 --- a/src/app/api/documents/inbound/[id]/extract/__tests__/route.test.ts +++ b/src/app/api/documents/inbound/[id]/extract/__tests__/route.test.ts @@ -19,8 +19,11 @@ import { NextRequest } from "next/server"; */ vi.mock("@/lib/db", () => { + // The staging transaction locks the document row and asks again whether + // it may stage (a review can finish while a background read runs). const tx = { - extractedFact: { deleteMany: vi.fn() }, + $queryRaw: vi.fn(async () => [{ status: "STORED" }]), + extractedFact: { deleteMany: vi.fn(), count: vi.fn(async () => 0) }, inboundDocument: { update: vi.fn(), findUniqueOrThrow: vi.fn() }, }; return { @@ -123,7 +126,11 @@ import { AiUnavailableError } from "@/lib/ai/capabilities/refusal"; const tx = ( prisma as unknown as { __tx: { - extractedFact: { deleteMany: ReturnType }; + $queryRaw: ReturnType; + extractedFact: { + deleteMany: ReturnType; + count: ReturnType; + }; inboundDocument: { update: ReturnType; findUniqueOrThrow: ReturnType; @@ -304,6 +311,27 @@ describe("POST /api/documents/inbound/[id]/extract — stored mode", () => { ); }); + it("refuses to stage when a fact was approved while the read ran, keeping the review", async () => { + tx.extractedFact.count.mockResolvedValueOnce(1); + const res = await POST(storedReq("doc-1") as never, ctx("doc-1") as never); + expect(res.status).toBe(409); + const body = await res.json(); + expect(body.meta.errorCode).toBe( + "documents.inbound.alreadyPartlyConfirmed", + ); + expect(tx.extractedFact.deleteMany).not.toHaveBeenCalled(); + expect(tx.inboundDocument.update).not.toHaveBeenCalled(); + }); + + it("refuses to stage when the document was confirmed while the read ran", async () => { + tx.$queryRaw.mockResolvedValueOnce([{ status: "CONFIRMED" }]); + const res = await POST(storedReq("doc-1") as never, ctx("doc-1") as never); + expect(res.status).toBe(422); + const body = await res.json(); + expect(body.meta.errorCode).toBe("documents.inbound.alreadyConfirmed"); + expect(tx.extractedFact.deleteMany).not.toHaveBeenCalled(); + }); + it("422s (notIndexed) without stored text, before any bucket charge", async () => { vi.mocked(loadDocumentChatText).mockResolvedValue(null as never); const res = await POST(storedReq("doc-1") as never, ctx("doc-1") as never); diff --git a/src/app/api/documents/inbound/[id]/extract/route.ts b/src/app/api/documents/inbound/[id]/extract/route.ts index 930d8c895..1fc8338c5 100644 --- a/src/app/api/documents/inbound/[id]/extract/route.ts +++ b/src/app/api/documents/inbound/[id]/extract/route.ts @@ -22,6 +22,15 @@ * The document is UNTRUSTED (prompt-injection): the server never acts on an * instruction inside it. The staged facts land PENDING for the mandatory * review-then-confirm screen; nothing reaches the structured stores here. + * + * v1.40 — `Prefer: respond-async` (RFC 7240) runs the extraction in the + * background worker: every refusal that can be answered quickly is still + * answered here, then the route answers 202 with a run id. The run's result is + * `{ documentId, factsStaged, status }` rather than the whole detail (the + * facts are staged on the document; the client reads them from there). + * Without the header the route behaves exactly as before; the iPhone app + * relies on that. Both paths run one body (`executeDocumentExtract`), which + * re-checks the document under a row lock before it stages anything. */ import { Buffer } from "node:buffer"; @@ -40,24 +49,13 @@ import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; import { requireAiCapability } from "@/lib/ai/capabilities/gate"; import { buildDateKey, - reconcileSpend, reserveBudget, resolveCostOwner, resolveDailyCap, } from "@/lib/ai/coach/budget"; -import { auditLog } from "@/lib/auth/audit"; +import type { ProviderChainType } from "@/lib/ai/provider-chain"; import { prisma } from "@/lib/db"; -import { - InboundExtractError, - runInboundExtraction, - type InboundExtractionResult, -} from "@/lib/documents/extract"; -import { - decryptDocumentContent, - encryptFactData, - encryptFactProvenance, - serialiseDocumentDetail, -} from "@/lib/documents/store"; +import { serialiseDocumentDetail } from "@/lib/documents/store"; import { requireDocumentTextProvider, requireDocumentVisionProvider, @@ -66,73 +64,34 @@ import { checkDocumentAiRateLimit, documentAiRateLimited, DOCUMENT_AI_TEXT_BODY_MAX_BYTES, + loadOwnedDocument, refundDocumentAiSlot, + type LoadedDocument, } from "@/lib/documents/ai-route-support"; +import { + executeDocumentExtract, + type DocumentExtractInput, +} from "@/lib/documents/ai-runs/extract-run"; +import { + acceptedRunResponse, + outcomeResponse, + prefersRespondAsync, +} from "@/lib/documents/ai-runs/http"; +import { findLiveDocumentRun, startAiRun } from "@/lib/documents/ai-runs/start"; +import type { AiRunBudget } from "@/lib/documents/ai-runs/types"; import { loadDocumentChatText } from "@/lib/documents/content-index"; -import { detectOcrMimeType } from "@/lib/labs/ocr-upload"; import { annotate } from "@/lib/logging/context"; import { requireModuleEnabled } from "@/lib/modules/gate"; import { inboundStoredExtractSchema, inboundTextExtractSchema, } from "@/lib/validations/inbound-documents"; -import { dateOnlyAtNoonUtc } from "@/lib/tz/date-only"; export const dynamic = "force-dynamic"; type RouteParams = { params: Promise<{ id: string }> }; -/** A loaded, owner-scoped, still-extractable document row. */ -type LoadedDocument = { - id: string; - kind: string; - contentEncrypted: Uint8Array; - contentCodec: string; - mimeType: string; - status: string; -}; - -/** - * Replace the document's staged facts and flip it to EXTRACTED in one - * transaction. Re-extraction is allowed only when NO fact is APPROVED yet (the - * POST handler refuses it otherwise), so clearing every prior staged fact here - * can never drop an approved row or sever a committed-record provenance link — - * the only facts present are PENDING / REJECTED leftovers from an earlier run. - */ -async function stageExtraction( - documentId: string, - userId: string, - result: InboundExtractionResult, -) { - return prisma.$transaction(async (tx) => { - await tx.extractedFact.deleteMany({ where: { documentId, userId } }); - await tx.inboundDocument.update({ - where: { id: documentId }, - data: { - status: "EXTRACTED", - providerType: result.providerType, - reportDate: result.reportDate - ? dateOnlyAtNoonUtc(result.reportDate) - : null, - facts: { - create: result.facts.map((f) => ({ - userId, - factType: f.factType, - status: "PENDING" as const, - confidence: f.confidence, - needsReview: f.needsReview, - dataEncrypted: encryptFactData(f.data), - provenanceEncrypted: encryptFactProvenance(f.provenance), - })), - }, - }, - }); - return tx.inboundDocument.findUniqueOrThrow({ - where: { id: documentId }, - include: { facts: { orderBy: { createdAt: "asc" } } }, - }); - }); -} +type ExtractInputMode = "text" | "stored" | "vision"; export const POST = apiHandler( async (request: NextRequest, { params }: RouteParams) => { @@ -146,17 +105,7 @@ export const POST = apiHandler( await requireAiCapability("documentAi", { pickDecides: true }); const { id } = await params; - const document = await prisma.inboundDocument.findFirst({ - where: { id, userId: user.id, deletedAt: null }, - select: { - id: true, - kind: true, - contentEncrypted: true, - contentCodec: true, - mimeType: true, - status: true, - }, - }); + const document = await loadOwnedDocument(user.id, id); if (!document) { return apiError("Document not found", 404, { errorCode: "documents.inbound.notFound", @@ -175,7 +124,8 @@ export const POST = apiHandler( // committed-record provenance link — and re-stage the same facts as PENDING, // letting the user approve them a second time and duplicate the committed // lab / condition / medication. Block it: the user must finish reviewing or - // discard the document first. + // discard the document first. Staging asks again under a row lock, because + // a review can finish while the read runs. const approvedCount = await prisma.extractedFact.count({ where: { documentId: document.id, userId: user.id, status: "APPROVED" }, }); @@ -191,6 +141,7 @@ export const POST = apiHandler( ); } + const background = prefersRespondAsync(request); const contentType = request.headers.get("content-type") ?? ""; if (contentType.includes("application/json")) { // One body read for both JSON modes, BEFORE any bucket charge — a @@ -200,20 +151,106 @@ export const POST = apiHandler( }); if (jsonError) return jsonError; if (inboundStoredExtractSchema.safeParse(body).success) { - return handleStoredExtract(request, user.id, document); + return handleStoredExtract(request, user.id, document, background); } - return handleTextExtract(request, user.id, document, body); + return handleTextExtract(request, user.id, document, body, background); } - return handleVisionExtract(request, user.id, document); + return handleVisionExtract(request, user.id, document, background); }, ); +/** + * An extraction of this document from the same input, already queued or + * running: attach to it rather than charge a second one. Background only. + */ +async function liveRunResponse( + userId: string, + documentId: string, + input: ExtractInputMode, +): Promise { + const live = await findLiveDocumentRun( + userId, + documentId, + "DOCUMENT_EXTRACT", + (params) => params.extract?.input === input, + ); + return live ? acceptedRunResponse(live) : null; +} + +/** Reserve the read's budget; null (slot refunded) when the day is spent. */ +async function reserveFor( + userId: string, + providerType: ProviderChainType, + tokens: number, +): Promise { + const dateKey = buildDateKey(); + const reservation = await reserveBudget( + userId, + tokens, + dateKey, + resolveDailyCap([{ providerType }]), + resolveCostOwner([{ providerType }]), + "coach", + ); + if (!reservation.allowed) { + await refundDocumentAiSlot(userId); + return null; + } + return { reserved: reservation.reserved, owner: reservation.owner, dateKey }; +} + +function budgetExceeded(): Response { + return apiError("Your AI usage budget for today is reached.", 429, { + errorCode: "documents.inbound.budgetExceeded", + }); +} + +/** Queue the extraction for the background worker. */ +function queueExtract( + userId: string, + document: LoadedDocument, + input: ExtractInputMode, + budget: AiRunBudget, + text: string | null, +): Promise { + return startAiRun({ + userId, + kind: "DOCUMENT_EXTRACT", + documentId: document.id, + params: { + mode: input === "vision" ? "vision" : "text", + budget, + extract: { input }, + }, + input: text === null ? null : Buffer.from(text, "utf8"), + refundSlot: () => refundDocumentAiSlot(userId), + }); +} + +/** The synchronous path: run the shared body and answer with the detail. */ +async function runNow( + request: NextRequest, + userId: string, + document: LoadedDocument, + input: DocumentExtractInput, +): Promise { + const outcome = await executeDocumentExtract({ + userId, + document, + input, + origin: { ipAddress: getClientIp(request), worker: false }, + }); + if (!outcome.ok) return outcomeResponse(outcome); + return apiSuccess(serialiseDocumentDetail(outcome.data, outcome.data.facts)); +} + /** TEXT mode — structure in-browser-OCR'd text against the stored row. */ async function handleTextExtract( request: NextRequest, userId: string, document: LoadedDocument, body: unknown, + background: boolean, ): Promise { const row = await prisma.user.findUnique({ where: { id: userId }, @@ -228,6 +265,11 @@ async function handleTextExtract( const pick = await requireDocumentTextProvider(userId); + if (background) { + const live = await liveRunResponse(userId, document.id, "text"); + if (live) return live; + } + const rl = await checkDocumentAiRateLimit(userId); if (!rl.allowed) return documentAiRateLimited(rl); @@ -244,68 +286,22 @@ async function handleTextExtract( ); } - const dateKey = buildDateKey(); - const reservation = await reserveBudget( + const budget = await reserveFor( userId, + pick.entry.providerType, AI_BUDGETS.ocrExtractText.maxTokens, - dateKey, - resolveDailyCap([{ providerType: pick.entry.providerType }]), - resolveCostOwner([{ providerType: pick.entry.providerType }]), - "coach", ); - if (!reservation.allowed) { - await refundDocumentAiSlot(userId); - return apiError("Your AI usage budget for today is reached.", 429, { - errorCode: "documents.inbound.budgetExceeded", - }); - } + if (!budget) return budgetExceeded(); - try { - const result = await runInboundExtraction({ - provider: pick.entry.instance, - providerType: pick.providerType, - ocrText: parsed.data.text, - }); - await reconcileSpend( - userId, - reservation.reserved, - reservation.reserved, - dateKey, - 0, - { servedBy: pick.entry.providerType, reservedOwner: reservation.owner }, - ); - - const updated = await stageExtraction(document.id, userId, result); - - await auditLog("documents.inbound.extract", { - userId, - ipAddress: getClientIp(request), - details: { - documentId: document.id, - facts: updated.facts.length, - mode: "text", - }, - }); - - return apiSuccess(serialiseDocumentDetail(updated, updated.facts)); - } catch (err) { - await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, { - servedBy: null, - reservedOwner: reservation.owner, - }); - if (err instanceof InboundExtractError) { - return apiError("Couldn't read the document. Try a clearer copy.", 422, { - errorCode: "documents.inbound.extractFailed", - }); - } - annotate({ - action: { name: "documents.inbound.extractFailed" }, - meta: { reason: "provider_error", mode: "text" }, - }); - return apiError("Couldn't read the document. Try a clearer copy.", 502, { - errorCode: "documents.inbound.extractFailed", - }); + if (background) { + return queueExtract(userId, document, "text", budget, parsed.data.text); } + return runNow(request, userId, document, { + mode: "text", + text: parsed.data.text, + pick, + budget, + }); } /** @@ -324,6 +320,7 @@ async function handleStoredExtract( request: NextRequest, userId: string, document: LoadedDocument, + background: boolean, ): Promise { const chat = await loadDocumentChatText(userId, document.id); if (!chat || !chat.text.trim()) { @@ -334,71 +331,31 @@ async function handleStoredExtract( const pick = await requireDocumentTextProvider(userId); + if (background) { + const live = await liveRunResponse(userId, document.id, "stored"); + if (live) return live; + } + const rl = await checkDocumentAiRateLimit(userId); if (!rl.allowed) return documentAiRateLimited(rl); - const dateKey = buildDateKey(); - const reservation = await reserveBudget( + const budget = await reserveFor( userId, + pick.entry.providerType, AI_BUDGETS.ocrExtractText.maxTokens, - dateKey, - resolveDailyCap([{ providerType: pick.entry.providerType }]), - resolveCostOwner([{ providerType: pick.entry.providerType }]), - "coach", ); - if (!reservation.allowed) { - await refundDocumentAiSlot(userId); - return apiError("Your AI usage budget for today is reached.", 429, { - errorCode: "documents.inbound.budgetExceeded", - }); - } + if (!budget) return budgetExceeded(); - try { - const result = await runInboundExtraction({ - provider: pick.entry.instance, - providerType: pick.providerType, - ocrText: chat.text, - }); - await reconcileSpend( - userId, - reservation.reserved, - reservation.reserved, - dateKey, - 0, - { servedBy: pick.entry.providerType, reservedOwner: reservation.owner }, - ); - - const updated = await stageExtraction(document.id, userId, result); - - await auditLog("documents.inbound.extract", { - userId, - ipAddress: getClientIp(request), - details: { - documentId: document.id, - facts: updated.facts.length, - mode: "stored", - }, - }); - - return apiSuccess(serialiseDocumentDetail(updated, updated.facts)); - } catch (err) { - await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, { - servedBy: null, - reservedOwner: reservation.owner, - }); - if (err instanceof InboundExtractError) { - return apiError("Couldn't read the document. Try a clearer copy.", 422, { - errorCode: "documents.inbound.extractFailed", - }); - } - annotate({ - action: { name: "documents.inbound.extractFailed" }, - meta: { reason: "provider_error", mode: "stored" }, - }); - return apiError("Couldn't read the document. Try a clearer copy.", 502, { - errorCode: "documents.inbound.extractFailed", - }); + if (background) { + // The worker loads the stored text itself; nothing is sealed into the run. + return queueExtract(userId, document, "stored", budget, null); } + return runNow(request, userId, document, { + mode: "stored", + text: chat.text, + pick, + budget, + }); } /** VISION mode — run the stored original through the vision provider. */ @@ -406,116 +363,29 @@ async function handleVisionExtract( request: NextRequest, userId: string, document: LoadedDocument, + background: boolean, ): Promise { const pick = await requireDocumentVisionProvider(userId); + if (background) { + const live = await liveRunResponse(userId, document.id, "vision"); + if (live) return live; + } + const rl = await checkDocumentAiRateLimit(userId); if (!rl.allowed) return documentAiRateLimited(rl); - // Decrypt the stored original and re-derive its MIME from the bytes (never - // trust a stored label for the provider call). Every miss below happens - // before the provider dispatch, so the charged slot goes back. - let buffer: Buffer; - try { - buffer = decryptDocumentContent( - document.contentEncrypted, - document.contentCodec, - ); - } catch { - await refundDocumentAiSlot(userId); - return apiError("Couldn't read the stored document.", 422, { - errorCode: "documents.inbound.extractFailed", - }); - } - - const mime = detectOcrMimeType(buffer); - if (!mime) { - await refundDocumentAiSlot(userId); - return apiError( - "This document can't be scanned. Use local OCR (text mode).", - 422, - { errorCode: "documents.inbound.fileType" }, - ); - } - - if (mime === "application/pdf" && !pick.pdfSupported) { - await refundDocumentAiSlot(userId); - return apiError( - "PDF scanning needs a Claude vision provider; use local OCR instead.", - 422, - { errorCode: "documents.inbound.pdfNeedsAnthropic" }, - ); - } - - const dateKey = buildDateKey(); - const reservation = await reserveBudget( + const budget = await reserveFor( userId, + pick.entry.providerType, AI_BUDGETS.ocrExtract.maxTokens, - dateKey, - resolveDailyCap([{ providerType: pick.entry.providerType }]), - resolveCostOwner([{ providerType: pick.entry.providerType }]), - "coach", ); - if (!reservation.allowed) { - await refundDocumentAiSlot(userId); - return apiError("Your AI usage budget for today is reached.", 429, { - errorCode: "documents.inbound.budgetExceeded", - }); - } + if (!budget) return budgetExceeded(); - const dataBase64 = buffer.toString("base64"); - const images = - mime === "application/pdf" ? [] : [{ mediaType: mime, dataBase64 }]; - const documents = - mime === "application/pdf" - ? [{ mediaType: "application/pdf" as const, dataBase64 }] - : []; - - try { - const result = await runInboundExtraction({ - provider: pick.entry.instance, - providerType: pick.providerType, - images, - documents, - }); - await reconcileSpend( - userId, - reservation.reserved, - reservation.reserved, - dateKey, - 0, - { servedBy: pick.entry.providerType, reservedOwner: reservation.owner }, - ); - - const updated = await stageExtraction(document.id, userId, result); - - await auditLog("documents.inbound.extract", { - userId, - ipAddress: getClientIp(request), - details: { - documentId: document.id, - facts: updated.facts.length, - mode: "vision", - }, - }); - - return apiSuccess(serialiseDocumentDetail(updated, updated.facts)); - } catch (err) { - await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, { - servedBy: null, - reservedOwner: reservation.owner, - }); - if (err instanceof InboundExtractError) { - return apiError("Couldn't read the document. Try a clearer copy.", 422, { - errorCode: "documents.inbound.extractFailed", - }); - } - annotate({ - action: { name: "documents.inbound.extractFailed" }, - meta: { reason: "provider_error", mode: "vision" }, - }); - return apiError("Couldn't read the document. Try a clearer copy.", 502, { - errorCode: "documents.inbound.extractFailed", - }); + if (background) { + // The worker decrypts the stored original itself, after it has picked + // the provider again and re-checked the wire for it. + return queueExtract(userId, document, "vision", budget, null); } + return runNow(request, userId, document, { mode: "vision", pick, budget }); } diff --git a/src/components/documents/__tests__/document-facts-review.test.tsx b/src/components/documents/__tests__/document-facts-review.test.tsx index c2c64d314..33c2b7b47 100644 --- a/src/components/documents/__tests__/document-facts-review.test.tsx +++ b/src/components/documents/__tests__/document-facts-review.test.tsx @@ -125,6 +125,8 @@ describe("", () => { expect(html).toContain("Extract lab values"); // Honest about the mechanism: stored text, review-first, no auto-save. expect(html).toContain("nothing is saved automatically"); + // No background read is running, so no run line takes the hint's place. + expect(html).not.toContain("document-facts-extract-run-phase"); }); it("withdraws the extract offer once facts were approved", () => { diff --git a/src/components/documents/document-facts-review.tsx b/src/components/documents/document-facts-review.tsx index ae20eedff..f3ac38128 100644 --- a/src/components/documents/document-facts-review.tsx +++ b/src/components/documents/document-facts-review.tsx @@ -46,6 +46,7 @@ import type { ObservationFactData, } from "@/lib/validations/inbound-documents"; +import { AiRunPhaseNote } from "./document-ai-panels"; import { useDocumentAiErrorText } from "./use-document-assist"; import { useConfirmFacts, @@ -625,12 +626,12 @@ export function DocumentFactsSection({ storedExtract.mutate( { documentId: doc.id }, { - onSuccess: (detail) => { + onSuccess: (result) => { setFailures({}); setIncluded({}); - if ( - detail.facts.some((fact) => fact.status === "PENDING") - ) { + // The staged facts arrive with the document refetch the + // hook starts; the review opens onto them. + if (result.factsStaged > 0) { setOpen(true); } else { toast.info(t("documents.review.extractEmpty")); @@ -654,9 +655,19 @@ export function DocumentFactsSection({ ? t("documents.review.extracting") : t("documents.review.extract")} -

- {t("documents.review.extractHint")} -

+ {storedExtract.runPhase === "idle" ? ( +

+ {t("documents.review.extractHint")} +

+ ) : ( + // While the read runs, the hint gives way to where it stands: + // one muted line, the same one every document read shows. + + )} )} diff --git a/src/components/documents/use-document-facts.ts b/src/components/documents/use-document-facts.ts index 497724761..469155b9b 100644 --- a/src/components/documents/use-document-facts.ts +++ b/src/components/documents/use-document-facts.ts @@ -15,11 +15,15 @@ */ import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { resolveAiRun, useAiRunPhase } from "@/hooks/use-ai-run"; import { apiPatch, apiPost } from "@/lib/api/api-fetch"; +import type { + AiRunAccepted, + DocumentExtractRunResult, +} from "@/lib/documents/ai-runs/types"; import { invalidateKeys, queryKeys } from "@/lib/query-keys"; import type { ExtractedFactDto, - InboundDocumentDetailDto, InboundFactEdit, } from "@/lib/validations/inbound-documents"; @@ -35,17 +39,35 @@ export interface ConfirmFactsResult { * Structure the document's own stored extracted text into staged facts — * the manual recovery when the automatic staging run was skipped or failed. * Stages PENDING facts only; the review + confirm steps stay mandatory. - * A provider call is slow, so the request opts out of the 15 s default. + * + * The extraction runs in the background (v1.40): the request only queues it, + * so no proxy can cut a slow model, and `runPhase` says so while it runs. The + * run reports how many facts it staged; the facts themselves are read from + * the document, which is refetched when the run ends. */ export function useStoredExtract() { const queryClient = useQueryClient(); - return useMutation({ - mutationFn: ({ documentId }) => - apiPost( - `/api/documents/inbound/${documentId}/extract`, - { mode: "stored" }, - { signal: AbortSignal.timeout(120_000) }, + const { phase, onQueued, onProgress, reset } = useAiRunPhase(); + const mutation = useMutation< + DocumentExtractRunResult, + Error, + { documentId: string } + >({ + mutationFn: async ({ documentId }) => + resolveAiRun( + queryClient, + await apiPost( + `/api/documents/inbound/${documentId}/extract`, + { mode: "stored" }, + { + // The request only queues the read: the enqueue, no model. + signal: AbortSignal.timeout(60_000), + headers: { Prefer: "respond-async" }, + }, + ), + { onQueued, onProgress }, ), + onSettled: reset, onSuccess: (_data, { documentId }) => { void invalidateKeys(queryClient, [ queryKeys.inboundDocument(documentId), @@ -53,6 +75,7 @@ export function useStoredExtract() { ]); }, }); + return { ...mutation, runPhase: mutation.isPending ? phase : "idle" }; } /** diff --git a/src/lib/documents/ai-runs/extract-run.ts b/src/lib/documents/ai-runs/extract-run.ts new file mode 100644 index 000000000..226a6c98f --- /dev/null +++ b/src/lib/documents/ai-runs/extract-run.ts @@ -0,0 +1,287 @@ +/** + * Fact extraction from a stored document: the body both the synchronous + * extract route and the background run execute, so the two paths cannot + * drift. + * + * One extraction (a read and its one corrective retry) over text the browser + * read, the document's own stored text, or the stored original, against a + * reservation the caller already took and which is settled here. The facts + * land PENDING for the mandatory review step; the confirm route stays the only + * write into the person's record. + * + * Staging re-checks the document under a row lock. The route refuses a + * confirmed or partly confirmed document before it charges anything, but a + * background read can take minutes, and a review the person finishes in the + * meantime must never have its approved facts replaced by fresh PENDING ones + * (which could then be approved a second time). So the staging transaction + * locks the document row, asks again, and refuses with the route's own code. + */ +import { Buffer } from "node:buffer"; + +import type { ProviderChainType } from "@/lib/ai/provider-chain"; +import { auditLog } from "@/lib/auth/audit"; +import { prisma } from "@/lib/db"; +import { + refundDocumentAiSlot, + type LoadedDocument, +} from "@/lib/documents/ai-route-support"; +import { + InboundExtractError, + runInboundExtraction, + type InboundExtractionResult, + type RunInboundExtractionArgs, +} from "@/lib/documents/extract"; +import { + decryptDocumentContent, + encryptFactData, + encryptFactProvenance, +} from "@/lib/documents/store"; +import { detectOcrMimeType } from "@/lib/labs/ocr-upload"; +import { annotate } from "@/lib/logging/context"; +import { dateOnlyAtNoonUtc } from "@/lib/tz/date-only"; + +import { + runFailure, + unreadableDocument, + type DocumentTextPick, + type DocumentVisionPick, + type RunOrigin, +} from "./run-support"; +import { settleAiRunBudget } from "./store"; +import type { AiRunBudget, AiRunOutcome } from "./types"; + +/** The extraction and its one corrective retry. */ +export const DOCUMENT_EXTRACT_MODEL_CALLS = 2; + +/** + * What the extraction reads: text the browser read (`text`), the document's + * own content index (`stored`, loaded by the caller), or the stored original + * (`vision`). + */ +export type DocumentExtractInput = + | { + mode: "text" | "stored"; + text: string; + pick: DocumentTextPick; + budget: AiRunBudget; + } + | { mode: "vision"; pick: DocumentVisionPick; budget: AiRunBudget }; + +/** The staged document with its facts, for the route's detail body. */ +export type StagedDocument = Awaited>; + +class StagingRefused extends Error { + constructor(readonly outcome: AiRunOutcome) { + super("staging refused"); + } +} + +/** + * Replace the document's staged facts and flip it to EXTRACTED in one + * transaction, after locking the row and asking again whether it may be. + * + * Re-extraction is allowed only when NO fact is APPROVED and the document is + * not CONFIRMED, so clearing every prior staged fact here can never drop an + * approved row or sever a committed-record provenance link: the only facts + * present are PENDING / REJECTED leftovers from an earlier run. + */ +async function stageExtraction( + documentId: string, + userId: string, + result: InboundExtractionResult, +) { + return prisma.$transaction(async (tx) => { + const locked = await tx.$queryRaw<{ status: string }[]>` + SELECT status::text AS status + FROM inbound_documents + WHERE id = ${documentId} AND user_id = ${userId} AND deleted_at IS NULL + FOR UPDATE`; + const status = locked[0]?.status; + if (!status) { + throw new StagingRefused( + runFailure({ + status: 404, + message: "Document not found", + errorCode: "documents.inbound.notFound", + }), + ); + } + if (status === "CONFIRMED") { + throw new StagingRefused( + runFailure({ + status: 422, + message: "This document has already been confirmed.", + errorCode: "documents.inbound.alreadyConfirmed", + }), + ); + } + const approved = await tx.extractedFact.count({ + where: { documentId, userId, status: "APPROVED" }, + }); + if (approved > 0) { + throw new StagingRefused( + runFailure({ + status: 409, + message: + "Some facts from this document are already confirmed. Finish reviewing or discard it before extracting again.", + errorCode: "documents.inbound.alreadyPartlyConfirmed", + }), + ); + } + await tx.extractedFact.deleteMany({ where: { documentId, userId } }); + await tx.inboundDocument.update({ + where: { id: documentId }, + data: { + status: "EXTRACTED", + providerType: result.providerType, + reportDate: result.reportDate + ? dateOnlyAtNoonUtc(result.reportDate) + : null, + facts: { + create: result.facts.map((f) => ({ + userId, + factType: f.factType, + status: "PENDING" as const, + confidence: f.confidence, + needsReview: f.needsReview, + dataEncrypted: encryptFactData(f.data), + provenanceEncrypted: encryptFactProvenance(f.provenance), + })), + }, + }, + }); + return tx.inboundDocument.findUniqueOrThrow({ + where: { id: documentId }, + include: { facts: { orderBy: { createdAt: "asc" } } }, + }); + }); +} + +/** + * The provider input for the stored original: decrypted, its MIME re-derived + * from the bytes (never the stored label), a PDF only for a provider that + * reads one. Every miss happens before the dispatch. + */ +function visionArgs( + document: LoadedDocument, + pick: DocumentVisionPick, +): + Pick | AiRunOutcome { + let buffer: Buffer; + try { + buffer = decryptDocumentContent( + document.contentEncrypted, + document.contentCodec, + ); + } catch { + return runFailure({ + status: 422, + message: "Couldn't read the stored document.", + errorCode: "documents.inbound.extractFailed", + }); + } + const mime = detectOcrMimeType(buffer); + if (!mime) { + return runFailure({ + status: 422, + message: "This document can't be scanned. Use local OCR (text mode).", + errorCode: "documents.inbound.fileType", + }); + } + if (mime === "application/pdf" && !pick.pdfSupported) { + return runFailure({ + status: 422, + message: + "PDF scanning needs a Claude vision provider; use local OCR instead.", + errorCode: "documents.inbound.pdfNeedsAnthropic", + }); + } + const dataBase64 = buffer.toString("base64"); + return mime === "application/pdf" + ? { images: [], documents: [{ mediaType: "application/pdf", dataBase64 }] } + : { images: [{ mediaType: mime, dataBase64 }], documents: [] }; +} + +/** Extract and stage facts from one owned document. The caller authorised and charged it. */ +export async function executeDocumentExtract(args: { + userId: string; + document: LoadedDocument; + input: DocumentExtractInput; + origin: RunOrigin; +}): Promise> { + const { userId, document, input, origin } = args; + const { pick, budget } = input; + const servedBy = pick.entry.providerType as ProviderChainType; + + let source: Pick< + RunInboundExtractionArgs, + "images" | "documents" | "ocrText" + >; + if (input.mode === "vision") { + const prepared = visionArgs(document, input.pick); + if ("ok" in prepared) { + // Preparation failed before any provider dispatch: the reservation and + // the slot go back. + await settleAiRunBudget(userId, budget, 0, null); + await refundDocumentAiSlot(userId); + return prepared; + } + source = prepared; + } else { + source = { ocrText: input.text }; + } + + let result: InboundExtractionResult; + try { + result = await runInboundExtraction({ + provider: pick.entry.instance, + providerType: pick.providerType, + ...source, + }); + } catch (err) { + await settleAiRunBudget(userId, budget, 0, null); + if (!(err instanceof InboundExtractError)) { + annotate({ + action: { name: "documents.inbound.extractFailed" }, + meta: { reason: "provider_error", mode: input.mode }, + }); + } + return unreadableDocument(err instanceof InboundExtractError); + } + await settleAiRunBudget(userId, budget, budget.reserved, servedBy); + + let staged: StagedDocument; + try { + staged = await stageExtraction(document.id, userId, result); + } catch (err) { + if (!(err instanceof StagingRefused)) { + // The reservation is already settled for the call that was made; a + // write that failed is answered as the route always answered it. + annotate({ + action: { name: "documents.inbound.extractFailed" }, + meta: { reason: "staging_error", mode: input.mode }, + }); + return unreadableDocument(false); + } + annotate({ + action: { name: "documents.inbound.extractRefusedAtStaging" }, + meta: { + documentId: document.id, + errorCode: err.outcome.ok ? null : err.outcome.errorCode, + }, + }); + return err.outcome; + } + + await auditLog("documents.inbound.extract", { + userId, + ...(origin.worker ? { actorUserId: userId } : {}), + ipAddress: origin.ipAddress, + details: { + documentId: document.id, + facts: staged.facts.length, + mode: input.mode, + }, + }); + return { ok: true, data: staged }; +} diff --git a/src/lib/documents/ai-runs/types.ts b/src/lib/documents/ai-runs/types.ts index 74986eb77..2c63c7d92 100644 --- a/src/lib/documents/ai-runs/types.ts +++ b/src/lib/documents/ai-runs/types.ts @@ -10,7 +10,10 @@ */ import type { AiCapabilityKey } from "@/lib/ai/capabilities/types"; import type { Locale } from "@/lib/i18n/config"; -import type { InboundDocumentKindValue } from "@/lib/validations/inbound-documents"; +import type { + InboundDocumentKindValue, + InboundDocumentStatusValue, +} from "@/lib/validations/inbound-documents"; import type { OcrExtractResponseDto } from "@/lib/validations/labs-ocr"; export const DOCUMENT_AI_RUN_KINDS = [ @@ -18,6 +21,7 @@ export const DOCUMENT_AI_RUN_KINDS = [ "LABS_OCR_EXTRACT", "DOCUMENT_SUMMARY", "DOCUMENT_SUGGEST", + "DOCUMENT_EXTRACT", ] as const; export type DocumentAiRunKindValue = (typeof DOCUMENT_AI_RUN_KINDS)[number]; @@ -33,6 +37,7 @@ export const DOCUMENT_AI_RUN_CAPABILITY: Record< LABS_OCR_EXTRACT: "labsOcr", DOCUMENT_SUMMARY: "documentAi", DOCUMENT_SUGGEST: "documentAi", + DOCUMENT_EXTRACT: "documentAi", }; export const DOCUMENT_AI_RUN_STATUSES = [ @@ -90,6 +95,8 @@ export interface AiRunParams { replace: boolean; locale: Locale; }; + /** An extract run: which text it structures. `stored` reads the content index. */ + extract?: { input: "text" | "stored" | "vision" }; } /** "Read with AI": the same body the synchronous index route answers with. */ @@ -114,11 +121,23 @@ export interface DocumentSuggestRunResult { }; } +/** + * An extract run. Smaller than the synchronous route's body (the whole + * document detail): the facts are staged on the document, and the client + * reads them from there. + */ +export interface DocumentExtractRunResult { + documentId: string; + factsStaged: number; + status: InboundDocumentStatusValue; +} + export type AiRunResult = | DocumentIndexRunResult | OcrExtractResponseDto | DocumentSummaryRunResult - | DocumentSuggestRunResult; + | DocumentSuggestRunResult + | DocumentExtractRunResult; /** * What one run body produced: the success body, or the failure the diff --git a/src/lib/jobs/__tests__/document-ai-run.test.ts b/src/lib/jobs/__tests__/document-ai-run.test.ts index 752157713..d66bdd9c2 100644 --- a/src/lib/jobs/__tests__/document-ai-run.test.ts +++ b/src/lib/jobs/__tests__/document-ai-run.test.ts @@ -79,6 +79,16 @@ vi.mock("@/lib/documents/ai-runs/suggest-run", () => ({ executeDocumentSuggest: (...a: unknown[]) => executeDocumentSuggest(...a), })); +const executeDocumentExtract = vi.fn(); +vi.mock("@/lib/documents/ai-runs/extract-run", () => ({ + DOCUMENT_EXTRACT_MODEL_CALLS: 2, + executeDocumentExtract: (...a: unknown[]) => executeDocumentExtract(...a), +})); +const loadDocumentChatText = vi.fn(); +vi.mock("@/lib/documents/content-index", () => ({ + loadDocumentChatText: (...a: unknown[]) => loadDocumentChatText(...a), +})); + const loadOwnedDocument = vi.fn(); vi.mock("@/lib/documents/ai-route-support", () => ({ loadOwnedDocument: (...a: unknown[]) => loadOwnedDocument(...a), @@ -554,3 +564,109 @@ describe("summary and suggestion runs", () => { expect(markQueuedSummaryUnavailable).not.toHaveBeenCalled(); }); }); + +describe("extract runs", () => { + const extractRun = (input: "stored" | "vision" | "text") => + claimed({ + kind: "DOCUMENT_EXTRACT", + params: { + mode: input === "vision" ? "vision" : "text", + budget, + extract: { input }, + }, + input: input === "text" ? Buffer.from("Hb 14 g/dL") : null, + }); + + beforeEach(() => { + executeDocumentExtract.mockResolvedValue({ + ok: true, + data: { + id: "d1", + status: "EXTRACTED", + facts: [{ id: "f1" }, { id: "f2" }], + }, + }); + loadDocumentChatText.mockResolvedValue({ text: "Hb 14 g/dL" }); + requireDocumentTextProvider.mockResolvedValue({ + entry: { + providerType: "local", + instance: { responseTimeoutSeconds: 100 }, + }, + providerType: "local", + }); + }); + + it("stores only the count and the status, with two calls' deadline", async () => { + claimAiRun.mockResolvedValue(extractRun("vision")); + requireDocumentVisionProvider.mockResolvedValue(visionPick(300)); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe( + "succeeded", + ); + expect(updateMany).toHaveBeenCalledWith({ + where: { id: "run1", status: "RUNNING" }, + data: { expiresAt: new Date(NOW + 2 * 300_000 + 60_000) }, + }); + expect(completeAiRun).toHaveBeenCalledWith( + "run1", + { documentId: "d1", factsStaged: 2, status: "EXTRACTED" }, + new Date(NOW), + ); + }); + + it("reads the stored text for a stored extract and re-checks the text provider's wire", async () => { + claimAiRun.mockResolvedValue(extractRun("stored")); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe( + "succeeded", + ); + expect(loadDocumentChatText).toHaveBeenCalledWith("u1", "d1"); + expect(aiEgressRefusal).toHaveBeenCalledWith("documentAi", "u1", ["local"]); + expect(executeDocumentExtract).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ mode: "stored", text: "Hb 14 g/dL" }), + }), + ); + }); + + it("fails a stored extract whose text is gone, before any provider pick", async () => { + claimAiRun.mockResolvedValue(extractRun("stored")); + loadDocumentChatText.mockResolvedValue(null); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed"); + expect(requireDocumentTextProvider).not.toHaveBeenCalled(); + expect(executeDocumentExtract).not.toHaveBeenCalled(); + expect(settleAiRunBudget).toHaveBeenCalledOnce(); + expect(failAiRun).toHaveBeenCalledWith( + "run1", + expect.objectContaining({ + status: 422, + errorCode: "documents.inbound.notIndexed", + }), + "RUNNING", + new Date(NOW), + ); + }); + + it("stores the staging refusal when a review finished while the read ran", async () => { + claimAiRun.mockResolvedValue(extractRun("text")); + executeDocumentExtract.mockResolvedValue({ + ok: false, + status: 409, + message: "Some facts from this document are already confirmed.", + errorCode: "documents.inbound.alreadyPartlyConfirmed", + }); + expect(await runDocumentAiRun("run1", NOW + 2_700_000, now)).toBe("failed"); + expect(executeDocumentExtract).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ mode: "text", text: "Hb 14 g/dL" }), + }), + ); + expect(failAiRun).toHaveBeenCalledWith( + "run1", + expect.objectContaining({ + status: 409, + errorCode: "documents.inbound.alreadyPartlyConfirmed", + }), + "RUNNING", + new Date(NOW), + ); + }); +}); diff --git a/src/lib/jobs/document-ai-run.ts b/src/lib/jobs/document-ai-run.ts index c8244de7e..40388d7e0 100644 --- a/src/lib/jobs/document-ai-run.ts +++ b/src/lib/jobs/document-ai-run.ts @@ -39,7 +39,12 @@ import { import { callTimeoutMs } from "@/lib/ai/effective-timeout"; import { prisma } from "@/lib/db"; import { loadOwnedDocument } from "@/lib/documents/ai-route-support"; +import { loadDocumentChatText } from "@/lib/documents/content-index"; import { refusalAsFailure } from "@/lib/documents/ai-runs/http"; +import { + DOCUMENT_EXTRACT_MODEL_CALLS, + executeDocumentExtract, +} from "@/lib/documents/ai-runs/extract-run"; import { DOCUMENT_INDEX_MODEL_CALLS, executeDocumentIndex, @@ -389,6 +394,62 @@ async function runDocumentSuggestRead( }); } +/** The text an extract run structures: the sealed browser text, or the content index. */ +async function extractText(run: ClaimedAiRun): Promise { + if (run.params.extract?.input !== "stored") return runText(run); + const chat = run.documentId + ? await loadDocumentChatText(run.userId, run.documentId) + : null; + if (!chat || !chat.text.trim()) { + throw new RunRefused({ + status: 422, + message: "Read the document first, then extract.", + errorCode: "documents.inbound.notIndexed", + }); + } + return chat.text; +} + +async function runDocumentExtractRead( + run: ClaimedAiRun, + deadlineAt: number | undefined, + now: () => number, +): Promise> { + const document = await loadRunDocument(run); + const text = run.params.mode === "text" ? await extractText(run) : null; + const read = await pickForDocumentRead( + run, + DOCUMENT_EXTRACT_MODEL_CALLS, + deadlineAt, + now, + ); + const budget = runBudget(run); + const outcome = await executeDocumentExtract({ + userId: run.userId, + document, + input: + read.mode === "text" + ? { + mode: run.params.extract?.input === "stored" ? "stored" : "text", + text: text ?? "", + pick: read.pick, + budget, + } + : { mode: "vision", pick: read.pick, budget }, + origin: { ipAddress: null, worker: true }, + }); + if (!outcome.ok) return outcome; + // The facts are staged on the document; the run carries only the count. + return { + ok: true, + data: { + documentId: outcome.data.id, + factsStaged: outcome.data.facts.length, + status: outcome.data.status, + }, + }; +} + type RunDispatcher = ( run: ClaimedAiRun, deadlineAt: number | undefined, @@ -405,6 +466,7 @@ const RUN_DISPATCH: Record = { LABS_OCR_EXTRACT: runLabsOcr, DOCUMENT_SUMMARY: runDocumentSummaryRead, DOCUMENT_SUGGEST: runDocumentSuggestRead, + DOCUMENT_EXTRACT: runDocumentExtractRead, }; /** diff --git a/src/lib/openapi/routes/ai-runs.ts b/src/lib/openapi/routes/ai-runs.ts index e65d7c1e0..d2eeefba6 100644 --- a/src/lib/openapi/routes/ai-runs.ts +++ b/src/lib/openapi/routes/ai-runs.ts @@ -7,16 +7,18 @@ * Part of the OpenAPI route table; aggregated in `./index.ts`. * * iOS coordination (v1.40): `POST /api/documents/inbound/{id}/index`, - * `…/summary` and `…/suggest` keep their 200 bodies when the request carries - * no `Prefer` header. A client that - * sends `Prefer: respond-async` gets 202 with a run id and polls the run here; - * the result is the same body the route would have answered with. + * `…/summary`, `…/suggest` and `…/extract` keep their 200 bodies when the + * request carries no `Prefer` header. A client that sends + * `Prefer: respond-async` gets 202 with a run id and polls the run here; the + * result is the body the route would have answered with, except for extract, + * whose run carries `DocumentExtractRunResult`. * `POST /api/labs/ocr/extract` (web only) always answers 202. */ import type { ZodOpenApiObject } from "zod-openapi"; import { z } from "zod/v4"; import { + documentExtractRunResult, documentIndexResponse, documentSuggestResponse, documentSummaryResponse, @@ -32,6 +34,7 @@ const aiRun = z "LABS_OCR_EXTRACT", "DOCUMENT_SUMMARY", "DOCUMENT_SUGGEST", + "DOCUMENT_EXTRACT", ]), documentId: z .string() @@ -44,10 +47,11 @@ const aiRun = z ocrExtractResponse, documentSummaryResponse, documentSuggestResponse, + documentExtractRunResult, ]) .nullable() .describe( - "Present once `status` is `SUCCEEDED`: the body the queuing route answers with synchronously. `DOCUMENT_INDEX` carries `DocumentIndexResponse`, `LABS_OCR_EXTRACT` carries `OcrExtractResponse`, `DOCUMENT_SUMMARY` carries `DocumentSummaryResponse`, `DOCUMENT_SUGGEST` carries `DocumentSuggestResponse`. Decode by `kind`; treat an unknown `kind` as a run this client did not start.", + "Present once `status` is `SUCCEEDED`: the body the queuing route answers with synchronously. `DOCUMENT_INDEX` carries `DocumentIndexResponse`, `LABS_OCR_EXTRACT` carries `OcrExtractResponse`, `DOCUMENT_SUMMARY` carries `DocumentSummaryResponse`, `DOCUMENT_SUGGEST` carries `DocumentSuggestResponse`, `DOCUMENT_EXTRACT` carries `DocumentExtractRunResult` (not the extract route's detail body). Decode by `kind`; treat an unknown `kind` as a run this client did not start.", ), error: z .object({ diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index bc63f1941..4d413fd4f 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -287,6 +287,23 @@ const documentUploadEnvelope = dataEnvelope( "DocumentUploadEnvelope", ); +/** + * A `DOCUMENT_EXTRACT` run's result. Smaller than the synchronous extract + * body: the facts are staged on the document and read from it. + */ +export const documentExtractRunResult = z + .object({ + documentId: z.string(), + factsStaged: z + .number() + .int() + .describe( + "How many facts were staged PENDING for review. Read them from `GET /api/documents/inbound/{id}`; the confirm route stays the only write into the record.", + ), + status: statusEnum, + }) + .meta({ id: "DocumentExtractRunResult" }); + /** The suggest route's body; also a `DOCUMENT_SUGGEST` run's result. */ export const documentSuggestResponse = z .object({ @@ -919,7 +936,7 @@ export const inboundDocumentPaths: NonNullable = { tags: ["Documents"], summary: "Extract facts from a stored document", description: - 'Optional AI enhancement on an already-stored document. Runs the dedicated OCR/vision provider over the stored original and stages STRUCTURED FACTS for review. Answers under the `documentAi` capability (403 with the capability envelope when it is closed), then rate / budget gated. With no provider configured this returns 422 (`documents.inbound.providerUnsupported`, with `meta.capability` and `meta.reason = "no_provider"`) — the stored document is untouched, only the enhancement fails. Three modes: VISION (empty body) decrypts and scans the stored original (PDF needs an Anthropic vision provider); TEXT (`application/json`, opt-in local OCR) `{ mode: "text", text }` structures browser-OCR\'d text; STORED (`application/json`) `{ mode: "stored" }` structures the document\'s own stored extracted text (its content index) — the manual recovery for a skipped/failed automatic staging run, 422 `documents.inbound.notIndexed` when no stored text exists. Extraction reproduces what the document states — it never interprets. Nothing reaches the structured stores here; the confirm route is the only write path.', + 'Optional AI enhancement on an already-stored document. Runs the dedicated OCR/vision provider over the stored original and stages STRUCTURED FACTS for review. Answers under the `documentAi` capability (403 with the capability envelope when it is closed), then rate / budget gated. With no provider configured this returns 422 (`documents.inbound.providerUnsupported`, with `meta.capability` and `meta.reason = "no_provider"`) — the stored document is untouched, only the enhancement fails. Three modes: VISION (empty body) decrypts and scans the stored original (PDF needs an Anthropic vision provider); TEXT (`application/json`, opt-in local OCR) `{ mode: "text", text }` structures browser-OCR\'d text; STORED (`application/json`) `{ mode: "stored" }` structures the document\'s own stored extracted text (its content index) — the manual recovery for a skipped/failed automatic staging run, 422 `documents.inbound.notIndexed` when no stored text exists. Extraction reproduces what the document states — it never interprets. Nothing reaches the structured stores here; the confirm route is the only write path. With `Prefer: respond-async` (v1.40) the extraction runs in the background: after the same quick refusals the route answers 202 with a run id, a second request while an extraction of this document from the same input is queued or running answers with that run, and `GET /api/ai-runs/{id}` serves `DocumentExtractRunResult` (`{ documentId, factsStaged, status }`, not the document detail) as the run\'s `result`. Staging re-checks the document under a row lock, so a review finished while the read ran fails the run with `documents.inbound.alreadyPartlyConfirmed` (409) or `documents.inbound.alreadyConfirmed` (422) and leaves the review untouched; the synchronous form answers the same in that race.', parameters: [ { name: "id", @@ -927,6 +944,7 @@ export const inboundDocumentPaths: NonNullable = { required: true, schema: { type: "string" }, }, + preferRespondAsyncParameter, ], requestBody: { required: false, @@ -964,6 +982,7 @@ export const inboundDocumentPaths: NonNullable = { "Re-extraction refused: at least one fact on this document is already APPROVED. `meta.errorCode` = `documents.inbound.alreadyPartlyConfirmed`. Re-extracting would sever committed-record provenance and duplicate committed records, so the user must finish reviewing or discard the document first.", content: { "application/json": { schema: errorEnvelope } }, }, + ...aiRunAccepted, ...aiExtractionRefusals("documentAi"), ...stdResponses, }, diff --git a/tests/integration/document-ai-runs.test.ts b/tests/integration/document-ai-runs.test.ts index ee54eb1e3..2b82d6575 100644 --- a/tests/integration/document-ai-runs.test.ts +++ b/tests/integration/document-ai-runs.test.ts @@ -16,6 +16,9 @@ * anything reaches a model. * 6. The reaper fails a run no worker took (and hands its reservation back) * and deletes a finished run an hour after it ended. + * 8. A stored-text extraction stages its facts from the worker, and a + * review finished while the read ran is never overwritten: staging + * re-checks the document under a row lock and fails the run instead. * 7. Summary and suggestion runs (0370) round-trip the routes' own bodies; a * summary that is to be stored marks the document PENDING while queued, * READY when it lands and UNAVAILABLE when the run fails; a transient one @@ -93,6 +96,29 @@ vi.mock("@/lib/labs/ocr-extract", async (importOriginal) => ({ rows: [], })), })); +vi.mock("@/lib/documents/extract", async (importOriginal) => ({ + ...(await importOriginal()), + runInboundExtraction: vi.fn(async () => ({ + reportDate: "2027-01-10", + kind: "OTHER", + providerType: "anthropic", + facts: [ + { + factType: "OBSERVATION", + confidence: 0.95, + needsReview: false, + data: { label: "Hemoglobin", value: 13.9, unit: "g/dL" }, + provenance: { + sourceText: "Hemoglobin 13.9 g/dL", + anchored: true, + sourceOffset: 0, + page: null, + confidence: 0.95, + }, + }, + ], + })), +})); vi.mock("@/lib/documents/auto-stage-labs", () => ({ maybeAutoStageLabFacts: vi.fn(async () => ({ staged: false, @@ -105,6 +131,7 @@ import { runDocumentSummary, transcribeDocument, } from "@/lib/documents/describe"; +import { runInboundExtraction } from "@/lib/documents/extract"; import { runOcrExtraction } from "@/lib/labs/ocr-extract"; const PNG = Buffer.from( @@ -267,6 +294,7 @@ beforeEach(async () => { vi.mocked(runOcrExtraction).mockClear(); vi.mocked(runDocumentSummary).mockClear(); vi.mocked(runDocumentAssist).mockClear(); + vi.mocked(runInboundExtraction).mockClear(); }); describe("Read with AI in the background", () => { @@ -680,3 +708,143 @@ describe("filing suggestions in the background", () => { expect(foreignPoll.body.meta?.errorCode).toBe("aiRuns.notFound"); }); }); + +async function postStoredExtract(documentId: string, prefer: boolean) { + const { POST } = + await import("@/app/api/documents/inbound/[id]/extract/route"); + const { NextRequest } = await import("next/server"); + const headers: Record = { + "content-type": "application/json", + }; + if (prefer) headers.prefer = "respond-async"; + const res = await POST( + new NextRequest( + `http://localhost/api/documents/inbound/${documentId}/extract`, + { method: "POST", headers, body: JSON.stringify({ mode: "stored" }) }, + ) as never, + { params: Promise.resolve({ id: documentId }) } as never, + ); + return { status: res.status, body: (await res.json()) as Envelope }; +} + +async function indexedDocument(userId: string) { + const document = await storeDocument(userId); + const { upsertContentIndex } = await import("@/lib/documents/content-index"); + await upsertContentIndex({ + userId, + documentId: document.id, + text: "Hemoglobin 13.9 g/dL reference range 12-16", + source: "text-ocr", + providerType: null, + }); + return document; +} + +describe("fact extraction in the background", () => { + it("stages the facts in the worker and serves the count", async () => { + const user = await makeUser(); + await grantConsent(user.id); + const document = await indexedDocument(user.id); + await signIn(user.id); + + const queued = await postStoredExtract(document.id, true); + expect(queued.status).toBe(202); + const { runId } = queued.body.data as { runId: string }; + const stored = await getPrismaClient().documentAiRun.findUniqueOrThrow({ + where: { id: runId }, + }); + // The worker reads the content index itself; nothing is sealed. + expect(stored.inputEncrypted).toBeNull(); + expect(runInboundExtraction).not.toHaveBeenCalled(); + + expect(await work(runId)).toBe("succeeded"); + expect(runInboundExtraction).toHaveBeenCalledWith( + expect.objectContaining({ + ocrText: expect.stringContaining("Hemoglobin 13.9"), + }), + ); + const done = await poll(runId); + expect(done.body.data).toMatchObject({ + status: "SUCCEEDED", + kind: "DOCUMENT_EXTRACT", + result: { + documentId: document.id, + factsStaged: 1, + status: "EXTRACTED", + }, + }); + const facts = await getPrismaClient().extractedFact.findMany({ + where: { documentId: document.id }, + }); + expect(facts.map((f) => f.status)).toEqual(["PENDING"]); + }); + + it("never replaces a review that was finished while the read ran", async () => { + const user = await makeUser(); + await grantConsent(user.id); + const document = await indexedDocument(user.id); + await signIn(user.id); + + const queued = await postStoredExtract(document.id, true); + const { runId } = queued.body.data as { runId: string }; + + // The person approves a fact from an earlier staging before the worker + // gets to the run. + const { encryptFactData, encryptFactProvenance } = + await import("@/lib/documents/store"); + const approved = await getPrismaClient().extractedFact.create({ + data: { + documentId: document.id, + userId: user.id, + factType: "OBSERVATION", + status: "APPROVED", + confidence: 0.9, + needsReview: false, + dataEncrypted: encryptFactData({ + label: "Glucose", + value: 95, + unit: "mg/dL", + } as never), + provenanceEncrypted: encryptFactProvenance({ + sourceText: "Glucose 95", + anchored: true, + sourceOffset: 0, + page: null, + confidence: 0.9, + } as never), + }, + }); + + expect(await work(runId)).toBe("failed"); + const done = await poll(runId); + expect(done.body.data).toMatchObject({ + status: "FAILED", + error: { + code: "documents.inbound.alreadyPartlyConfirmed", + status: 409, + }, + }); + const facts = await getPrismaClient().extractedFact.findMany({ + where: { documentId: document.id }, + }); + expect(facts.map((f) => [f.id, f.status])).toEqual([ + [approved.id, "APPROVED"], + ]); + }); + + it("answers with the document detail, exactly as before, without the header", async () => { + const user = await makeUser(); + await grantConsent(user.id); + const document = await indexedDocument(user.id); + await signIn(user.id); + + const res = await postStoredExtract(document.id, false); + expect(res.status).toBe(200); + expect(res.body.data).toMatchObject({ + id: document.id, + status: "EXTRACTED", + }); + expect((res.body.data as { facts: unknown[] }).facts).toHaveLength(1); + expect(await getPrismaClient().documentAiRun.count()).toBe(0); + }); +}); From 9272daa59c58d7016b214b28054d6aa37bbbb439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 3 Oct 2026 21:24:12 +0200 Subject: [PATCH 07/14] feat(symptoms): log a symptom in one step and show it on the illness pages - A new capture kind "symptom" (illness module, illness write grant) in the bottom bar's picker and the dashboard Add menu. The sheet follows the measurement and mood forms: the person's symptoms as chips with the cycle sheet's dashed "new symptom" chip, the 0-10 slider with its anchors, the time prefilled with now, an optional note, and a "during" selector only while an illness episode is open. - "Your symptoms" on the illness page: one line per symptom (how often in 30 days, how strong at worst, when last), the last 30 days grouped by day, edit and delete per entry, rename, icon, hide and delete per symptom. Renders nothing until a symptom exists. - "Symptoms logged" on an episode's page: the entries filed against it or logged while it ran, the filed ones marked. This is the reader of the episode link. - Query keys live under the illness root, so either side's writes refresh both. --- .../__tests__/dashboard-header.test.tsx | 4 +- src/components/dashboard/dashboard-header.tsx | 17 +- .../dashboard/quick-entry-sheets.tsx | 25 +- .../illness/illness-episode-detail.tsx | 5 + src/components/illness/illness-view.tsx | 36 +- src/components/layout/capture-picker.tsx | 25 +- .../__tests__/symptom-entry-form.test.tsx | 114 ++++ src/components/symptoms/add-symptom-chip.tsx | 165 ++++++ .../symptoms/episode-symptoms-card.tsx | 135 +++++ .../symptoms/symptom-edit-sheets.tsx | 267 +++++++++ .../symptoms/symptom-entry-form.tsx | 395 +++++++++++++ src/components/symptoms/symptom-icons.ts | 79 +++ src/components/symptoms/symptoms-section.tsx | 524 ++++++++++++++++++ src/components/symptoms/use-symptoms.ts | 154 +++++ .../__tests__/surface-module-registry.test.ts | 1 + src/lib/modules/surface.ts | 2 + src/lib/query-keys/illness.ts | 11 + 17 files changed, 1940 insertions(+), 19 deletions(-) create mode 100644 src/components/symptoms/__tests__/symptom-entry-form.test.tsx create mode 100644 src/components/symptoms/add-symptom-chip.tsx create mode 100644 src/components/symptoms/episode-symptoms-card.tsx create mode 100644 src/components/symptoms/symptom-edit-sheets.tsx create mode 100644 src/components/symptoms/symptom-entry-form.tsx create mode 100644 src/components/symptoms/symptom-icons.ts create mode 100644 src/components/symptoms/symptoms-section.tsx create mode 100644 src/components/symptoms/use-symptoms.ts diff --git a/src/components/dashboard/__tests__/dashboard-header.test.tsx b/src/components/dashboard/__tests__/dashboard-header.test.tsx index 0e5579c44..ab0909ef2 100644 --- a/src/components/dashboard/__tests__/dashboard-header.test.tsx +++ b/src/components/dashboard/__tests__/dashboard-header.test.tsx @@ -153,7 +153,8 @@ describe(" — every entry follows its module, as the capture p { mood: false }, { medications: false }, { workouts: false }, - { mood: false, medications: false, workouts: false }, + { illness: false }, + { mood: false, medications: false, workouts: false, illness: false }, ]; for (const modules of cases) { modulesRef.value = modules; @@ -170,6 +171,7 @@ describe(" — every entry follows its module, as the capture p measurement: "Log measurement", medication: "Log medication intake", mood: "Log mood", + symptom: "Log symptom", workout: "Log workout", } as const; for (const kind of CAPTURE_KIND_ORDER) { diff --git a/src/components/dashboard/dashboard-header.tsx b/src/components/dashboard/dashboard-header.tsx index db6ad7cc0..c969eb495 100644 --- a/src/components/dashboard/dashboard-header.tsx +++ b/src/components/dashboard/dashboard-header.tsx @@ -12,7 +12,15 @@ */ import { useMemo } from "react"; import Link from "next/link"; -import { Activity, Footprints, Pill, Plus, Waves, Wrench } from "lucide-react"; +import { + Activity, + Footprints, + Pill, + Plus, + Stethoscope, + Waves, + Wrench, +} from "lucide-react"; import { Button } from "@/components/ui/button"; import { DropdownMenu, @@ -62,6 +70,7 @@ export function DashboardHeader({ const canAddMeasurement = offered.includes("measurement"); const canAddIntake = offered.includes("medication"); const canAddMood = offered.includes("mood"); + const canAddSymptom = offered.includes("symptom"); const canAddWorkout = offered.includes("workout"); const canAddAnything = offered.length > 0; @@ -198,6 +207,12 @@ export function DashboardHeader({ {t("dashboard.quickAddMedicationIntake")} )} + {canAddSymptom && ( + onQuickEntry("symptom")}> + + )} {canAddWorkout && ( onQuickEntry("workout")}>