From 58141cb48bf9a9faecefe2145e5d9a18114f2b80 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:24:48 +0000 Subject: [PATCH 01/41] fix(documents): date an undated lab value by its document A lab report usually prints the collection date once, in its header. The extraction puts that date on the document (reportDate) and leaves each value's own date empty, and the confirm step filed every such value under the moment it was confirmed, so an old report showed up as today's results. The confirm route now reads the document's report date and filing date with the document it already loads, and an undated value takes the report date, else the filing date, else now: the order the day view places a document by. A stated date, a lab value's or a condition's onset, now counts only when it names a real calendar day. 2021-02-30 used to be stored as 2 March, and 2021-13-01 became an invalid date that failed the whole confirm. --- .../documents/inbound/[id]/confirm/route.ts | 16 +- src/lib/documents/__tests__/commit.test.ts | 189 +++++++++++++++++- src/lib/documents/commit.ts | 52 ++++- 3 files changed, 244 insertions(+), 13 deletions(-) diff --git a/src/app/api/documents/inbound/[id]/confirm/route.ts b/src/app/api/documents/inbound/[id]/confirm/route.ts index 564af8e62..919398228 100644 --- a/src/app/api/documents/inbound/[id]/confirm/route.ts +++ b/src/app/api/documents/inbound/[id]/confirm/route.ts @@ -43,7 +43,8 @@ async function confirmDocument( const document = await prisma.inboundDocument.findFirst({ where: { id, userId: user.id, deletedAt: null }, - select: { id: true }, + // The two dates date a value that states none of its own. + select: { id: true, reportDate: true, documentDate: true }, }); if (!document) { return apiError("Document not found", 404, { @@ -94,10 +95,15 @@ async function confirmDocument( try { // v1.43 — an OBSERVATION may name the marker the person chose // on the review screen; the label is then remembered as its alias. - const ref = await commitApprovedFact(user.id, fact, { - biomarkerId: decision.biomarkerId, - rememberAlias: decision.rememberAlias, - }); + const ref = await commitApprovedFact( + user.id, + fact, + { + biomarkerId: decision.biomarkerId, + rememberAlias: decision.rememberAlias, + }, + document, + ); await prisma.extractedFact.update({ where: { id: fact.id }, data: { diff --git a/src/lib/documents/__tests__/commit.test.ts b/src/lib/documents/__tests__/commit.test.ts index 591ebae47..81deb22dd 100644 --- a/src/lib/documents/__tests__/commit.test.ts +++ b/src/lib/documents/__tests__/commit.test.ts @@ -12,6 +12,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("@/lib/db", () => ({ prisma: { labResult: { create: vi.fn() }, + illnessEpisode: { create: vi.fn() }, }, })); @@ -27,8 +28,16 @@ import { commitApprovedFact, FactCommitError } from "@/lib/documents/commit"; import { prisma } from "@/lib/db"; import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store"; import { decryptFactData } from "@/lib/documents/store"; -import type { ExtractedFact } from "@/generated/prisma/client"; +import type { + ExtractedFact, + IllnessEpisode, + LabResult, +} from "@/generated/prisma/client"; import { userDayKey } from "@/lib/tz/format"; +import type { + ConditionFactData, + ObservationFactData, +} from "@/lib/validations/inbound-documents"; function observationFact(): ExtractedFact { return { @@ -213,3 +222,181 @@ describe("commitApprovedFact — OBSERVATION unit integrity", () => { ).resolves.toEqual({ recordType: "labResult", recordId: "lr-2" }); }); }); + +describe("commitApprovedFact — OBSERVATION date when the value states none", () => { + // A lab report that prints the collection date once in its header carries + // no date beside each value. Mutation check: drop the document day from + // `takenAt` → the report-date, filing-date and impossible-date cases file + // the reading under today. + function undated(effectiveDate: string | null = null) { + const data: ObservationFactData = { + label: "Ferritin", + code: null, + codeSystem: null, + value: 80, + valueText: null, + unit: "ng/mL", + referenceLow: null, + referenceHigh: null, + effectiveDate, + }; + vi.mocked(decryptFactData).mockReturnValue(data); + vi.mocked(resolveOrMintBiomarker).mockResolvedValue({ + id: "bm-3", + name: "Ferritin", + unit: "ng/mL", + lowerBound: null, + upperBound: null, + panel: null, + analyteKey: null, + }); + vi.mocked(prisma.labResult.create).mockResolvedValue({ + id: "lr-3", + } as LabResult); + } + + /** The dates as the confirm route reads them off the stored document. */ + function dates(reportDate: string | null, documentDate: string | null) { + return { + // An older report date sits at UTC midnight, newer ones at noon UTC. + reportDate: reportDate ? new Date(`${reportDate}T00:00:00.000Z`) : null, + documentDate: documentDate + ? new Date(`${documentDate}T12:00:00.000Z`) + : null, + }; + } + + function storedTakenAt(): Date { + return vi.mocked(prisma.labResult.create).mock.calls[0]![0].data + .takenAt as Date; + } + + it("keeps the value's own date", async () => { + undated("2021-03-04"); + + await commitApprovedFact( + "user-1", + observationFact(), + {}, + dates("2021-03-01", "2026-10-20"), + ); + + expect(storedTakenAt().toISOString()).toBe("2021-03-04T12:00:00.000Z"); + }); + + it("takes the report date the document states over its filing date", async () => { + undated(); + + await commitApprovedFact( + "user-1", + observationFact(), + {}, + dates("2021-03-01", "2026-10-20"), + ); + + expect(storedTakenAt().toISOString()).toBe("2021-03-01T12:00:00.000Z"); + }); + + it("takes the filing date when the document states no report date", async () => { + undated(); + + await commitApprovedFact( + "user-1", + observationFact(), + {}, + dates(null, "2021-02-15"), + ); + + expect(storedTakenAt().toISOString()).toBe("2021-02-15T12:00:00.000Z"); + }); + + it.each([ + ["not a date", "March 2021"], + ["a day the month does not have", "2021-02-30"], + ["a month that does not exist", "2021-13-01"], + ])( + "reads a value whose own date is %s as undated", + async (_label, effectiveDate) => { + undated(effectiveDate); + + await commitApprovedFact( + "user-1", + observationFact(), + {}, + dates("2021-03-01", null), + ); + + // Not 2 March for 30 February, and no Invalid Date for month 13. + expect(storedTakenAt().toISOString()).toBe("2021-03-01T12:00:00.000Z"); + }, + ); + + it("files the reading under now when neither the value nor the document has a date", async () => { + undated(); + + const before = Date.now(); + await commitApprovedFact( + "user-1", + observationFact(), + {}, + dates(null, null), + ); + const after = Date.now(); + + const takenAt = storedTakenAt().getTime(); + expect(takenAt).toBeGreaterThanOrEqual(before); + expect(takenAt).toBeLessThanOrEqual(after); + }); +}); + +describe("commitApprovedFact — CONDITION onset", () => { + // The onset goes through the same stated-day check as a lab value's date. + function condition(onsetDate: string | null) { + const data: ConditionFactData = { + label: "Iron deficiency", + code: null, + codeSystem: null, + clinicalStatus: null, + verificationStatus: null, + onsetDate, + }; + vi.mocked(decryptFactData).mockReturnValue(data); + vi.mocked(prisma.illnessEpisode.create).mockResolvedValue({ + id: "ep-1", + } as IllnessEpisode); + } + + function conditionFact(): ExtractedFact { + return { + id: "fact-2", + factType: "CONDITION", + dataEncrypted: new Uint8Array(), + } as unknown as ExtractedFact; + } + + function storedOnset(): Date { + return vi.mocked(prisma.illnessEpisode.create).mock.calls[0]![0].data + .onsetAt as Date; + } + + it("keeps a stated onset day", async () => { + condition("2021-03-04"); + + await commitApprovedFact("user-1", conditionFact()); + + expect(storedOnset().toISOString()).toBe("2021-03-04T12:00:00.000Z"); + }); + + it("files an onset that is not a real day under now, not an invalid date", async () => { + // 2021-13-01 used to become an invalid date that failed the whole confirm. + condition("2021-13-01"); + + const before = Date.now(); + await commitApprovedFact("user-1", conditionFact()); + const after = Date.now(); + + const onset = storedOnset().getTime(); + expect(onset).toBeGreaterThanOrEqual(before); + expect(onset).toBeLessThanOrEqual(after); + }); +}); diff --git a/src/lib/documents/commit.ts b/src/lib/documents/commit.ts index db39f3d31..17bae2712 100644 --- a/src/lib/documents/commit.ts +++ b/src/lib/documents/commit.ts @@ -44,7 +44,11 @@ import type { MedicationStatementFactData, ObservationFactData, } from "@/lib/validations/inbound-documents"; -import { dateOnlyAtNoonUtc, isDateOnlyKey } from "@/lib/tz/date-only"; +import { + dateOnlyAtNoonUtc, + dateOnlyKey, + isCalendarDateKey, +} from "@/lib/tz/date-only"; /** A per-fact commit failure the caller maps to a per-fact 422 entry. */ export class FactCommitError extends Error { @@ -61,14 +65,41 @@ export interface CommittedRecordRef { recordId: string; } +/** A stated day, or null when it is not a real calendar date. */ +function statedDay(date: string | null): string | null { + return date && isCalendarDateKey(date) ? date : null; +} + /** * A stated YYYY-MM-DD as the instant a date-only value is stored at (noon - * UTC), or now when the document states none. UTC midnight read back as the + * UTC), else the fallback day, else now. UTC midnight read back as the * previous day anywhere west of UTC. */ -function statedDateOrNow(date: string | null): Date { - if (date && isDateOnlyKey(date)) return dateOnlyAtNoonUtc(date); - return new Date(); +function statedDateOrNow( + date: string | null, + fallback: string | null = null, +): Date { + const day = statedDay(date) ?? fallback; + return day ? dateOnlyAtNoonUtc(day) : new Date(); +} + +/** The two dates a stored document carries, as the confirm route reads them. */ +export interface FactDocumentDates { + reportDate: Date | null; + documentDate: Date | null; +} + +/** + * The day a value from this document took place when it states none of its + * own: the report date the model read off the document, else the document's + * filing date, the same order the day view and the timeline place a document + * by. A lab report that prints the collection date once in its header carries + * no date beside each value, and every reading used to be filed under the day + * it was confirmed. + */ +function documentDay(dates: FactDocumentDates | null): string | null { + const day = dates?.reportDate ?? dates?.documentDate ?? null; + return day ? dateOnlyKey(day) : null; } /** @@ -84,6 +115,7 @@ async function commitObservation( userId: string, data: ObservationFactData, choice: ObservationChoice = {}, + fallbackDay: string | null = null, ): Promise { const isQualitative = typeof data.value !== "number"; // A numeric reading needs a unit so the catalog has something to mint with — @@ -228,7 +260,7 @@ async function commitObservation( sourceReferenceLow: storedSourceLow, sourceReferenceHigh: storedSourceHigh, sourceReferenceText: isQualitative ? null : sourceText, - takenAt: statedDateOrNow(data.effectiveDate), + takenAt: statedDateOrNow(data.effectiveDate, fallbackDay), source: "DOCUMENT", noteEncrypted: null, }, @@ -317,12 +349,18 @@ export async function commitApprovedFact( userId: string, fact: ExtractedFact, choice: ObservationChoice = {}, + documentDates: FactDocumentDates | null = null, ): Promise { // Decrypt the staged payload at confirm time to write it into the normal // structured store. Fail-closed: a bad key id throws and aborts the commit. const data = decryptFactData(fact.dataEncrypted); if (fact.factType === "OBSERVATION") { - return commitObservation(userId, data as ObservationFactData, choice); + return commitObservation( + userId, + data as ObservationFactData, + choice, + documentDay(documentDates), + ); } if (fact.factType === "CONDITION") { return commitCondition(userId, data as ConditionFactData); From 4ca7573f9093618dea7b601d850f4f2920b91d76 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:24:49 +0000 Subject: [PATCH 02/41] test(documents): pin an undated lab value's date through the confirm route Against a real Postgres and the real confirm route: an undated value takes the report date over the filing date, a value's own date is kept, an impossible own date counts as none, and a document without a report date dates the value by its filing date. --- .../document-undated-lab-value.test.ts | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 tests/integration/document-undated-lab-value.test.ts diff --git a/tests/integration/document-undated-lab-value.test.ts b/tests/integration/document-undated-lab-value.test.ts new file mode 100644 index 000000000..154bb4619 --- /dev/null +++ b/tests/integration/document-undated-lab-value.test.ts @@ -0,0 +1,222 @@ +/** + * A lab value without its own date, confirmed from a stored document, against + * a real Postgres and through the real confirm route. + * + * A lab report usually prints the collection date once, in its header. The + * extraction puts that date on the document (`reportDate`) and leaves each + * value's own date empty, so the confirm step has to date the value by its + * document. It used to file every such reading under the moment it was + * confirmed, so a report from years ago showed up as today's results. + * + * The route reads the document's two dates once and hands them to the commit + * step: the report date first, else the filing date (`documentDate`), the order + * the day view places a document by. A value's own date still wins, but only + * when it names a real calendar day. + */ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +const USER_ID = "user-undated-lab-value"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + const prisma = getPrismaClient(); + await prisma.user.create({ + data: { + id: USER_ID, + username: "undated-lab-value", + email: "undated-lab-value@example.test", + modulePreferencesJson: { inboundDocuments: true, labs: true }, + }, + }); + const session = await prisma.session.create({ + data: { userId: USER_ID, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + for (const [name, unit] of [ + ["Ferritin", "ng/mL"], + ["Glucose", "mmol/L"], + ["Haemoglobin", "g/L"], + ]) { + await prisma.biomarker.create({ data: { userId: USER_ID, name, unit } }); + } +}); + +const noon = (day: string) => new Date(`${day}T12:00:00.000Z`); + +async function storedDocument(dates: { + reportDate: string | null; + documentDate: string | null; +}) { + return getPrismaClient().inboundDocument.create({ + data: { + userId: USER_ID, + filename: "befund.pdf", + mimeType: "application/pdf", + byteSize: 1024, + contentEncrypted: Buffer.from("stand-in for the stored bytes"), + status: "EXTRACTED", + reportDate: dates.reportDate ? noon(dates.reportDate) : null, + documentDate: dates.documentDate ? noon(dates.documentDate) : null, + }, + }); +} + +async function stagedValue( + documentId: string, + label: string, + value: number, + unit: string, + effectiveDate: string | null, +) { + const { encryptFactData, encryptFactProvenance } = + await import("@/lib/documents/store"); + return getPrismaClient().extractedFact.create({ + data: { + documentId, + userId: USER_ID, + factType: "OBSERVATION", + status: "PENDING", + confidence: 0.95, + needsReview: false, + dataEncrypted: encryptFactData({ + label, + code: null, + codeSystem: null, + value, + valueText: null, + unit, + referenceLow: null, + referenceHigh: null, + referenceText: null, + effectiveDate, + }), + provenanceEncrypted: encryptFactProvenance({ + sourceText: `${label} ${value} ${unit}`, + anchored: true, + sourceOffset: 0, + page: 0, + confidence: 0.95, + }), + }, + }); +} + +async function confirm(documentId: string, factIds: string[]) { + const { POST } = + await import("@/app/api/documents/inbound/[id]/confirm/route"); + const res = await POST( + new NextRequest( + `http://localhost/api/documents/inbound/${documentId}/confirm`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + decisions: factIds.map((factId) => ({ factId, action: "approve" })), + }), + }, + ), + { params: Promise.resolve({ id: documentId }) }, + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data.failed).toEqual([]); + expect(body.data.approved).toHaveLength(factIds.length); + return body.data.approved as { factId: string; recordId: string }[]; +} + +async function takenAtOf(recordId: string) { + const row = await getPrismaClient().labResult.findUniqueOrThrow({ + where: { id: recordId }, + }); + return row.takenAt.toISOString(); +} + +describe("an undated lab value is dated by its document", () => { + it("takes the report date over the filing date, keeps a value's own date, and ignores an impossible one", async () => { + const today = new Date().toISOString().slice(0, 10); + // Collection date printed once in the header; filed today (the upload-day + // default for documentDate). + const doc = await storedDocument({ + reportDate: "2021-03-01", + documentDate: today, + }); + const undated = await stagedValue(doc.id, "Ferritin", 80, "ng/mL", null); + const ownDate = await stagedValue( + doc.id, + "Glucose", + 5.1, + "mmol/L", + "2021-02-20", + ); + const impossible = await stagedValue( + doc.id, + "Haemoglobin", + 142, + "g/L", + "2021-02-30", + ); + + const approved = await confirm(doc.id, [ + undated.id, + ownDate.id, + impossible.id, + ]); + const recordOf = (factId: string) => + approved.find((a) => a.factId === factId)!.recordId; + + expect(await takenAtOf(recordOf(undated.id))).toBe( + "2021-03-01T12:00:00.000Z", + ); + expect(await takenAtOf(recordOf(ownDate.id))).toBe( + "2021-02-20T12:00:00.000Z", + ); + // Not 2 March: a day the month does not have is no date at all. + expect(await takenAtOf(recordOf(impossible.id))).toBe( + "2021-03-01T12:00:00.000Z", + ); + }); + + it("takes the filing date when the document has no report date", async () => { + const doc = await storedDocument({ + reportDate: null, + documentDate: "2021-02-15", + }); + const undated = await stagedValue(doc.id, "Ferritin", 80, "ng/mL", null); + + const [approved] = await confirm(doc.id, [undated.id]); + + expect(await takenAtOf(approved!.recordId)).toBe( + "2021-02-15T12:00:00.000Z", + ); + }); +}); From d88313e92996cf9aa2920a3c4b519a2f01f1640d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 29 Oct 2026 08:49:40 +0100 Subject: [PATCH 03/41] fix(labs): move document lab values filed under their confirmation day Before the confirm step dated an undated value by its document, a lab value confirmed from a report that prints its collection date only in the header was stored with takenAt set to the moment of confirmation. The fix covers new confirms; the values already filed that way stay on the wrong day until something moves them. A one-shot maintenance job, sent at boot only while there is something to repair, moves exactly those values to the document's report date (else its filing date) at noon UTC, the same rule the confirm step now uses. A value qualifies only when the chain is certain: written by the document confirm path, named by exactly one approved observation fact, takenAt still within five seconds of its own createdAt, no calendar date of its own in the decrypted fact, and a document day that differs from the day the value is filed under in the owner's zone. A value re-dated since, one really taken on its confirmation day, or one whose document has no date is left alone. A moved value no longer sits at its creation moment, so the pass converges and a second run changes nothing. The job definition lives in a new registry file beside maintenance.ts so that file stays under the size ratchet. --- .../labs-module-ai-gate-guard.test.ts | 2 + src/lib/documents/commit.ts | 4 +- .../__tests__/queue-policy-tables.test.ts | 9 + .../jobs/lab-document-date-repair-queue.ts | 10 + src/lib/jobs/lab-document-date-repair.ts | 234 ++++++++++++++++++ src/lib/jobs/queue-runtime.ts | 3 + .../jobs/registry/maintenance-discovery.ts | 9 + src/lib/jobs/registry/maintenance-repairs.ts | 25 ++ src/lib/jobs/registry/maintenance.ts | 2 + src/lib/jobs/subsystem-surface.ts | 1 + .../lab-document-date-repair.test.ts | 232 +++++++++++++++++ 11 files changed, 529 insertions(+), 2 deletions(-) create mode 100644 src/lib/jobs/lab-document-date-repair-queue.ts create mode 100644 src/lib/jobs/lab-document-date-repair.ts create mode 100644 src/lib/jobs/registry/maintenance-repairs.ts create mode 100644 tests/integration/lab-document-date-repair.test.ts diff --git a/src/__tests__/labs-module-ai-gate-guard.test.ts b/src/__tests__/labs-module-ai-gate-guard.test.ts index d798e69ef..70ca38c36 100644 --- a/src/__tests__/labs-module-ai-gate-guard.test.ts +++ b/src/__tests__/labs-module-ai-gate-guard.test.ts @@ -65,6 +65,8 @@ const NOT_MODEL_INPUT: Record = { "links stored rows to their biomarker; no model is involved", "src/lib/jobs/lab-biomarker-backfill.ts#enqueueBootTimeLabBiomarkerBackfill": "finds the accounts whose rows need linking; no model is involved", + "src/lib/jobs/lab-document-date-repair.ts#candidateQuery": + "finds document lab values filed at their confirmation moment to re-date them; no model is involved", "src/lib/jobs/soft-delete-retention.ts#SOFT_DELETE_RETENTION": "purges rows past the trash window; no model is involved", "src/lib/insights/correlation-channel-series.ts#fetchLabDraws": diff --git a/src/lib/documents/commit.ts b/src/lib/documents/commit.ts index 17bae2712..97633752b 100644 --- a/src/lib/documents/commit.ts +++ b/src/lib/documents/commit.ts @@ -66,7 +66,7 @@ export interface CommittedRecordRef { } /** A stated day, or null when it is not a real calendar date. */ -function statedDay(date: string | null): string | null { +export function statedDay(date: string | null): string | null { return date && isCalendarDateKey(date) ? date : null; } @@ -97,7 +97,7 @@ export interface FactDocumentDates { * no date beside each value, and every reading used to be filed under the day * it was confirmed. */ -function documentDay(dates: FactDocumentDates | null): string | null { +export function documentDay(dates: FactDocumentDates | null): string | null { const day = dates?.reportDate ?? dates?.documentDate ?? null; return day ? dateOnlyKey(day) : null; } diff --git a/src/lib/jobs/__tests__/queue-policy-tables.test.ts b/src/lib/jobs/__tests__/queue-policy-tables.test.ts index 7a728b6fb..41c5b2f3d 100644 --- a/src/lib/jobs/__tests__/queue-policy-tables.test.ts +++ b/src/lib/jobs/__tests__/queue-policy-tables.test.ts @@ -112,6 +112,15 @@ describe("maintenance group queue policies", () => { expectPolicy("maintenance", queue, policy); }); + // One-shot repairs on a fixed singleton key: a boot during a run must be + // able to queue the next pass, so only queued duplicates collapse. + it.each([["lab-document-date-repair", "short"]])( + "%s is %s", + (queue, policy) => { + expectPolicy("maintenance", queue, policy); + }, + ); + it("environment-fetch is deliberately left on standard", () => { // Its explicit-range backfill sends with no key at all, on purpose, so any // policy would collapse two different requested date ranges onto the shared diff --git a/src/lib/jobs/lab-document-date-repair-queue.ts b/src/lib/jobs/lab-document-date-repair-queue.ts new file mode 100644 index 000000000..34ccea6e6 --- /dev/null +++ b/src/lib/jobs/lab-document-date-repair-queue.ts @@ -0,0 +1,10 @@ +/** + * The queue facts of `lab-document-date-repair.ts`: its name and its one + * singleton key. They live apart from the job so the worker registry and the + * boot discovery can read them without loading it. + */ +export const LAB_DOCUMENT_DATE_REPAIR_QUEUE = "lab-document-date-repair"; + +/** One pass covers every account, so there is only ever one job. */ +export const LAB_DOCUMENT_DATE_REPAIR_SINGLETON_KEY = + "lab-document-date-repair"; diff --git a/src/lib/jobs/lab-document-date-repair.ts b/src/lib/jobs/lab-document-date-repair.ts new file mode 100644 index 000000000..57edb24e5 --- /dev/null +++ b/src/lib/jobs/lab-document-date-repair.ts @@ -0,0 +1,234 @@ +/** + * One-shot repair: lab values confirmed from a document before the confirm + * step learned to date an undated value by its document. + * + * A lab report usually prints the collection date once, in its header. The + * extraction stores that date on the document (`reportDate`) and leaves each + * value's own date empty, and the confirm step used to file such a value + * under the moment it was confirmed. The confirm step now takes the + * document's day (`documentDay` in `src/lib/documents/commit.ts`); this pass + * moves the values it filed wrongly before, by the same rule. + * + * A value is moved only when every link in the chain is certain: + * - the lab result was written by the document confirm path + * (`source = 'DOCUMENT'`, which no other writer sets) and is live; + * - exactly one APPROVED observation fact names it as its committed record, + * so the document it came from is known; + * - its `takenAt` still sits within a few seconds of its own `createdAt`, + * which is what a value filed under its confirmation moment looks like + * (both were stamped in the same create). A value the person re-dated + * since, or one that carried its own date, sits elsewhere; + * - the staged fact states no calendar date of its own (decrypted here; + * SQL cannot see it); + * - the document has a report date, else a filing date, and that day is + * not the day the value is filed under in the owner's time zone. A value + * really taken on the day it was confirmed stays where it is. + * + * The moved value lands at noon UTC on the document's day, the anchor every + * date-only lab value uses. Idempotent: a moved value no longer sits at its + * creation moment, so a second run finds nothing, and a boot with nothing to + * repair costs one query. + */ +import type { Job } from "pg-boss"; + +import { Prisma, type PrismaClient } from "@/generated/prisma/client"; +import { activeAccountSql } from "@/lib/account-deletion/active-account"; +import { invalidateUserHealthScore } from "@/lib/cache/invalidate"; +import { prisma as defaultPrisma } from "@/lib/db"; +import { documentDay, statedDay } from "@/lib/documents/commit"; +import { decryptFactData } from "@/lib/documents/store"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { workerLog } from "@/lib/jobs/reminder/shared"; +import { annotate } from "@/lib/logging/context"; +import { withBackgroundEvent } from "@/lib/logging/background"; +import { dateOnlyAtNoonUtc } from "@/lib/tz/date-only"; +import { userDayKey } from "@/lib/tz/format"; +import { resolveUserTimezone } from "@/lib/tz/resolver"; +import type { ObservationFactData } from "@/lib/validations/inbound-documents"; + +import { + LAB_DOCUMENT_DATE_REPAIR_QUEUE, + LAB_DOCUMENT_DATE_REPAIR_SINGLETON_KEY, +} from "./lab-document-date-repair-queue"; + +/** + * How far `takenAt` may sit from `createdAt` and still count as the moment of + * confirmation. The two are stamped a few milliseconds apart in one create; a + * few seconds leaves room for a clock step and nothing a person would type. + */ +const CONFIRM_MOMENT_TOLERANCE_SECONDS = 5; + +interface CandidateRow { + id: string; + userId: string; + takenAt: Date; + dataEncrypted: Uint8Array; + reportDate: Date | null; + documentDate: Date | null; +} + +/** + * The SQL half of the rule. `limit` turns it into the boot's existence + * check. `.planning/v1.46.2-fragments/lab-repair-count.sql` is the same + * predicate as a read-only count. + */ +function candidateQuery(limit: number | null): Prisma.Sql { + // The day comparison runs here as well as in the repair, so a value that + // really was taken on its confirmation day drops out of the boot check and + // the pass converges to an empty set. A zone Postgres does not know is read + // as UTC rather than failing the query; the repair re-checks every row in + // the zone the app resolves. + return Prisma.sql` + WITH "known_zones" AS (SELECT "name" FROM pg_timezone_names) + SELECT r."id", + r."user_id" AS "userId", + r."taken_at" AS "takenAt", + f."data_encrypted" AS "dataEncrypted", + d."report_date" AS "reportDate", + d."document_date" AS "documentDate" + FROM "lab_results" r + JOIN "extracted_facts" f + ON f."committed_record_id" = r."id" + AND f."committed_record_type" = 'labResult' + AND f."status" = 'APPROVED' + AND f."fact_type" = 'OBSERVATION' + AND f."user_id" = r."user_id" + JOIN "inbound_documents" d + ON d."id" = f."document_id" + AND d."user_id" = r."user_id" + JOIN "users" u ON u."id" = r."user_id" + WHERE r."source" = 'DOCUMENT' + AND r."deleted_at" IS NULL + AND abs(extract(epoch FROM (r."taken_at" - r."created_at"))) <= ${CONFIRM_MOMENT_TOLERANCE_SECONDS} + AND COALESCE(d."report_date", d."document_date") IS NOT NULL + AND to_char(COALESCE(d."report_date", d."document_date"), 'YYYY-MM-DD') + <> to_char( + (r."taken_at" AT TIME ZONE 'UTC') AT TIME ZONE ( + CASE WHEN u."timezone" IN (SELECT "name" FROM "known_zones") + THEN u."timezone" ELSE 'UTC' END + ), + 'YYYY-MM-DD' + ) + AND ( + SELECT count(*) FROM "extracted_facts" other + WHERE other."committed_record_id" = r."id" + AND other."committed_record_type" = 'labResult' + ) = 1 + AND ${activeAccountSql('r."user_id"')} + ORDER BY r."id" + ${limit === null ? Prisma.empty : Prisma.sql`LIMIT ${limit}`} + `; +} + +export interface LabDocumentDateRepairResult { + /** Rows the SQL predicate returned. */ + scanned: number; + /** Rows moved to their document's day. */ + updated: number; + /** Rows left alone: an own date, or already on the document's day. */ + unchanged: number; +} + +/** The repair itself, across every account. */ +export async function repairUndatedDocumentLabValues( + prisma: PrismaClient = defaultPrisma, +): Promise { + const rows = await prisma.$queryRaw(candidateQuery(null)); + const touchedUsers = new Set(); + let updated = 0; + let unchanged = 0; + + for (const row of rows) { + const data = decryptFactData(row.dataEncrypted) as ObservationFactData; + // A value with a date of its own is never touched, whatever its row says. + if (statedDay(data.effectiveDate ?? null) !== null) { + unchanged += 1; + continue; + } + const day = documentDay({ + reportDate: row.reportDate, + documentDate: row.documentDate, + }); + const tz = await resolveUserTimezone(row.userId); + if (!day || userDayKey(row.takenAt, tz) === day) { + unchanged += 1; + continue; + } + // Conditional on the value read: a re-date that landed since the read + // wins over the repair. + const moved = await prisma.labResult.updateMany({ + where: { + id: row.id, + userId: row.userId, + takenAt: row.takenAt, + deletedAt: null, + }, + data: { takenAt: dateOnlyAtNoonUtc(day) }, + }); + if (moved.count === 0) { + unchanged += 1; + continue; + } + updated += 1; + touchedUsers.add(row.userId); + } + + for (const userId of touchedUsers) invalidateUserHealthScore(userId); + return { scanned: rows.length, updated, unchanged }; +} + +/** The queue handler: one run over every account. */ +export async function handleLabDocumentDateRepair( + _jobs: Job>[], + prisma: PrismaClient = defaultPrisma, +): Promise { + return withBackgroundEvent("job.lab_document_date_repair", async () => { + let result: LabDocumentDateRepairResult; + try { + result = await repairUndatedDocumentLabValues(prisma); + } catch (err) { + workerLog("error", "[lab-document-date-repair] failed", err); + return jobFailed("lab document date repair failed", err); + } + workerLog( + "info", + `[lab-document-date-repair] scanned=${result.scanned} updated=${result.updated} unchanged=${result.unchanged}`, + ); + annotate({ + action: { name: "labs.documentDate.repair" }, + meta: { ...result }, + }); + return jobDone({ + candidates_scanned: result.scanned, + updated: result.updated, + unchanged: result.unchanged, + }); + }); +} + +/** + * Boot discovery: one existence query, and the one job only when there is + * something to repair. + */ +export async function enqueueBootTimeLabDocumentDateRepair( + prisma: PrismaClient = defaultPrisma, +): Promise<{ enqueued: boolean; error: string | null }> { + const boss = getGlobalBoss(); + if (!boss) return { enqueued: false, error: null }; + try { + const any = await prisma.$queryRaw(candidateQuery(1)); + if (any.length === 0) return { enqueued: false, error: null }; + const id = await boss.send( + LAB_DOCUMENT_DATE_REPAIR_QUEUE, + {}, + { singletonKey: LAB_DOCUMENT_DATE_REPAIR_SINGLETON_KEY, retryLimit: 2 }, + ); + return { enqueued: id !== null, error: null }; + } catch (err) { + return { + enqueued: false, + error: err instanceof Error ? err.message : String(err), + }; + } +} diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 49fc6d154..7d6a3ae9d 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -485,6 +485,9 @@ export const QUEUE_RUNTIME: Readonly> = { }, why: "One account's folded means, a day at a time; stops at three quarters of the default expiry and sends a follow-up that resumes at the next day.", }, + "lab-document-date-repair": short( + "Reads the document lab values still filed at their confirmation moment, a handful per account, and moves each with one update.", + ), "measurement-maintenance": { runtime: "long", expireInSeconds: MEASUREMENT_MAINTENANCE_SEND_OPTIONS.expireInSeconds, diff --git a/src/lib/jobs/registry/maintenance-discovery.ts b/src/lib/jobs/registry/maintenance-discovery.ts index c0f2c41e2..2de44262a 100644 --- a/src/lib/jobs/registry/maintenance-discovery.ts +++ b/src/lib/jobs/registry/maintenance-discovery.ts @@ -77,6 +77,15 @@ export const MAINTENANCE_DISCOVERY: readonly BootDiscovery[] = [ await import("@/lib/jobs/measurement-fold-repair") ).enqueueBootTimeMeasurementFoldRepair(), }, + // Converges to nothing once the misdated values are moved; a boot with + // none costs one query. + { + name: "lab-document-date-repair", + run: async () => + ( + await import("@/lib/jobs/lab-document-date-repair") + ).enqueueBootTimeLabDocumentDateRepair(), + }, // Self-limiting and idempotent; a boot with no backlog costs one query. { name: "compaction-tombstone-purge", diff --git a/src/lib/jobs/registry/maintenance-repairs.ts b/src/lib/jobs/registry/maintenance-repairs.ts new file mode 100644 index 000000000..2d3224e86 --- /dev/null +++ b/src/lib/jobs/registry/maintenance-repairs.ts @@ -0,0 +1,25 @@ +/** + * One-shot data repairs of the maintenance group, kept beside + * `maintenance.ts` so that file stays under the size ratchet. Each is sent + * by its boot discovery in `maintenance-discovery.ts` only while it has + * something to repair. + */ +import { LAB_DOCUMENT_DATE_REPAIR_QUEUE } from "@/lib/jobs/lab-document-date-repair-queue"; + +import { defineJob, type JobDefinition } from "./define-job"; + +export const MAINTENANCE_REPAIR_JOBS: readonly JobDefinition[] = [ + // Document lab values filed under their confirmation moment. + defineJob({ + queue: LAB_DOCUMENT_DATE_REPAIR_QUEUE, + policy: { + policy: "short", + reason: + "Fixed singleton key; collapse queued duplicates only, so a boot during a run can queue the next one.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/lab-document-date-repair")) + .handleLabDocumentDateRepair, + }), +]; diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts index 587123022..574d48352 100644 --- a/src/lib/jobs/registry/maintenance.ts +++ b/src/lib/jobs/registry/maintenance.ts @@ -202,6 +202,7 @@ import { } from "@/lib/medications/intake-slot-dedup"; import { MAINTENANCE_DISCOVERY } from "./maintenance-discovery"; +import { MAINTENANCE_REPAIR_JOBS } from "./maintenance-repairs"; import { defineJob, type JobDefinition, @@ -524,6 +525,7 @@ const jobs: readonly JobDefinition[] = [ (await import("@/lib/jobs/measurement-fold-repair")) .handleMeasurementFoldRepair, }), + ...MAINTENANCE_REPAIR_JOBS, // The nightly check vacuums when dead rows pile up; the same singleton key // as the operator's run, so the two collapse. defineJob({ diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index ea14eac89..419541cca 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -120,6 +120,7 @@ const subsystemSurface = { "health-connect-import": { audience: "account" }, "compaction-tombstone-purge": { audience: "system" }, "measurement-fold-repair": { audience: "system" }, + "lab-document-date-repair": { audience: "system" }, "measurement-maintenance": { audience: "system" }, "pulse-day-mean": { audience: "system" }, "medication-intake-import": { audience: "system" }, diff --git a/tests/integration/lab-document-date-repair.test.ts b/tests/integration/lab-document-date-repair.test.ts new file mode 100644 index 000000000..82c6aa086 --- /dev/null +++ b/tests/integration/lab-document-date-repair.test.ts @@ -0,0 +1,232 @@ +/** + * The one-shot repair of lab values a document confirm filed under the + * moment of confirmation (real Postgres). + * + * Before the confirm step dated an undated value by its document, such a + * value got `takenAt = now`, the same instant as its `createdAt`. The repair + * moves exactly those to the document's report date (else its filing date) + * at noon UTC, and leaves alone a value with a date of its own, a value that + * really was taken on its confirmation day in the owner's zone, one the + * person re-dated since, and one whose document has no date. A second run + * changes nothing. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { encryptFactData, encryptFactProvenance } from "@/lib/documents/store"; +import { repairUndatedDocumentLabValues } from "@/lib/jobs/lab-document-date-repair"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +const USER = "user-lab-document-date-repair"; +const noon = (day: string) => new Date(`${day}T12:00:00.000Z`); + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + await prisma.user.create({ + data: { + id: USER, + username: "lab-date-repair", + email: "lab-date-repair@example.test", + timezone: "Europe/Berlin", + }, + }); +}); + +/** + * A document, one confirmed value from it and the fact that links the two, + * as the confirm route left them. + */ +async function confirmedValue(opts: { + id: string; + reportDate: string | null; + documentDate: string | null; + effectiveDate: string | null; + createdAt: Date; + takenAt: Date; +}) { + const prisma = getPrismaClient(); + const doc = await prisma.inboundDocument.create({ + data: { + userId: USER, + filename: `${opts.id}.pdf`, + mimeType: "application/pdf", + byteSize: 1024, + contentEncrypted: Buffer.from("stand-in for the stored bytes"), + status: "CONFIRMED", + reportDate: opts.reportDate ? noon(opts.reportDate) : null, + documentDate: opts.documentDate ? noon(opts.documentDate) : null, + }, + }); + await prisma.labResult.create({ + data: { + id: opts.id, + userId: USER, + analyte: "Ferritin", + value: 80, + unit: "ng/mL", + source: "DOCUMENT", + takenAt: opts.takenAt, + createdAt: opts.createdAt, + }, + }); + await prisma.extractedFact.create({ + data: { + documentId: doc.id, + userId: USER, + factType: "OBSERVATION", + status: "APPROVED", + confidence: 0.95, + needsReview: false, + dataEncrypted: encryptFactData({ + label: "Ferritin", + code: null, + codeSystem: null, + value: 80, + valueText: null, + unit: "ng/mL", + referenceLow: null, + referenceHigh: null, + referenceText: null, + effectiveDate: opts.effectiveDate, + }), + provenanceEncrypted: encryptFactProvenance({ + sourceText: "Ferritin 80 ng/mL", + anchored: true, + sourceOffset: 0, + page: 0, + confidence: 0.95, + }), + committedRecordId: opts.id, + committedRecordType: "labResult", + }, + }); +} + +async function takenAt(): Promise> { + const rows = await getPrismaClient().labResult.findMany({ + where: { userId: USER }, + select: { id: true, takenAt: true }, + }); + return Object.fromEntries(rows.map((r) => [r.id, r.takenAt.toISOString()])); +} + +describe("lab document date repair", () => { + it("moves only values filed under their confirmation moment, and converges", async () => { + const confirmed = new Date("2026-10-20T08:00:00.123Z"); + + // Collection date in the header, value undated, filed at confirmation. + await confirmedValue({ + id: "misdated", + reportDate: "2021-03-01", + documentDate: "2026-10-18", + effectiveDate: null, + createdAt: confirmed, + takenAt: confirmed, + }); + // No report date: the filing date is the document's day. + await confirmedValue({ + id: "misdated-filing-date", + reportDate: null, + documentDate: "2021-02-15", + effectiveDate: null, + createdAt: confirmed, + takenAt: new Date(confirmed.getTime() + 2), + }); + // Its own date, even where the row looks like a confirmation moment. + await confirmedValue({ + id: "own-date", + reportDate: "2021-03-01", + documentDate: null, + effectiveDate: "2021-02-20", + createdAt: confirmed, + takenAt: confirmed, + }); + // Confirmed at 00:30 in Berlin on the report day itself: 22:30 UTC the + // evening before, which a UTC comparison would have moved. + const justAfterMidnight = new Date("2026-10-19T22:30:00.000Z"); + await confirmedValue({ + id: "taken-on-confirm-day", + reportDate: "2026-10-20", + documentDate: "2026-10-20", + effectiveDate: null, + createdAt: justAfterMidnight, + takenAt: justAfterMidnight, + }); + // Re-dated by the person after the confirm. + await confirmedValue({ + id: "re-dated", + reportDate: "2021-03-01", + documentDate: null, + effectiveDate: null, + createdAt: confirmed, + takenAt: new Date("2026-10-01T07:45:00.000Z"), + }); + // A document with no date at all leaves nothing to move to. + await confirmedValue({ + id: "undated-document", + reportDate: null, + documentDate: null, + effectiveDate: null, + createdAt: confirmed, + takenAt: confirmed, + }); + + const before = await takenAt(); + const first = await repairUndatedDocumentLabValues(getPrismaClient()); + + expect(first.updated).toBe(2); + expect(await takenAt()).toEqual({ + ...before, + misdated: "2021-03-01T12:00:00.000Z", + "misdated-filing-date": "2021-02-15T12:00:00.000Z", + }); + + const afterFirst = await takenAt(); + const second = await repairUndatedDocumentLabValues(getPrismaClient()); + expect(second.updated).toBe(0); + expect(await takenAt()).toEqual(afterFirst); + }); + + it("leaves a lab value alone when a second fact also names it", async () => { + const confirmed = new Date("2026-10-20T08:00:00.000Z"); + await confirmedValue({ + id: "linked-twice", + reportDate: "2021-03-01", + documentDate: null, + effectiveDate: null, + createdAt: confirmed, + takenAt: confirmed, + }); + const prisma = getPrismaClient(); + const second = await prisma.inboundDocument.create({ + data: { + userId: USER, + filename: "other.pdf", + mimeType: "application/pdf", + byteSize: 1, + contentEncrypted: Buffer.from("x"), + reportDate: noon("2020-01-01"), + }, + }); + const fact = await prisma.extractedFact.findFirstOrThrow({ + where: { committedRecordId: "linked-twice" }, + }); + await prisma.extractedFact.create({ + data: { + documentId: second.id, + userId: USER, + factType: "OBSERVATION", + status: "APPROVED", + dataEncrypted: fact.dataEncrypted, + provenanceEncrypted: fact.provenanceEncrypted, + committedRecordId: "linked-twice", + committedRecordType: "labResult", + }, + }); + + const result = await repairUndatedDocumentLabValues(prisma); + + expect(result.updated).toBe(0); + expect((await takenAt())["linked-twice"]).toBe(confirmed.toISOString()); + }); +}); From 1d8e2e0ee3a6eafe755c3c937dbfd9481ebb126f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 29 Oct 2026 08:52:36 +0100 Subject: [PATCH 04/41] fix(rollups): fold the full backfill a year at a time under its own budget On an account with tens of millions of readings the boot full backfill never finished. It ran four aggregates over the whole five-year window, each held to the connection's 60 s statement limit, so every attempt was cancelled with 57014, the reading extremes added by 0417 stayed NULL, and the readers that need them fell back to the raw table and timed out as well (#1298). The full backfill now folds the window one UTC calendar year per pass, newest first, and every statement runs under its own five-minute limit, the shape the pulse day-mean rebuild already uses. Calendar-year cuts write exactly the buckets one pass writes: day, month and year buckets fall inside one window and a week across New Year is folded whole by both. Each window commits on its own, so a run that reaches its job budget hands the remaining years to a follow-up job instead of failing and starting over on the retry. The boot discovery read every live reading in the window to find uncovered days, the same scan that kept the fold from finishing. It now walks the days per account and type off the live index, so its cost follows the number of days with readings, and runs under the same limit. The full backfill code moves to its own module. --- .../baselines/source-file-lines.json | 4 +- .../baselines/version-stamp-comments.json | 4 +- src/app/api/admin/rollups/recompute/route.ts | 12 +- src/lib/jobs/job-outcome.ts | 2 + src/lib/jobs/queue-runtime.ts | 4 +- src/lib/jobs/registry/rollup.ts | 6 +- src/lib/jobs/reminder/rollup-handlers.ts | 47 ++- .../rollups/__tests__/full-backfill.test.ts | 231 ++++++++++++ .../__tests__/measurement-rollups.test.ts | 21 +- src/lib/rollups/full-backfill.ts | 337 ++++++++++++++++++ src/lib/rollups/measurement-rollups.ts | 196 +--------- .../hrv-rmssd-fitbit-google-migration.test.ts | 6 +- tests/integration/measurement-rollups.test.ts | 2 + .../integration/rollup-full-backfill.test.ts | 306 ++++++++++++++++ 14 files changed, 943 insertions(+), 235 deletions(-) create mode 100644 src/lib/rollups/__tests__/full-backfill.test.ts create mode 100644 src/lib/rollups/full-backfill.ts create mode 100644 tests/integration/rollup-full-backfill.test.ts diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index 66e675fd4..d86b28d50 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 84, - "total": 94928, + "total": 94733, "entries": { "app/medications/page-client.tsx": 839, "components/charts/mood-chart.tsx": 1166, @@ -78,7 +78,7 @@ "lib/medications/scheduling/slot-upsert.ts": 912, "lib/notifications/senders/apns.ts": 1100, "lib/oura/sync.ts": 900, - "lib/rollups/measurement-rollups.ts": 1496, + "lib/rollups/measurement-rollups.ts": 1302, "lib/search/search-records.ts": 996, "lib/sharing/grants.ts": 806, "lib/support/report.ts": 903, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index f319e154c..0ff7da49e 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2897, - "total": 9236, + "total": 9230, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -2748,7 +2748,7 @@ "lib/rollups/measurement-coverage.ts": 1, "lib/rollups/measurement-read-wmy.ts": 4, "lib/rollups/measurement-read.ts": 10, - "lib/rollups/measurement-rollups.ts": 18, + "lib/rollups/measurement-rollups.ts": 12, "lib/rollups/mood-rollups.ts": 4, "lib/rollups/tiered-context.ts": 6, "lib/score-config/rows.ts": 1, diff --git a/src/app/api/admin/rollups/recompute/route.ts b/src/app/api/admin/rollups/recompute/route.ts index 480b36a97..63f612c49 100644 --- a/src/app/api/admin/rollups/recompute/route.ts +++ b/src/app/api/admin/rollups/recompute/route.ts @@ -15,8 +15,8 @@ * { userId?: string } * * - `userId` — single user mode. Synchronously calls - * `recomputeUserRollups(userId)` and returns the row count when - * the fold completes. The call is awaited inside the request, so + * `foldUserRollupHistory(userId)` (the full fold, a calendar year per + * statement) and returns the row count when the fold completes. The call is awaited inside the request, so * the operator sees the duration in the response. * - `userId` omitted — calls `enqueueBootTimeRollupBackfill()` to * re-run the worker-boot discovery (`{ enqueued, skipped }` shape). @@ -37,8 +37,8 @@ import { import { annotate } from "@/lib/logging/context"; import { enqueueBootTimeRollupBackfill, - recomputeUserRollups, -} from "@/lib/rollups/measurement-rollups"; + foldUserRollupHistory, +} from "@/lib/rollups/full-backfill"; interface RecomputeBody { userId?: string; @@ -68,9 +68,9 @@ export const POST = apiHandler(async (request: NextRequest) => { details: { targetUserId: userId, mode: "single" }, }); - let result: Awaited>; + let result: Awaited>; try { - result = await recomputeUserRollups(userId); + result = await foldUserRollupHistory(userId); } catch (err) { const message = err instanceof Error ? err.message : String(err); return apiError(message, 500); diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 3ede654d1..5cbc2d758 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -330,6 +330,8 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "users_synced", "users_useful", "wedge_skipped", + // The full rollup backfill: calendar-year windows one run folded. + "windows_folded", ]); export const MAX_JOB_FACTS = 32; diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 49fc6d154..fb14c254a 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -51,7 +51,7 @@ import { MEDICATION_INTAKE_IMPORT_SEND_OPTIONS } from "@/lib/jobs/medication-int import { HEALTH_CONNECT_IMPORT_SEND_OPTIONS } from "@/lib/jobs/health-connect-import-worker"; import { MEASUREMENT_MAINTENANCE_SEND_OPTIONS } from "@/lib/jobs/measurement-maintenance"; import { PULSE_DAY_MEAN_SEND_OPTIONS } from "@/lib/jobs/pulse-day-mean"; -import { ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS } from "@/lib/rollups/measurement-rollups"; +import { ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS } from "@/lib/rollups/full-backfill"; import { MORNING_DIGEST_REFRESH_EXPIRE_SECONDS } from "@/lib/jobs/morning-digest-refresh-shared"; import { INSIGHT_STATUS_GENERATE_EXPIRE_SECONDS } from "@/lib/jobs/insight-status-generate-queue"; import { WORKOUT_INSIGHT_GENERATE_EXPIRE_SECONDS } from "@/lib/jobs/workout-insight-generate-shared"; @@ -261,7 +261,7 @@ export const QUEUE_RUNTIME: Readonly> = { fn: "handleRollupFullBackfill", }, exclusive: "lockedPass", - why: "One account's five-year fold: four aggregates under the statement timeout and four persist transactions of up to 120 s each, about thirteen minutes at worst under the defaults and unbounded when the operator lifts the statement timeout.", + why: "One account's five-year fold, a calendar year at a time with every statement under its own five-minute limit; the run stops at its budget between windows and hands the remaining years to a follow-up.", }, "mood-rollup-recompute": short("One bucket."), "mood-rollup-full-backfill": short( diff --git a/src/lib/jobs/registry/rollup.ts b/src/lib/jobs/registry/rollup.ts index cb885c0dc..b4e68e834 100644 --- a/src/lib/jobs/registry/rollup.ts +++ b/src/lib/jobs/registry/rollup.ts @@ -56,9 +56,11 @@ import { import { ROLLUP_FULL_BACKFILL_CONCURRENCY, ROLLUP_FULL_BACKFILL_QUEUE, + type RollupFullBackfillPayload, +} from "@/lib/rollups/full-backfill"; +import { ROLLUP_RECOMPUTE_CONCURRENCY, ROLLUP_RECOMPUTE_QUEUE, - type RollupFullBackfillPayload, } from "@/lib/rollups/measurement-rollups"; import { MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY, @@ -213,7 +215,7 @@ const discovery: readonly BootDiscovery[] = [ name: "rollup-full-backfill", run: async () => ( - await import("@/lib/rollups/measurement-rollups") + await import("@/lib/rollups/full-backfill") ).enqueueBootTimeRollupBackfill(BOOT_BACKFILL_STAGGER_SECONDS * 0), }, { diff --git a/src/lib/jobs/reminder/rollup-handlers.ts b/src/lib/jobs/reminder/rollup-handlers.ts index 74c765d02..5a0db0c22 100644 --- a/src/lib/jobs/reminder/rollup-handlers.ts +++ b/src/lib/jobs/reminder/rollup-handlers.ts @@ -56,8 +56,13 @@ import { type CumulativePrRederivePayload, } from "@/lib/personal-records/cumulative-pr-rederivation"; import { - recomputeUserRollups, + foldUserRollupHistory, + ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + sendRollupFullBackfillContinuation, type RollupFullBackfillPayload, +} from "@/lib/rollups/full-backfill"; +import { + recomputeUserRollups, type RollupRecomputePayload, } from "@/lib/rollups/measurement-rollups"; import { @@ -91,35 +96,47 @@ export async function handleRollupRecompute( } /** - * The full fold of one uncovered account across the default window and every - * granularity. Each granularity commits on its own, so a run stopped on its - * budget throws and the retry folds the rest. + * The full fold of one account across the five-year window, a calendar year + * at a time and every statement under the full backfill's own limit (#1298). + * A run that reaches its job budget hands the years it did not reach to a + * follow-up and completes; only a run that could not finish even one window + * fails, and its retry starts that window again. */ export async function handleRollupFullBackfill( jobs: Job[], ): Promise { const shouldStop = jobBudget(jobs); let rowsUpsertedTotal = 0; + let windowsTotal = 0; + let continued = 0; for (const job of jobs) { - const { userId } = job.data; - const { rowsUpserted, durationMs, stopped } = await recomputeUserRollups( - userId, - { shouldStop }, - ); - if (stopped) { - throw new Error( - `rollup-full-backfill for user ${userId} stopped on its job budget; the retry folds the rest`, - ); + const { userId, resumeBefore } = job.data; + const done = await foldUserRollupHistory(userId, { + before: resumeBefore ? new Date(resumeBefore) : undefined, + shouldStop, + statementTimeoutMs: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + }); + rowsUpsertedTotal += done.rowsUpserted; + windowsTotal += done.windowsFolded; + if (done.resumeBefore) { + if (done.windowsFolded === 0) { + throw new Error( + `rollup-full-backfill for user ${userId} stopped on its job budget before it folded a window; the retry starts that window again`, + ); + } + await sendRollupFullBackfillContinuation(userId, done.resumeBefore); + continued += 1; } - rowsUpsertedTotal += rowsUpserted; workerLog( "info", - `[rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, + `[rollup-full-backfill] user=${userId} rows=${done.rowsUpserted} windows=${done.windowsFolded} duration=${done.durationMs}ms${done.resumeBefore ? ` continuesBefore=${done.resumeBefore.toISOString()}` : ""}`, ); } return jobDone({ users: jobs.length, rows_upserted: rowsUpsertedTotal, + windows_folded: windowsTotal, + continued, }); } diff --git a/src/lib/rollups/__tests__/full-backfill.test.ts b/src/lib/rollups/__tests__/full-backfill.test.ts new file mode 100644 index 000000000..5b87f54c2 --- /dev/null +++ b/src/lib/rollups/__tests__/full-backfill.test.ts @@ -0,0 +1,231 @@ +/** + * The full backfill's windowed fold and its handler (#1298): the windows it + * cuts, the per-statement limit every window runs under, and how a run that + * reaches its job budget hands the remaining years on instead of failing. + * + * The fold itself is mocked here; that the windows write exactly the buckets + * one pass over the whole span writes is pinned against Postgres in + * `tests/integration/rollup-full-backfill.test.ts`. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { Job } from "pg-boss"; + +const mocks = vi.hoisted(() => ({ + recomputeUserRollups: vi.fn(), + bossSend: vi.fn(), + getGlobalBoss: vi.fn(), +})); + +vi.mock("@/lib/rollups/measurement-rollups", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/rollups/measurement-rollups") + >()), + recomputeUserRollups: mocks.recomputeUserRollups, +})); + +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: () => mocks.getGlobalBoss(), +})); + +import { handleRollupFullBackfill } from "@/lib/jobs/reminder/rollup-handlers"; +import { + foldUserRollupHistory, + fullBackfillWindows, + ROLLUP_FULL_BACKFILL_QUEUE, + ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + type RollupFullBackfillPayload, +} from "@/lib/rollups/full-backfill"; +import { ROLLUP_FOLD_WINDOW_MS } from "@/lib/rollups/measurement-rollups"; + +const NOW = new Date("2026-10-29T10:00:00.000Z"); + +function job( + data: Partial = {}, +): Job { + return { + id: "job-1", + name: ROLLUP_FULL_BACKFILL_QUEUE, + data: { userId: "user-1", enqueuedAt: NOW.toISOString(), ...data }, + expireInSeconds: 3600, + signal: new AbortController().signal, + } as unknown as Job; +} + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(NOW); + mocks.recomputeUserRollups.mockReset(); + mocks.recomputeUserRollups.mockResolvedValue({ + rowsUpserted: 10, + durationMs: 1, + }); + mocks.bossSend.mockReset(); + mocks.bossSend.mockResolvedValue("job-next"); + mocks.getGlobalBoss.mockReset(); + mocks.getGlobalBoss.mockReturnValue({ send: mocks.bossSend }); +}); + +describe("fullBackfillWindows", () => { + it("cuts the span at UTC New Year, newest first, and covers it exactly", () => { + const from = new Date("2021-10-30T10:00:00.000Z"); + const windows = fullBackfillWindows(from, NOW); + expect( + windows.map((w) => [w.from.toISOString(), w.to.toISOString()]), + ).toEqual([ + ["2026-01-01T00:00:00.000Z", NOW.toISOString()], + ["2025-01-01T00:00:00.000Z", "2026-01-01T00:00:00.000Z"], + ["2024-01-01T00:00:00.000Z", "2025-01-01T00:00:00.000Z"], + ["2023-01-01T00:00:00.000Z", "2024-01-01T00:00:00.000Z"], + ["2022-01-01T00:00:00.000Z", "2023-01-01T00:00:00.000Z"], + [from.toISOString(), "2022-01-01T00:00:00.000Z"], + ]); + for (const w of windows) { + expect(w.to.getTime() - w.from.getTime()).toBeLessThanOrEqual( + 366 * 86_400_000, + ); + } + }); + + it("treats an end on New Year as the end of the year before", () => { + const windows = fullBackfillWindows( + new Date("2024-06-01T00:00:00.000Z"), + new Date("2025-01-01T00:00:00.000Z"), + ); + expect(windows).toEqual([ + { + from: new Date("2024-06-01T00:00:00.000Z"), + to: new Date("2025-01-01T00:00:00.000Z"), + }, + ]); + }); + + it("is empty for an empty span", () => { + expect(fullBackfillWindows(NOW, NOW)).toEqual([]); + }); +}); + +describe("foldUserRollupHistory", () => { + it("folds every window of the fold window under the given statement limit", async () => { + const done = await foldUserRollupHistory("user-1", { + statementTimeoutMs: 1234, + }); + const expected = fullBackfillWindows( + new Date(NOW.getTime() - ROLLUP_FOLD_WINDOW_MS), + NOW, + ); + expect(mocks.recomputeUserRollups).toHaveBeenCalledTimes(expected.length); + expected.forEach((w, i) => { + expect(mocks.recomputeUserRollups).toHaveBeenNthCalledWith( + i + 1, + "user-1", + expect.objectContaining({ + from: w.from, + to: w.to, + statementTimeoutMs: 1234, + }), + ); + }); + expect(done).toMatchObject({ + rowsUpserted: 10 * expected.length, + windowsFolded: expected.length, + }); + expect(done.resumeBefore).toBeUndefined(); + }); + + it("stops before the next window and names where the rest begins", async () => { + const done = await foldUserRollupHistory("user-1", { + shouldStop: () => mocks.recomputeUserRollups.mock.calls.length >= 1, + }); + expect(mocks.recomputeUserRollups).toHaveBeenCalledTimes(1); + expect(done.windowsFolded).toBe(1); + expect(done.resumeBefore).toEqual(new Date("2026-01-01T00:00:00.000Z")); + }); + + it("reports a window the fold stopped inside as still to fold", async () => { + mocks.recomputeUserRollups + .mockResolvedValueOnce({ rowsUpserted: 5, durationMs: 1 }) + .mockResolvedValueOnce({ rowsUpserted: 2, durationMs: 1, stopped: true }); + const done = await foldUserRollupHistory("user-1", { + shouldStop: () => false, + }); + expect(done.windowsFolded).toBe(1); + expect(done.resumeBefore).toEqual(new Date("2026-01-01T00:00:00.000Z")); + }); + + it("resumes with only the windows that end at or before `before`", async () => { + await foldUserRollupHistory("user-1", { + before: new Date("2025-01-01T00:00:00.000Z"), + }); + const tos = mocks.recomputeUserRollups.mock.calls.map( + (call) => (call[1] as { to: Date }).to, + ); + expect(tos[0]).toEqual(new Date("2025-01-01T00:00:00.000Z")); + for (const to of tos) { + expect(to.getTime()).toBeLessThanOrEqual( + new Date("2025-01-01T00:00:00.000Z").getTime(), + ); + } + }); +}); + +describe("handleRollupFullBackfill", () => { + it("runs every statement of the fold under the full backfill's own limit", async () => { + await handleRollupFullBackfill([job()]); + expect(mocks.recomputeUserRollups).toHaveBeenCalled(); + for (const call of mocks.recomputeUserRollups.mock.calls) { + expect(call[1]).toMatchObject({ + statementTimeoutMs: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + }); + } + expect(mocks.bossSend).not.toHaveBeenCalled(); + }); + + it("hands the years it did not reach to a follow-up and completes", async () => { + mocks.recomputeUserRollups + .mockResolvedValueOnce({ rowsUpserted: 5, durationMs: 1 }) + .mockResolvedValueOnce({ rowsUpserted: 0, durationMs: 1, stopped: true }); + const outcome = await handleRollupFullBackfill([job()]); + expect(outcome.ok).toBe(true); + expect(mocks.bossSend).toHaveBeenCalledTimes(1); + const [queue, payload, options] = mocks.bossSend.mock.calls[0]; + expect(queue).toBe(ROLLUP_FULL_BACKFILL_QUEUE); + expect(payload).toMatchObject({ + userId: "user-1", + resumeBefore: "2026-01-01T00:00:00.000Z", + }); + // Not the boot key: the running job still holds that one. + expect(options.singletonKey).toBe( + "boot-backfill|user-1|2026-01-01T00:00:00.000Z", + ); + expect(options.startAfter).toBeGreaterThan(0); + }); + + it("resumes a follow-up where the run before it stopped", async () => { + await handleRollupFullBackfill([ + job({ resumeBefore: "2024-01-01T00:00:00.000Z" }), + ]); + expect( + (mocks.recomputeUserRollups.mock.calls[0][1] as { to: Date }).to, + ).toEqual(new Date("2024-01-01T00:00:00.000Z")); + }); + + it("fails a run that could not finish one window, so its retry starts it again", async () => { + mocks.recomputeUserRollups.mockResolvedValueOnce({ + rowsUpserted: 0, + durationMs: 1, + stopped: true, + }); + await expect(handleRollupFullBackfill([job()])).rejects.toThrow( + /before it folded a window/, + ); + expect(mocks.bossSend).not.toHaveBeenCalled(); + }); + + it("fails the run when the follow-up cannot be queued", async () => { + mocks.getGlobalBoss.mockReturnValue(null); + mocks.recomputeUserRollups + .mockResolvedValueOnce({ rowsUpserted: 5, durationMs: 1 }) + .mockResolvedValueOnce({ rowsUpserted: 0, durationMs: 1, stopped: true }); + await expect(handleRollupFullBackfill([job()])).rejects.toThrow(/no queue/); + }); +}); diff --git a/src/lib/rollups/__tests__/measurement-rollups.test.ts b/src/lib/rollups/__tests__/measurement-rollups.test.ts index 69161ada6..88c499569 100644 --- a/src/lib/rollups/__tests__/measurement-rollups.test.ts +++ b/src/lib/rollups/__tests__/measurement-rollups.test.ts @@ -29,6 +29,7 @@ const mocks = vi.hoisted(() => ({ findMany: vi.fn(), findFirstMeasurement: vi.fn(), txExecuteRawUnsafe: vi.fn(), + txExecuteRaw: vi.fn(), bossSend: vi.fn(), getGlobalBossMock: vi.fn(), })); @@ -87,10 +88,12 @@ void queryRaw; import { ROLLUP_FULL_BACKFILL_QUEUE, + enqueueBootTimeRollupBackfill, +} from "../full-backfill"; +import { ROLLUP_RECOMPUTE_QUEUE, _resetEnsureUserRollupsFreshInFlightForTests, collapseToTypeDayKeys, - enqueueBootTimeRollupBackfill, enqueueRollupRecompute, ensureUserRollupsFresh, recomputeBucketsForMeasurement, @@ -127,6 +130,8 @@ beforeEach(() => { createMany: mocks.createMany, }, $executeRawUnsafe: mocks.txExecuteRawUnsafe, + $executeRaw: mocks.txExecuteRaw, + $queryRaw: mocks.queryRaw, }); } return Promise.all(arg as unknown[]); @@ -996,10 +1001,10 @@ describe("enqueueBootTimeRollupBackfill", () => { const sqlParts = queryRaw.mock.calls[0][0] as TemplateStringsArray; const sqlText = Array.isArray(sqlParts) ? sqlParts.join("?") : ""; - // Inner DISTINCT widens to `(user_id, type, bucket_start)`. - expect(sqlText).toContain("date_trunc('day', m.\"measured_at\")"); - // LEFT JOIN now compares bucket_start on both sides. - expect(sqlText).toContain('r."bucket_start" = mt."bucket_start"'); + // The day walk steps from each day's first reading to the next UTC day. + expect(sqlText).toContain("date_trunc('day', d.at) + interval '1 day'"); + // The coverage check compares the walked day with the DAY bucket. + expect(sqlText).toContain(`r."bucket_start" = date_trunc('day', d.at)`); }); // v1.16.10 — discovery and fold share one trailing window. Unbounded @@ -1017,12 +1022,14 @@ describe("enqueueBootTimeRollupBackfill", () => { const after = Date.now(); const sqlParts = queryRaw.mock.calls[0][0] as TemplateStringsArray; const sqlText = Array.isArray(sqlParts) ? sqlParts.join("?") : ""; - // The inner scan filters on measured_at >= . + // The day walk starts at measured_at >= . expect(sqlText).toContain('m."measured_at" >='); // The bound parameter is now − ROLLUP_FOLD_WINDOW_MS (5 years), // matching the `recomputeUserRollups` default the backfill worker // runs with. - const boundArg = queryRaw.mock.calls[0][1] as Date; + const boundArg = queryRaw.mock.calls[0].find( + (arg: unknown) => arg instanceof Date, + ) as Date; expect(boundArg).toBeInstanceOf(Date); const fiveYearsMs = 5 * 365 * 24 * 60 * 60 * 1000; expect(boundArg.getTime()).toBeGreaterThanOrEqual(before - fiveYearsMs); diff --git a/src/lib/rollups/full-backfill.ts b/src/lib/rollups/full-backfill.ts new file mode 100644 index 000000000..e531a1b56 --- /dev/null +++ b/src/lib/rollups/full-backfill.ts @@ -0,0 +1,337 @@ +/** + * The full fold of one account's rollup tier, and the boot discovery that + * decides which accounts need one. + * + * The write hooks keep the DAY bucket of each touched day current and the + * read path warms the last 90 days, but neither folds the deep WEEK / MONTH / + * YEAR history, and neither refills a column a migration adds to rows that + * already exist (the accumulators of 0190, the reading extremes of 0417). The + * full backfill does both, once per account the discovery finds. + * + * Sized for large accounts (#1298). The fold used to be four aggregates over + * the whole five-year window, each held to the connection's statement limit + * (60 s unless `DATABASE_STATEMENT_TIMEOUT_MS` says otherwise). On an account + * with tens of millions of readings no aggregate finished in time, the job + * failed on every boot, and the columns 0417 added never filled, so every + * reader that needs them stayed on the raw table and timed out as well. The + * fold now works through the window one UTC calendar year at a time, newest + * first, and every statement of the full backfill runs under its own limit + * (`ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS`), the shape the pulse day-mean + * rebuild already had. Each window commits on its own, so a run that reaches + * its job budget hands the remaining years to a follow-up job instead of + * starting over on the retry. + */ +import { activeAccountSql } from "@/lib/account-deletion/active-account"; +import { prisma } from "@/lib/db"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { + applyStatementBudget, + recomputeUserRollups, + ROLLUP_FOLD_WINDOW_MS, +} from "@/lib/rollups/measurement-rollups"; + +/** + * The boot-time full-fold queue: one job per account the discovery finds, so + * a self-hosted instance converges on the rollup tier without an operator. + */ +export const ROLLUP_FULL_BACKFILL_QUEUE = "rollup-full-backfill"; + +/** Boot-fold worker concurrency. Serial so we don't crowd the pool. */ +export const ROLLUP_FULL_BACKFILL_CONCURRENCY = 1; + +/** + * The full fold's expiry: an hour. A run stops itself at three quarters of it + * (`jobBudget`), between two windows or two granularities, and hands what is + * left to a follow-up, so the expiry bounds one run, not the whole account. + * The margin above the budget covers the statement in flight, which can run + * for `ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS`. + */ +export const ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS = 60 * 60; + +/** + * The limit of every statement the full backfill and its discovery run, in + * place of the connection's request limit. Five minutes, as the pulse + * day-mean rebuild uses: one calendar year of one account's readings, + * aggregated per granularity, stays well inside it on the largest accounts + * reported, while a statement that would run longer still ends. + */ +export const ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS = 5 * 60 * 1000; + +/** Seconds a follow-up waits before it picks up the remaining years. */ +export const ROLLUP_FULL_BACKFILL_CONTINUATION_DELAY_SECONDS = 60; + +/** Payload of one full-fold job. */ +export interface RollupFullBackfillPayload { + userId: string; + /** Wall-clock kick-off for debugging only. */ + enqueuedAt: string; + /** + * ISO instant. A follow-up folds only the windows that end at or before + * it: the run that sent it folded everything newer. + */ + resumeBefore?: string; +} + +/** The retry policy of every full-fold send. */ +const RETRY_OPTIONS = { + retryLimit: 3, + retryDelay: 60, + retryBackoff: true, +} as const; + +/** + * `[from, to)` windows covering `[from, to)`, cut at UTC New Year and + * returned newest first. Every DAY, MONTH and YEAR bucket lies inside one + * window, and a WEEK that spans New Year is folded whole by both of its + * windows (the fold snaps each window out to whole buckets), so the windows + * write exactly the buckets one fold over the whole span writes. Pure. + */ +export function fullBackfillWindows( + from: Date, + to: Date, +): Array<{ from: Date; to: Date }> { + const out: Array<{ from: Date; to: Date }> = []; + let end = to.getTime(); + const floor = from.getTime(); + while (end > floor) { + const year = new Date(end).getUTCFullYear(); + const newYear = Date.UTC(year, 0, 1); + const start = Math.max( + floor, + newYear === end ? Date.UTC(year - 1, 0, 1) : newYear, + ); + out.push({ from: new Date(start), to: new Date(end) }); + end = start; + } + return out; +} + +export interface RollupHistoryFold { + rowsUpserted: number; + durationMs: number; + /** Windows folded completely by this call. */ + windowsFolded: number; + /** + * Set when `shouldStop` ended the fold: every window that ends at or + * before this instant is still to fold, and nothing newer is. + */ + resumeBefore?: Date; +} + +/** + * Fold one account's rollup tier over the five-year window, every type and + * granularity, one calendar year at a time, newest first. The same buckets + * `recomputeUserRollups(userId)` writes in one pass; see + * `fullBackfillWindows` for why the windows change nothing about them. + * + * `before` resumes a fold that stopped: only the windows ending at or before + * it run. `shouldStop` is asked before each window and between the + * granularities of a window; a window it interrupts is folded again whole by + * the follow-up, so `resumeBefore` is always a window's end. + */ +export async function foldUserRollupHistory( + userId: string, + opts: { + before?: Date; + shouldStop?: () => boolean; + statementTimeoutMs?: number; + now?: Date; + } = {}, +): Promise { + const startedAt = Date.now(); + const now = opts.now ?? new Date(); + const floor = new Date(now.getTime() - ROLLUP_FOLD_WINDOW_MS); + const ceiling = opts.before && opts.before < now ? opts.before : now; + let rowsUpserted = 0; + let windowsFolded = 0; + const result = (resumeBefore?: Date): RollupHistoryFold => ({ + rowsUpserted, + durationMs: Date.now() - startedAt, + windowsFolded, + ...(resumeBefore ? { resumeBefore } : {}), + }); + for (const window of fullBackfillWindows(floor, ceiling)) { + if (opts.shouldStop?.()) return result(window.to); + const done = await recomputeUserRollups(userId, { + from: window.from, + to: window.to, + statementTimeoutMs: opts.statementTimeoutMs, + shouldStop: opts.shouldStop, + }); + rowsUpserted += done.rowsUpserted; + if (done.stopped) return result(window.to); + windowsFolded += 1; + } + return result(); +} + +/** + * Hand the windows a stopped fold did not reach to a follow-up job. Keyed on + * the account and the resume point: the running job still holds the account's + * boot key, and a second follow-up for the same point is the same work. + * Throws when no queue is attached or the send fails, so the run fails and + * its retry folds the same windows again rather than leaving them. + */ +export async function sendRollupFullBackfillContinuation( + userId: string, + resumeBefore: Date, +): Promise { + const boss = getGlobalBoss(); + if (!boss) { + throw new Error("rollup-full-backfill: no queue to hand the rest to"); + } + const payload: RollupFullBackfillPayload = { + userId, + enqueuedAt: new Date().toISOString(), + resumeBefore: resumeBefore.toISOString(), + }; + const jobId = await boss.send(ROLLUP_FULL_BACKFILL_QUEUE, payload, { + ...RETRY_OPTIONS, + expireInSeconds: ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS, + singletonKey: `boot-backfill|${userId}|${payload.resumeBefore}`, + startAfter: ROLLUP_FULL_BACKFILL_CONTINUATION_DELAY_SECONDS, + }); + return jobId !== null; +} + +/** + * Boot-time discovery. Finds the accounts whose rollup tier misses something + * inside the fold window and enqueues one full fold per account. + * + * Two arms: + * + * 1. Missing coverage per day: a (user, type, UTC day) with a live reading + * and no DAY rollup row. Per day rather than per type, because a sync + * or a restore that writes without the write hook leaves gaps in a type + * that already has buckets. + * 2. A DAY rollup row inside the window whose regression accumulators + * (0190) or reading extremes (0417) are NULL: a row folded before the + * column existed. New folds write them, so the arm drains to empty. + * + * Both are bounded to the fold window. The fold writes nothing older, so an + * older gap would flag the account on every boot and never close. + * + * Arm 1 used to read every live reading in the window, which on an account + * with tens of millions of rows is the same scan that kept the fold itself + * from finishing. It now walks the days instead: per account and type, the + * first live reading at or after the start of each next UTC day, read off + * the live `(user_id, type, measured_at)` index, so the cost follows the + * number of days with readings, not the number of readings. The statement + * also runs under the full backfill's own limit. + * + * Idempotent across reboots: an account drops off once it is covered, and + * the per-account `singletonKey` coalesces a send while a fold is queued. + * Best-effort: errors come back in the result so the worker boot never fails. + */ +export async function enqueueBootTimeRollupBackfill( + // Optional boot-storm stagger, seconds. Default 0 keeps immediate semantics + // for any non-boot caller. + startAfterSeconds: number = 0, +): Promise<{ + enqueued: number; + skipped: number; + error: string | null; +}> { + const boss = getGlobalBoss(); + if (!boss) { + return { enqueued: 0, skipped: 0, error: null }; + } + + try { + const foldWindowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); + const users = await prisma.$transaction( + async (tx) => { + await applyStatementBudget( + tx, + ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + ); + return tx.$queryRaw>` + WITH RECURSIVE pairs AS ( + SELECT u."id" AS user_id, t.type + FROM users u + CROSS JOIN unnest(enum_range(NULL::"measurement_type")) AS t(type) + WHERE ${activeAccountSql('u."id"')} + ), + days AS ( + SELECT p.user_id, p.type, f.at + FROM pairs p + CROSS JOIN LATERAL ( + SELECT min(m."measured_at") AS at + FROM measurements m + WHERE m."user_id" = p.user_id + AND m."type" = p.type + AND m."deleted_at" IS NULL + AND m."measured_at" >= ${foldWindowStart} + ) f + WHERE f.at IS NOT NULL + UNION ALL + SELECT d.user_id, d.type, n.at + FROM days d + CROSS JOIN LATERAL ( + SELECT min(m."measured_at") AS at + FROM measurements m + WHERE m."user_id" = d.user_id + AND m."type" = d.type + AND m."deleted_at" IS NULL + AND m."measured_at" >= date_trunc('day', d.at) + interval '1 day' + ) n + WHERE n.at IS NOT NULL + ) + SELECT DISTINCT id FROM ( + SELECT d.user_id AS id + FROM days d + WHERE NOT EXISTS ( + SELECT 1 FROM measurement_rollups r + WHERE r."user_id" = d.user_id + AND r."type" = d.type + AND r."granularity" = 'DAY' + AND r."bucket_start" = date_trunc('day', d.at) + ) + + UNION + + SELECT r."user_id" AS id + FROM measurement_rollups r + WHERE r."granularity" = 'DAY' + AND r."bucket_start" >= ${foldWindowStart} + AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL) + AND ${activeAccountSql('r."user_id"')} + ) discovered + `; + }, + { + maxWait: 10_000, + timeout: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS + 30_000, + }, + ); + + let enqueued = 0; + let skipped = 0; + for (const { id } of users) { + const payload: RollupFullBackfillPayload = { + userId: id, + enqueuedAt: new Date().toISOString(), + }; + const jobId = await boss.send(ROLLUP_FULL_BACKFILL_QUEUE, payload, { + ...RETRY_OPTIONS, + expireInSeconds: ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS, + // Coalesce: if a backfill for this user is already queued and + // we restart, pg-boss returns null instead of duplicating. + singletonKey: `boot-backfill|${id}`, + ...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {}), + }); + if (jobId) { + enqueued += 1; + } else { + skipped += 1; + } + } + return { enqueued, skipped, error: null }; + } catch (err) { + return { + enqueued: 0, + skipped: 0, + error: err instanceof Error ? err.message : String(err), + }; + } +} diff --git a/src/lib/rollups/measurement-rollups.ts b/src/lib/rollups/measurement-rollups.ts index ad330a2e9..0bda33e66 100644 --- a/src/lib/rollups/measurement-rollups.ts +++ b/src/lib/rollups/measurement-rollups.ts @@ -34,9 +34,7 @@ * took. Nothing here reads both tiers; the rollup replaces the legacy read and * falls back to live SQL only on a coverage miss. */ -import { activeAccountSql } from "@/lib/account-deletion/active-account"; import { prisma } from "@/lib/db"; -import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { annotate } from "@/lib/logging/context"; import { @@ -113,39 +111,6 @@ export interface RollupRecomputePayload { enqueuedAt: string; } -/** - * v1.4.35.1 — boot-time full-fold queue. The write-path hooks keep - * the DAY bucket current per measurement, and `ensureUserRollupsFresh` - * warm-on-read covers the trailing 90-day DAY window on cold mount, - * but neither path folds the deep WEEK / MONTH / YEAR history for an - * account that has measurements pre-dating the v1.4.35 deploy. This - * queue exists so the worker boot can enqueue one full - * `recomputeUserRollups(userId)` per uncovered user — no operator - * action needed on self-hosted instances. - */ -export const ROLLUP_FULL_BACKFILL_QUEUE = "rollup-full-backfill"; - -/** Boot-fold worker concurrency. Serial so we don't crowd the pool. */ -export const ROLLUP_FULL_BACKFILL_CONCURRENCY = 1; - -/** - * The full fold's expiry: an hour. One fold is four granularities, each an - * aggregate held to the connection's statement timeout (60 s by default) and - * a persist transaction of up to 120 s, so the worst case under the defaults - * is close to thirteen minutes, too near pg-boss's fifteen-minute default to - * rely on, and an operator who raises `DATABASE_STATEMENT_TIMEOUT_MS` (or - * sets it to 0) lifts that bound altogether. Past the expiry pg-boss retries - * the job, and the retry folded the same account a second time. - */ -export const ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS = 60 * 60; - -/** Payload the boot-time backfill helper enqueues. */ -export interface RollupFullBackfillPayload { - userId: string; - /** Wall-clock kick-off for debugging only. */ - enqueuedAt: string; -} - /** * Trailing fold window for `recomputeUserRollups` defaults AND the * boot-time discovery scan. The two MUST share this bound: the fold @@ -639,7 +604,7 @@ export async function readRollupBuckets( * comprehensive-aggregator + summaries-slice convention. */ /** Give the statements of a transaction their own limit (SET LOCAL). */ -async function applyStatementBudget( +export async function applyStatementBudget( tx: Pick, ms: number, ): Promise { @@ -1316,165 +1281,6 @@ async function ensureUserRollupsFreshImpl( } } -/** - * v1.4.35.1 — boot-time enqueue helper. Finds users with measurements - * but zero rollup coverage and enqueues one full-fold job per account - * onto `ROLLUP_FULL_BACKFILL_QUEUE`. Designed to be called once at - * worker boot so every self-hosted instance auto-converges on the - * persistent rollup tier without operator intervention. - * - * Idempotent across reboots: - * - the discovery query matches accounts that either have zero - * rollup coverage at all OR have at least one measurement type - * whose DAY-rollup partition is empty. Once every type the user - * has logged has at least one DAY bucket, the user drops off the - * list and subsequent boots skip them. The v1.4.35.1 shape (zero- - * rollups only) silently stranded users who logged a brand-new - * type after the initial fold — their `isFullyCovered` probe stays - * `false`, every read fans out to the live SQL aggregator, and - * `/api/analytics` cold-mounts in tens of seconds instead of - * 1.5-3 s. v1.4.38.5 widens discovery to per-type missing coverage. - * - pg-boss `singletonKey` coalesces duplicate sends within the - * queue, so a fast restart while a backfill is queued doesn't - * double up - * - * Best-effort: errors are swallowed and reported through the return - * value so the worker boot never fails because of a backfill miss. - */ -export async function enqueueBootTimeRollupBackfill( - // Optional boot-storm stagger. When > 0 the per-user sends carry a - // `startAfter` delay (seconds) so this full-history fold does not drain in - // parallel with the other boot backfills onto one heavy tenant. Default 0 - // keeps immediate semantics for any non-boot caller. - startAfterSeconds: number = 0, -): Promise<{ - enqueued: number; - skipped: number; - error: string | null; -}> { - const boss = getGlobalBoss(); - if (!boss) { - return { enqueued: 0, skipped: 0, error: null }; - } - - try { - // Users where at least one (type, day) pair in `measurements` - // has no matching DAY rollup row. - // - // v1.4.39.1 — tightened from per-type to per-day. The legacy shape - // matched users where ANY type had zero rollup rows. That stranded - // accounts whose Withings sync / `/api/import` / admin restore - // wrote measurements without firing the rollup write hook: as soon - // as ONE rollup row existed for the type (even from a prior boot - // backfill), the LEFT JOIN found a match and the user dropped off - // the discovery list — leaving the dashboard chart's - // `source=rollup` fast-path under-counting the missing days. - // Switching the join to `(user, type, bucketStart)` surfaces every - // per-day gap. Cost: the inner DISTINCT widens from - // `(user_id, type)` (≈15 rows/user) to - // `(user_id, type, day)` (≈5 k rows/user on a 5-year power user), - // which is still bounded by the `(user_id, type, measured_at)` - // index path. The LEFT JOIN walks `measurement_rollups` against - // its composite primary key. - // - // v1.4.41 — the v1.4.39 W-SUM legacy-NULL `sum_value` UNION arm - // has been retired. The maintainer's tenant converged at v1.4.40, so the - // arm was a permanent no-op seq scan over `measurement_rollups`. - // Per-day missing coverage remains the sole discovery anchor. - // - // v1.16.10 — discovery is bounded to the same trailing window the - // fold writes (`ROLLUP_FOLD_WINDOW_MS`). Unbounded, any account - // with measurements older than the window was re-flagged on every - // boot: the fold can never produce buckets for those days, so the - // per-day gap re-surfaced forever and the full backfill re-ran on - // every restart. Days outside the window are not foldable by - // design and must not anchor discovery. - const foldWindowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); - // Two discovery arms, UNION-ed: - // - // 1. Per-day MISSING coverage — a (user, type, day) pair in - // `measurements` with no matching DAY rollup row (v1.4.39.1). - // - // 2. v1.20.0 F6 NULL-accumulator arm — a user whose DAY rollup rows - // predate migration 0190 (regression accumulators are NULL) needs a - // one-time re-fold so the windowed slope / r² / sd readers compose - // from the accumulators instead of falling back to the live REGR_* - // scan. Anchored on `sum_xy IS NULL` (the column the read tier - // treats as the coverage marker). New writes self-populate the - // accumulators, so this arm drains to empty once the boot re-fold - // catches up — mirrors the v1.4.39 sum_value rollout, which the - // v1.4.41 note confirms converged via this same boot queue. Bounded - // to the fold window so days outside it (un-foldable by design) - // never re-flag a user on every boot. The same arm refills the - // reading extremes of migration 0417 (`reading_min_value IS NULL`); - // the extremes reader treats a NULL as uncovered until it lands. - const users = await prisma.$queryRaw>` - SELECT DISTINCT id FROM ( - SELECT DISTINCT mt."user_id" AS id - FROM ( - SELECT DISTINCT - m."user_id", - m."type", - date_trunc('day', m."measured_at") AS bucket_start - FROM measurements m - WHERE m."deleted_at" IS NULL - AND m."measured_at" >= ${foldWindowStart} - ) mt - LEFT JOIN measurement_rollups r - ON r."user_id" = mt."user_id" - AND r."type" = mt."type" - AND r."granularity" = 'DAY' - AND r."bucket_start" = mt."bucket_start" - WHERE r."bucket_start" IS NULL - - UNION - - SELECT DISTINCT r."user_id" AS id - FROM measurement_rollups r - WHERE r."granularity" = 'DAY' - AND r."bucket_start" >= ${foldWindowStart} - AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL) - ) discovered - WHERE ${activeAccountSql('discovered."id"')} - `; - - if (users.length === 0) { - return { enqueued: 0, skipped: 0, error: null }; - } - - let enqueued = 0; - let skipped = 0; - for (const { id } of users) { - const payload: RollupFullBackfillPayload = { - userId: id, - enqueuedAt: new Date().toISOString(), - }; - const jobId = await boss.send(ROLLUP_FULL_BACKFILL_QUEUE, payload, { - expireInSeconds: ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS, - retryLimit: 3, - retryDelay: 60, - retryBackoff: true, - // Coalesce: if a backfill for this user is already queued and - // we restart, pg-boss returns null instead of duplicating. - singletonKey: `boot-backfill|${id}`, - ...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {}), - }); - if (jobId) { - enqueued += 1; - } else { - skipped += 1; - } - } - return { enqueued, skipped, error: null }; - } catch (err) { - return { - enqueued: 0, - skipped: 0, - error: err instanceof Error ? err.message : String(err), - }; - } -} - /** * Collapse a list of `(type, day)` keys to the set of distinct * `(type, dayStart)` pairs. Used by the batch-ingest hooks so a 500- diff --git a/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts index f31ce755c..cbbadf508 100644 --- a/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts +++ b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts @@ -12,10 +12,8 @@ import { beforeAll, describe, expect, it, vi } from "vitest"; import { getPrismaClient, truncateAllTables } from "./setup"; import type { Prisma } from "@/generated/prisma/client"; -import { - enqueueBootTimeRollupBackfill, - recomputeUserRollups, -} from "@/lib/rollups/measurement-rollups"; +import { enqueueBootTimeRollupBackfill } from "@/lib/rollups/full-backfill"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; vi.mock("@/lib/db-compat", () => ({ ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), diff --git a/tests/integration/measurement-rollups.test.ts b/tests/integration/measurement-rollups.test.ts index 2e382c9a5..c813fdebf 100644 --- a/tests/integration/measurement-rollups.test.ts +++ b/tests/integration/measurement-rollups.test.ts @@ -29,6 +29,8 @@ import { getPrismaClient, truncateAllTables } from "./setup"; import { ROLLUP_FULL_BACKFILL_QUEUE, enqueueBootTimeRollupBackfill, +} from "@/lib/rollups/full-backfill"; +import { recomputeBucketsForKeys, recomputeBucketsForMeasurement, recomputeUserRollups, diff --git a/tests/integration/rollup-full-backfill.test.ts b/tests/integration/rollup-full-backfill.test.ts new file mode 100644 index 000000000..01f6fbbd6 --- /dev/null +++ b/tests/integration/rollup-full-backfill.test.ts @@ -0,0 +1,306 @@ +/** + * The full backfill against Postgres (#1298). + * + * - The windowed fold (`foldUserRollupHistory`, a UTC calendar year per + * pass, newest first) writes exactly the rollup rows one pass of + * `recomputeUserRollups` over the whole five-year window writes: every + * granularity, every source, a week across New Year, the YEAR bucket the + * window's oldest edge snaps out to, and a reading older than the window. + * - A fold stopped part way and resumed with `before` ends in the same rows. + * - The boot discovery, rewritten to walk days instead of reading every + * reading in the window, finds the same accounts as the statement it + * replaced, kept verbatim below as the reference. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { Prisma, type MeasurementSource } from "@/generated/prisma/client"; +import { activeAccountSql } from "@/lib/account-deletion/active-account"; +import { + enqueueBootTimeRollupBackfill, + foldUserRollupHistory, + ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, +} from "@/lib/rollups/full-backfill"; +import { + recomputeUserRollups, + ROLLUP_FOLD_WINDOW_MS, +} from "@/lib/rollups/measurement-rollups"; + +import { makeUser } from "../kit/factories"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: vi.fn(() => null), +})); + +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; + +const prisma = getPrismaClient(); +const DAY_MS = 86_400_000; +const HOUR_MS = 3_600_000; + +beforeEach(async () => { + await truncateAllTables(prisma); + vi.mocked(getGlobalBoss).mockReturnValue(null as never); +}); + +function utc(y: number, m: number, d: number, h = 12, min = 0): Date { + return new Date(Date.UTC(y, m - 1, d, h, min)); +} + +/** Readings that can tell a windowed fold from one pass. */ +async function seedHistory(userId: string): Promise { + const now = new Date(); + const year = now.getUTCFullYear(); + const floor = new Date(now.getTime() - ROLLUP_FOLD_WINDOW_MS); + const rows: Prisma.MeasurementCreateManyInput[] = []; + const add = ( + type: "WEIGHT" | "PULSE", + source: MeasurementSource, + at: Date, + value: number, + ) => + rows.push({ + userId, + type, + source, + value, + unit: type === "WEIGHT" ? "kg" : "bpm", + measuredAt: at, + externalId: `${source}:${type}:${at.toISOString()}`, + }); + const days: Date[] = []; + // A week across each of two New Years. + for (let d = 0; d < 9; d++) { + days.push(new Date(utc(year - 3, 12, 27).getTime() + d * DAY_MS)); + days.push(new Date(utc(year - 1, 12, 28).getTime() + d * DAY_MS)); + } + days.push(utc(year - 2, 6, 15), utc(year - 2, 6, 16)); + days.push(new Date(now.getTime() - 3 * DAY_MS)); + // Just inside the window, and inside the YEAR bucket its edge snaps to. + days.push(new Date(floor.getTime() + 2 * DAY_MS)); + days.push(new Date(floor.getTime() - 20 * DAY_MS)); + days.forEach((day, i) => { + const base = Date.UTC( + day.getUTCFullYear(), + day.getUTCMonth(), + day.getUTCDate(), + ); + add("WEIGHT", "MANUAL", new Date(base + 7 * HOUR_MS), 80 + (i % 5) / 3); + if (i % 2 === 0) { + add("WEIGHT", "WITHINGS", new Date(base + 8 * HOUR_MS), 81 + i / 7); + } + for (let k = 0; k < 6; k++) { + add( + "PULSE", + "APPLE_HEALTH", + new Date(base + (k < 4 ? 9 : 15) * HOUR_MS + k * 7 * 60_000), + 55 + ((i * 7 + k * 11) % 40), + ); + } + add("PULSE", "GOOGLE_HEALTH", new Date(base + 23 * HOUR_MS), 60 + i); + }); + await prisma.measurement.createMany({ data: rows }); +} + +async function snapshot(userId: string) { + const rows = await prisma.measurementRollup.findMany({ + where: { userId }, + orderBy: [ + { type: "asc" }, + { granularity: "asc" }, + { bucketStart: "asc" }, + { source: "asc" }, + ], + }); + return rows.map(({ computedAt, ...rest }) => { + void computedAt; + return rest; + }); +} + +function expectSameRows( + actual: Awaited>, + expected: Awaited>, +): void { + expect(actual.length).toBe(expected.length); + expected.forEach((want, i) => { + const got = actual[i] as Record; + for (const [key, value] of Object.entries(want)) { + if (typeof value === "number" && key !== "count") { + // A week across two windows is read in another range, so its sums + // may be added in another order: equal to rounding, not to the bit. + expect(got[key] as number).toBeCloseTo(value, 6); + } else { + expect(got[key]).toEqual(value); + } + } + }); +} + +describe("foldUserRollupHistory", () => { + it("writes the rows one pass over the whole window writes", async () => { + const user = await makeUser(); + await seedHistory(user.id); + + await recomputeUserRollups(user.id); + const onePass = await snapshot(user.id); + expect(onePass.length).toBeGreaterThan(50); + expect(onePass.some((r) => r.granularity === "WEEK")).toBe(true); + + await prisma.measurementRollup.deleteMany({ where: { userId: user.id } }); + const done = await foldUserRollupHistory(user.id, { + statementTimeoutMs: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + }); + expect(done.windowsFolded).toBeGreaterThanOrEqual(5); + expect(done.resumeBefore).toBeUndefined(); + expectSameRows(await snapshot(user.id), onePass); + expect( + (await snapshot(user.id)).every( + (r) => r.readingMinValue !== null && r.readingMaxValue !== null, + ), + ).toBe(true); + }); + + it("ends in the same rows when stopped part way and resumed", async () => { + const user = await makeUser(); + await seedHistory(user.id); + await recomputeUserRollups(user.id); + const onePass = await snapshot(user.id); + await prisma.measurementRollup.deleteMany({ where: { userId: user.id } }); + + // Stop on the seventh question: inside the second window. + let asked = 0; + const first = await foldUserRollupHistory(user.id, { + shouldStop: () => ++asked >= 7, + }); + expect(first.windowsFolded).toBe(1); + expect(first.resumeBefore).toBeDefined(); + + const rest = await foldUserRollupHistory(user.id, { + before: first.resumeBefore, + }); + expect(rest.resumeBefore).toBeUndefined(); + expectSameRows(await snapshot(user.id), onePass); + }); +}); + +/** The discovery statement before the day walk, verbatim. */ +async function referenceDiscovery(): Promise { + const foldWindowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); + const users = await prisma.$queryRaw>` + SELECT DISTINCT id FROM ( + SELECT DISTINCT mt."user_id" AS id + FROM ( + SELECT DISTINCT + m."user_id", + m."type", + date_trunc('day', m."measured_at") AS bucket_start + FROM measurements m + WHERE m."deleted_at" IS NULL + AND m."measured_at" >= ${foldWindowStart} + ) mt + LEFT JOIN measurement_rollups r + ON r."user_id" = mt."user_id" + AND r."type" = mt."type" + AND r."granularity" = 'DAY' + AND r."bucket_start" = mt."bucket_start" + WHERE r."bucket_start" IS NULL + + UNION + + SELECT DISTINCT r."user_id" AS id + FROM measurement_rollups r + WHERE r."granularity" = 'DAY' + AND r."bucket_start" >= ${foldWindowStart} + AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL) + ) discovered + WHERE ${activeAccountSql('discovered."id"')} + `; + return users.map((u) => u.id).sort(); +} + +describe("enqueueBootTimeRollupBackfill", () => { + it("finds the accounts the statement it replaced found", async () => { + const covered = await makeUser(); + const missingDay = await makeUser(); + const nullExtremes = await makeUser(); + const tombstoneOnly = await makeUser(); + const tooOld = await makeUser(); + const pendingDeletion = await makeUser(); + for (const u of [covered, missingDay, nullExtremes, pendingDeletion]) { + await seedHistory(u.id); + await recomputeUserRollups(u.id); + } + // A reading on a day with no DAY bucket. + await prisma.measurement.create({ + data: { + userId: missingDay.id, + type: "BODY_FAT", + value: 20, + unit: "%", + source: "MANUAL", + measuredAt: new Date(Date.now() - 40 * DAY_MS), + }, + }); + await prisma.measurement.create({ + data: { + userId: pendingDeletion.id, + type: "BODY_FAT", + value: 20, + unit: "%", + source: "MANUAL", + measuredAt: new Date(Date.now() - 40 * DAY_MS), + }, + }); + await prisma.user.update({ + where: { id: pendingDeletion.id }, + data: { deletionRequestedAt: new Date() }, + }); + // A row folded before 0417 added the reading extremes. + await prisma.$executeRaw` + UPDATE measurement_rollups SET reading_min_value = NULL + WHERE user_id = ${nullExtremes.id} AND granularity = 'DAY' + AND bucket_start = ( + SELECT max(bucket_start) FROM measurement_rollups + WHERE user_id = ${nullExtremes.id} AND granularity = 'DAY' + ) + `; + // Only a tombstone inside the window: nothing to fold. + await prisma.measurement.create({ + data: { + userId: tombstoneOnly.id, + type: "WEIGHT", + value: 70, + unit: "kg", + source: "MANUAL", + measuredAt: new Date(Date.now() - 10 * DAY_MS), + deletedAt: new Date(), + }, + }); + // Only readings older than the window: the fold writes nothing there. + await prisma.measurement.create({ + data: { + userId: tooOld.id, + type: "WEIGHT", + value: 70, + unit: "kg", + source: "MANUAL", + measuredAt: new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS - 5 * DAY_MS), + }, + }); + + const send = vi.fn(async () => "job-id"); + vi.mocked(getGlobalBoss).mockReturnValue({ send } as never); + const result = await enqueueBootTimeRollupBackfill(); + expect(result.error).toBeNull(); + const found = ( + send.mock.calls as unknown as Array<[string, { userId: string }]> + ) + .map((call) => call[1].userId) + .sort(); + + expect(found).toEqual([missingDay.id, nullExtremes.id].sort()); + expect(found).toEqual(await referenceDiscovery()); + }); +}); From e2998183c8b304eec2f38657b6a1e704c4ecc236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 29 Oct 2026 08:56:19 +0100 Subject: [PATCH 05/41] fix(measurements): delete a blood-pressure reading as one Systolic, diastolic and the cuff's pulse are three rows with no link column, and the single DELETE tombstoned only the row it was given, so deleting a reading's systolic left its diastolic and pulse behind as half a reading. The bulk delete had the same gap. A reading is the rows with the same owner, the same source and the identical measuredAt: that is how the entry form, the MCP write, a Withings measure group and a HealthKit correlation all store one, and how the FHIR read pairs a diastolic onto its systolic. The looser five-minute pairing of the chart reads is right for drawing a point and wrong for taking rows away. Deleting a systolic or diastolic now tombstones the rest of its reading in the same transaction with the same deletedAt, each row with its own syncVersion bump so paired clients see every tombstone. The restore route brings back the rows deleted together with a named half (identical deletedAt), so the undo, which names only the row that was tapped, puts the whole reading back, from the web list and from any client that calls it the same way. A pulse is still deleted on its own, and a row deleted at another moment stays deleted. The response shapes do not change; the OpenAPI descriptions say what the counts now include. --- docs/api/openapi.yaml | 13 +- src/app/api/measurements/[id]/route.ts | 47 ++++- .../bulk-delete/__tests__/route.test.ts | 28 ++- src/app/api/measurements/bulk-delete/route.ts | 33 ++- .../restore/__tests__/route.test.ts | 28 ++- src/app/api/measurements/restore/route.ts | 43 +++- .../measurements/blood-pressure-reading.ts | 82 ++++++++ src/lib/openapi/routes/measurements.ts | 6 +- .../blood-pressure-reading-delete.test.ts | 198 ++++++++++++++++++ 9 files changed, 430 insertions(+), 48 deletions(-) create mode 100644 src/lib/measurements/blood-pressure-reading.ts create mode 100644 tests/integration/blood-pressure-reading-delete.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 479a40268..f8c36f55f 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -4825,7 +4825,10 @@ paths: tags: - Measurements summary: Delete a measurement - description: Soft-deletes the row (tombstone) so paired clients can reconcile the deletion. + description: "Soft-deletes the row (tombstone) so paired clients can reconcile the deletion. A blood-pressure systolic + or diastolic takes the rest of its reading with it: the other half and the pulse with the same source and the + identical `measuredAt`, tombstoned in the same transaction with the same `deletedAt`, each with its own + `syncVersion` bump. A pulse is deleted on its own." parameters: - in: path name: id @@ -4936,7 +4939,8 @@ paths: description: Soft-deletes (tombstones) up to 200 of the caller's measurement rows in one call, mirroring the single-DELETE contract. Idempotent via the `Idempotency-Key` header; rate-limited `measurements:bulk-delete:`. Forged / foreign / already-deleted ids are silently skipped — `deleted` is - the count of rows actually tombstoned. The deletions surface in `/api/sync/changes` as tombstones. + the count of rows actually tombstoned, which includes the rest of any blood-pressure reading a selected systolic + or diastolic belongs to (as on the single DELETE). The deletions surface in `/api/sync/changes` as tombstones. parameters: - *a6 requestBody: @@ -4965,7 +4969,10 @@ paths: description: Un-tombstones up to 200 of the caller's soft-deleted measurement rows in one call — the delete-Undo counterpart to `/api/measurements/bulk-delete`. Idempotent via the `Idempotency-Key` header; rate-limited `measurements:restore:`. Forged / foreign / not-deleted ids are silently skipped — `restored` is the - count of rows actually un-tombstoned. The restored rows re-surface in `/api/sync/changes` as upserts. + count of rows actually un-tombstoned. A blood-pressure systolic or diastolic also restores the rows of its + reading that were deleted with it (same source, identical `measuredAt` and `deletedAt`), so naming one half + undoes the whole reading; a row deleted at another moment stays deleted. The restored rows re-surface in + `/api/sync/changes` as upserts. parameters: - *a6 requestBody: diff --git a/src/app/api/measurements/[id]/route.ts b/src/app/api/measurements/[id]/route.ts index 84e18c624..be24ec3a2 100644 --- a/src/app/api/measurements/[id]/route.ts +++ b/src/app/api/measurements/[id]/route.ts @@ -23,6 +23,7 @@ import { } from "@/lib/measurements/apple-health-mirror"; import { invalidateUserMeasurements } from "@/lib/cache/invalidate"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; +import { bloodPressureSiblings } from "@/lib/measurements/blood-pressure-reading"; import { Prisma, type MeasurementType } from "@/generated/prisma/client"; import { NextRequest } from "next/server"; import { z } from "zod"; @@ -237,18 +238,45 @@ export const DELETE = apiHandler( // (see `measurements/route.ts:100`), so the row is invisible to // normal reads from this point on. A row that is already tombstoned // re-bumps `syncVersion` harmlessly (idempotent re-delete). - await prisma.measurement.update({ - where: { id }, - data: { - deletedAt: new Date(), - syncVersion: { increment: 1 }, - }, + // + // A systolic or diastolic takes the rest of its reading with it (the + // other half and the cuff's pulse, `bloodPressureSiblings`), in the same + // transaction and with the same `deletedAt`, which is how the restore + // route knows to bring them back together. Each sibling bumps its own + // `syncVersion`, so a paired client sees every row's tombstone. + const deletedAt = new Date(); + const siblings = await prisma.$transaction(async (tx) => { + const found = await bloodPressureSiblings( + tx, + user.id, + [existing], + "live", + ); + await tx.measurement.update({ + where: { id }, + data: { deletedAt, syncVersion: { increment: 1 } }, + }); + if (found.length > 0) { + await tx.measurement.updateMany({ + where: { + id: { in: found.map((s) => s.id) }, + userId: user.id, + deletedAt: null, + }, + data: { deletedAt, syncVersion: { increment: 1 } }, + }); + } + return found; }); await auditLog("measurement.delete", { userId: user.id, ipAddress: getClientIp(request), - details: { measurementId: id, type: existing.type }, + details: { + measurementId: id, + type: existing.type, + readingRows: siblings.length, + }, }); annotate({ @@ -269,7 +297,10 @@ export const DELETE = apiHandler( // the rollup row when the day's measurement count goes to zero). await afterMeasurementMutation( user.id, - [{ type: existing.type, measuredAt: existing.measuredAt }], + [ + { type: existing.type, measuredAt: existing.measuredAt }, + ...siblings.map((s) => ({ type: s.type, measuredAt: s.measuredAt })), + ], { source: "manual" }, ); diff --git a/src/app/api/measurements/bulk-delete/__tests__/route.test.ts b/src/app/api/measurements/bulk-delete/__tests__/route.test.ts index b41d53ccb..f546f195d 100644 --- a/src/app/api/measurements/bulk-delete/__tests__/route.test.ts +++ b/src/app/api/measurements/bulk-delete/__tests__/route.test.ts @@ -10,15 +10,19 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; -vi.mock("@/lib/db", () => ({ - prisma: { +vi.mock("@/lib/db", () => { + const prisma = { measurement: { findMany: vi.fn(), updateMany: vi.fn(), }, auditLog: { create: vi.fn() }, - }, -})); + // The read and the write run in one transaction; the stub hands the + // callback the same mocked client. + $transaction: vi.fn(async (fn: (tx: unknown) => unknown) => fn(prisma)), + }; + return { prisma }; +}); vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); vi.mock("@/lib/auth/audit", () => ({ @@ -113,8 +117,16 @@ describe("POST /api/measurements/bulk-delete", () => { it("soft-deletes only owned, non-tombstoned rows and returns that count", async () => { // Caller asked for 3 ids but only 2 are owned + live. vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { type: "WEIGHT", measuredAt: new Date("2026-01-01T08:00:00Z") }, - { type: "WEIGHT", measuredAt: new Date("2026-01-01T20:00:00Z") }, + { + id: "m1", + type: "WEIGHT", + measuredAt: new Date("2026-01-01T08:00:00Z"), + }, + { + id: "m2", + type: "WEIGHT", + measuredAt: new Date("2026-01-01T20:00:00Z"), + }, ] as never); vi.mocked(prisma.measurement.updateMany).mockResolvedValue({ count: 2, @@ -129,7 +141,9 @@ describe("POST /api/measurements/bulk-delete", () => { // never 404s, it's just excluded from the count. const call = vi.mocked(prisma.measurement.updateMany).mock.calls[0][0]; expect(call.where).toMatchObject({ - id: { in: ["m1", "m2", "foreign"] }, + // The write names the rows the scoped read found, never the + // forged id the caller sent. + id: { in: ["m1", "m2"] }, userId: "user-1", deletedAt: null, }); diff --git a/src/app/api/measurements/bulk-delete/route.ts b/src/app/api/measurements/bulk-delete/route.ts index 03ea9abc3..16e2a22ee 100644 --- a/src/app/api/measurements/bulk-delete/route.ts +++ b/src/app/api/measurements/bulk-delete/route.ts @@ -7,6 +7,8 @@ * * Soft-deletes (tombstones) every owned, not-already-deleted row in one * `updateMany` — matching the single-DELETE iOS tombstone contract. A + * selected blood-pressure half takes the rest of its reading with it, as the + * single DELETE does, and `deleted` counts every row tombstoned. A * forged / foreign id is a silent no-op (never a 404 existence leak): the * mutation `where` pins `userId` so it only ever touches the caller's rows. * @@ -26,6 +28,7 @@ import { withIdempotency } from "@/lib/idempotency"; import { checkRateLimit } from "@/lib/rate-limit"; import { invalidateUserMeasurements } from "@/lib/cache/invalidate"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; +import { bloodPressureSiblings } from "@/lib/measurements/blood-pressure-reading"; import type { MeasurementType } from "@/generated/prisma/client"; const MAX_IDS_PER_BATCH = 200; @@ -73,17 +76,27 @@ async function postBulkDelete(request: NextRequest): Promise { // a status-insight cache drop. Ownership-scoped + `deletedAt: null` so // a forged / foreign / already-tombstoned id is excluded here exactly // as it is from the mutation below — no existence leak. - const affected = await prisma.measurement.findMany({ - where: { id: { in: ids }, userId: user.id, deletedAt: null }, - select: { type: true, measuredAt: true }, - }); + // + // A selected systolic or diastolic brings the rest of its reading (the + // other half and the cuff's pulse, `bloodPressureSiblings`) into the same + // statement, so the shared `deletedAt` lets the restore route bring the + // reading back whole. + const { affected, count } = await prisma.$transaction(async (tx) => { + const selected = await tx.measurement.findMany({ + where: { id: { in: ids }, userId: user.id, deletedAt: null }, + select: { id: true, type: true, source: true, measuredAt: true }, + }); + const siblings = await bloodPressureSiblings(tx, user.id, selected, "live"); + const rowIds = [...selected, ...siblings].map((row) => row.id); - // Soft-delete (tombstone) in one statement. The `where` pins `userId` - // so a forged id for another user is silently a no-op; `deletedAt: null` - // keeps the count honest (an already-tombstoned id doesn't re-count). - const { count } = await prisma.measurement.updateMany({ - where: { id: { in: ids }, userId: user.id, deletedAt: null }, - data: { deletedAt: now, syncVersion: { increment: 1 } }, + // Soft-delete (tombstone) in one statement. The `where` pins `userId` + // so a forged id for another user is silently a no-op; `deletedAt: null` + // keeps the count honest (an already-tombstoned id doesn't re-count). + const { count } = await tx.measurement.updateMany({ + where: { id: { in: rowIds }, userId: user.id, deletedAt: null }, + data: { deletedAt: now, syncVersion: { increment: 1 } }, + }); + return { affected: [...selected, ...siblings], count }; }); await auditLog("measurement.delete.bulk", { diff --git a/src/app/api/measurements/restore/__tests__/route.test.ts b/src/app/api/measurements/restore/__tests__/route.test.ts index 0fa7e8427..73ce2540e 100644 --- a/src/app/api/measurements/restore/__tests__/route.test.ts +++ b/src/app/api/measurements/restore/__tests__/route.test.ts @@ -10,15 +10,19 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; -vi.mock("@/lib/db", () => ({ - prisma: { +vi.mock("@/lib/db", () => { + const prisma = { measurement: { findMany: vi.fn(), updateMany: vi.fn(), }, auditLog: { create: vi.fn() }, - }, -})); + // The read and the write run in one transaction; the stub hands the + // callback the same mocked client. + $transaction: vi.fn(async (fn: (tx: unknown) => unknown) => fn(prisma)), + }; + return { prisma }; +}); vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); vi.mock("@/lib/auth/audit", () => ({ @@ -110,8 +114,16 @@ describe("POST /api/measurements/restore", () => { it("un-tombstones only owned, tombstoned rows and returns that count", async () => { // Caller asked for 3 ids but only 2 are owned + tombstoned. vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { type: "WEIGHT", measuredAt: new Date("2026-01-01T08:00:00Z") }, - { type: "WEIGHT", measuredAt: new Date("2026-01-01T20:00:00Z") }, + { + id: "m1", + type: "WEIGHT", + measuredAt: new Date("2026-01-01T08:00:00Z"), + }, + { + id: "m2", + type: "WEIGHT", + measuredAt: new Date("2026-01-01T20:00:00Z"), + }, ] as never); vi.mocked(prisma.measurement.updateMany).mockResolvedValue({ count: 2, @@ -126,7 +138,9 @@ describe("POST /api/measurements/restore", () => { // forged / live id never 404s, it's just excluded from the count. const call = vi.mocked(prisma.measurement.updateMany).mock.calls[0][0]; expect(call.where).toMatchObject({ - id: { in: ["m1", "m2", "foreign"] }, + // The write names the rows the scoped read found, never the + // forged id the caller sent. + id: { in: ["m1", "m2"] }, userId: "user-1", deletedAt: { not: null }, }); diff --git a/src/app/api/measurements/restore/route.ts b/src/app/api/measurements/restore/route.ts index 3c2a7b92d..62aeab9a7 100644 --- a/src/app/api/measurements/restore/route.ts +++ b/src/app/api/measurements/restore/route.ts @@ -28,6 +28,7 @@ import { withIdempotency } from "@/lib/idempotency"; import { checkRateLimit } from "@/lib/rate-limit"; import { invalidateUserMeasurements } from "@/lib/cache/invalidate"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; +import { bloodPressureSiblings } from "@/lib/measurements/blood-pressure-reading"; import type { MeasurementType } from "@/generated/prisma/client"; const MAX_IDS_PER_BATCH = 200; @@ -73,17 +74,39 @@ async function postRestore(request: NextRequest): Promise { // a status-insight cache drop. Ownership-scoped + `deletedAt != null` // so a forged / foreign / never-deleted id is excluded here exactly as // it is from the mutation below — no existence leak. - const affected = await prisma.measurement.findMany({ - where: { id: { in: ids }, userId: user.id, deletedAt: { not: null } }, - select: { type: true, measuredAt: true }, - }); + // + // A blood-pressure half brings back the rows of its reading that were + // tombstoned in the same statement (identical `deletedAt`, + // `bloodPressureSiblings`), so an undo that names only the row the person + // tapped restores the reading the delete took. A row deleted on its own, + // at another moment, stays deleted. + const { affected, count } = await prisma.$transaction(async (tx) => { + const selected = await tx.measurement.findMany({ + where: { id: { in: ids }, userId: user.id, deletedAt: { not: null } }, + select: { + id: true, + type: true, + source: true, + measuredAt: true, + deletedAt: true, + }, + }); + const siblings = await bloodPressureSiblings( + tx, + user.id, + selected, + "deletedWith", + ); + const rowIds = [...selected, ...siblings].map((row) => row.id); - // Un-tombstone in one statement. The `where` pins `userId` so a forged - // id for another user is silently a no-op; `deletedAt: { not: null }` - // keeps the count honest (a live row doesn't re-count). - const { count } = await prisma.measurement.updateMany({ - where: { id: { in: ids }, userId: user.id, deletedAt: { not: null } }, - data: { deletedAt: null, syncVersion: { increment: 1 } }, + // Un-tombstone in one statement. The `where` pins `userId` so a forged + // id for another user is silently a no-op; `deletedAt: { not: null }` + // keeps the count honest (a live row doesn't re-count). + const { count } = await tx.measurement.updateMany({ + where: { id: { in: rowIds }, userId: user.id, deletedAt: { not: null } }, + data: { deletedAt: null, syncVersion: { increment: 1 } }, + }); + return { affected: [...selected, ...siblings], count }; }); await auditLog("measurement.restore", { diff --git a/src/lib/measurements/blood-pressure-reading.ts b/src/lib/measurements/blood-pressure-reading.ts new file mode 100644 index 000000000..589a7fb66 --- /dev/null +++ b/src/lib/measurements/blood-pressure-reading.ts @@ -0,0 +1,82 @@ +/** + * One blood-pressure reading is up to three rows: systolic, diastolic and + * the pulse the cuff measured with them. No column links them. What makes + * them one reading is how every writer stores it: the entry form, the MCP + * write, a Withings measure group and a HealthKit correlation all write the + * halves with the same source at the same instant, and the FHIR read pairs a + * diastolic onto its systolic by exactly that instant. + * + * So the rule here is the strict one: same owner, same source, identical + * `measuredAt`. The series and in-target reads pair more loosely (nearest + * diastolic within five minutes) because they only draw a point; a delete + * takes rows away, so it takes only the rows that can be nothing else. + * + * Deleting a systolic or a diastolic takes the whole reading with it. + * Deleting a pulse takes only the pulse: a pulse is also a value of its own, + * and removing one is no reason to remove a blood pressure. + */ +import type { MeasurementType, Prisma } from "@/generated/prisma/client"; + +/** The types that anchor a reading: deleting one deletes the reading. */ +const BP_HALVES: ReadonlySet = new Set([ + "BLOOD_PRESSURE_SYS", + "BLOOD_PRESSURE_DIA", +]); + +/** Every type a reading's rows can carry. */ +const BP_READING_TYPES: MeasurementType[] = [ + "BLOOD_PRESSURE_SYS", + "BLOOD_PRESSURE_DIA", + "PULSE", +]; + +export interface ReadingAnchor { + id: string; + type: MeasurementType | string; + source: string; + measuredAt: Date; + deletedAt?: Date | null; +} + +type MeasurementReader = { + measurement: Pick; +}; + +export interface ReadingSibling { + id: string; + type: MeasurementType; + measuredAt: Date; +} + +/** + * The other rows of the readings `rows` belong to, not counting `rows` + * themselves. `state: "live"` finds the rows a delete takes with it; + * `state: "deletedWith"` finds the rows that were tombstoned in the same + * statement as their anchor (the identical `deletedAt`), which is what a + * restore brings back with it, and never a row deleted on its own. + */ +export async function bloodPressureSiblings( + db: MeasurementReader, + userId: string, + rows: readonly ReadingAnchor[], + state: "live" | "deletedWith", +): Promise { + const anchors = rows.filter( + (r) => BP_HALVES.has(r.type) && (state === "live" || r.deletedAt), + ); + if (anchors.length === 0) return []; + const found = await db.measurement.findMany({ + where: { + userId, + id: { notIn: rows.map((r) => r.id) }, + type: { in: BP_READING_TYPES }, + OR: anchors.map((a) => ({ + source: a.source as Prisma.MeasurementWhereInput["source"], + measuredAt: a.measuredAt, + deletedAt: state === "live" ? null : a.deletedAt, + })), + }, + select: { id: true, type: true, measuredAt: true }, + }); + return found; +} diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index c8a3dc8af..b04e40220 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -1055,7 +1055,7 @@ export const measurementPaths: NonNullable = { tags: ["Measurements"], summary: "Delete a measurement", description: - "Soft-deletes the row (tombstone) so paired clients can reconcile the deletion.", + "Soft-deletes the row (tombstone) so paired clients can reconcile the deletion. A blood-pressure systolic or diastolic takes the rest of its reading with it: the other half and the pulse with the same source and the identical `measuredAt`, tombstoned in the same transaction with the same `deletedAt`, each with its own `syncVersion` bump. A pulse is deleted on its own.", requestParams: { path: z.object({ id: z.string() }) }, responses: { ...recordRefusal(), @@ -1145,7 +1145,7 @@ export const measurementPaths: NonNullable = { tags: ["Measurements"], summary: "Bulk soft-delete measurements", description: - "Soft-deletes (tombstones) up to 200 of the caller's measurement rows in one call, mirroring the single-DELETE contract. Idempotent via the `Idempotency-Key` header; rate-limited `measurements:bulk-delete:`. Forged / foreign / already-deleted ids are silently skipped — `deleted` is the count of rows actually tombstoned. The deletions surface in `/api/sync/changes` as tombstones.", + "Soft-deletes (tombstones) up to 200 of the caller's measurement rows in one call, mirroring the single-DELETE contract. Idempotent via the `Idempotency-Key` header; rate-limited `measurements:bulk-delete:`. Forged / foreign / already-deleted ids are silently skipped — `deleted` is the count of rows actually tombstoned, which includes the rest of any blood-pressure reading a selected systolic or diastolic belongs to (as on the single DELETE). The deletions surface in `/api/sync/changes` as tombstones.", requestBody: { required: true, content: { @@ -1176,7 +1176,7 @@ export const measurementPaths: NonNullable = { tags: ["Measurements"], summary: "Restore soft-deleted measurements", description: - "Un-tombstones up to 200 of the caller's soft-deleted measurement rows in one call — the delete-Undo counterpart to `/api/measurements/bulk-delete`. Idempotent via the `Idempotency-Key` header; rate-limited `measurements:restore:`. Forged / foreign / not-deleted ids are silently skipped — `restored` is the count of rows actually un-tombstoned. The restored rows re-surface in `/api/sync/changes` as upserts.", + "Un-tombstones up to 200 of the caller's soft-deleted measurement rows in one call — the delete-Undo counterpart to `/api/measurements/bulk-delete`. Idempotent via the `Idempotency-Key` header; rate-limited `measurements:restore:`. Forged / foreign / not-deleted ids are silently skipped — `restored` is the count of rows actually un-tombstoned. A blood-pressure systolic or diastolic also restores the rows of its reading that were deleted with it (same source, identical `measuredAt` and `deletedAt`), so naming one half undoes the whole reading; a row deleted at another moment stays deleted. The restored rows re-surface in `/api/sync/changes` as upserts.", requestBody: { required: true, content: { diff --git a/tests/integration/blood-pressure-reading-delete.test.ts b/tests/integration/blood-pressure-reading-delete.test.ts new file mode 100644 index 000000000..d7f4028d9 --- /dev/null +++ b/tests/integration/blood-pressure-reading-delete.test.ts @@ -0,0 +1,198 @@ +/** + * Deleting one row of a blood-pressure reading deletes the reading, through + * the real routes against a real Postgres. + * + * Systolic, diastolic and the cuff's pulse are three rows with no link + * column; they are one reading because every writer stores them with the + * same source at the same instant. Deleting the systolic used to tombstone + * only the systolic and leave a diastolic and a pulse behind. Now the + * single DELETE and the bulk delete take the whole reading, every row bumps + * its own syncVersion (so the sync feed carries each tombstone), and the + * undo, which names only the row the person tapped, brings the whole + * reading back. A reading of another source at the same instant, a reading + * a minute later, and a pulse deleted on its own are left alone. + */ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const USER = "user-bp-reading-delete"; +const AT = new Date("2026-10-20T07:30:00.000Z"); +const MINUTE_LATER = new Date(AT.getTime() + 60_000); + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + cookieJar.clear(); + headerJar.clear(); + await prisma.user.create({ + data: { + id: USER, + username: "bp-reading-delete", + email: "bp-reading-delete@example.test", + }, + }); + const session = await prisma.session.create({ + data: { userId: USER, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + + const row = ( + id: string, + type: "BLOOD_PRESSURE_SYS" | "BLOOD_PRESSURE_DIA" | "PULSE", + value: number, + source: "MANUAL" | "WITHINGS", + measuredAt: Date, + ) => ({ + id, + userId: USER, + type, + value, + unit: type === "PULSE" ? "bpm" : "mmHg", + source, + measuredAt, + }); + await prisma.measurement.createMany({ + data: [ + // The reading under test. + row("sys", "BLOOD_PRESSURE_SYS", 128, "MANUAL", AT), + row("dia", "BLOOD_PRESSURE_DIA", 82, "MANUAL", AT), + row("pulse", "PULSE", 64, "MANUAL", AT), + // The same instant from another source: another reading. + row("w-sys", "BLOOD_PRESSURE_SYS", 131, "WITHINGS", AT), + row("w-dia", "BLOOD_PRESSURE_DIA", 84, "WITHINGS", AT), + // A minute later from the same source: another reading. + row("next-sys", "BLOOD_PRESSURE_SYS", 125, "MANUAL", MINUTE_LATER), + row("next-dia", "BLOOD_PRESSURE_DIA", 80, "MANUAL", MINUTE_LATER), + row("next-pulse", "PULSE", 62, "MANUAL", MINUTE_LATER), + ], + }); +}); + +async function rows(): Promise< + Record +> { + const all = await getPrismaClient().measurement.findMany({ + where: { userId: USER }, + select: { id: true, deletedAt: true, syncVersion: true }, + }); + return Object.fromEntries( + all.map((r) => [ + r.id, + { deleted: r.deletedAt !== null, syncVersion: r.syncVersion }, + ]), + ); +} + +const deletedIds = async () => + Object.entries(await rows()) + .filter(([, r]) => r.deleted) + .map(([id]) => id) + .sort(); + +async function deleteOne(id: string) { + const { DELETE } = await import("@/app/api/measurements/[id]/route"); + const res = await DELETE( + new NextRequest(`http://localhost/api/measurements/${id}`, { + method: "DELETE", + }), + { params: Promise.resolve({ id }) }, + ); + expect(res.status).toBe(200); +} + +async function post(path: "bulk-delete" | "restore", ids: string[]) { + const route = + path === "bulk-delete" + ? await import("@/app/api/measurements/bulk-delete/route") + : await import("@/app/api/measurements/restore/route"); + const res = await route.POST( + new NextRequest(`http://localhost/api/measurements/${path}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ids }), + }), + ); + expect(res.status).toBe(200); + return (await res.json()).data as { deleted?: number; restored?: number }; +} + +describe("deleting a blood-pressure reading", () => { + it("takes the other half and the pulse with the systolic, and the undo brings all three back", async () => { + await deleteOne("sys"); + + expect(await deletedIds()).toEqual(["dia", "pulse", "sys"]); + const after = await rows(); + for (const id of ["sys", "dia", "pulse"]) { + expect(after[id]!.syncVersion).toBe(2); + } + for (const id of ["w-sys", "w-dia", "next-sys", "next-dia"]) { + expect(after[id]!.syncVersion).toBe(1); + } + + // The undo names only the row that was tapped. + const { restored } = await post("restore", ["sys"]); + expect(restored).toBe(3); + expect(await deletedIds()).toEqual([]); + }); + + it("takes the systolic and the pulse with the diastolic", async () => { + await deleteOne("dia"); + expect(await deletedIds()).toEqual(["dia", "pulse", "sys"]); + }); + + it("deletes a pulse on its own, and its undo leaves the reading as it was", async () => { + await deleteOne("pulse"); + expect(await deletedIds()).toEqual(["pulse"]); + await post("restore", ["pulse"]); + expect(await deletedIds()).toEqual([]); + }); + + it("does the same for the bulk delete, and restores only what went in the same delete", async () => { + // The pulse of the second reading was deleted on its own earlier. + await deleteOne("next-pulse"); + + const { deleted } = await post("bulk-delete", ["sys", "next-dia"]); + expect(deleted).toBe(5); + expect(await deletedIds()).toEqual([ + "dia", + "next-dia", + "next-pulse", + "next-sys", + "pulse", + "sys", + ]); + + const { restored } = await post("restore", ["sys", "next-dia"]); + expect(restored).toBe(5); + // The pulse deleted at another moment stays deleted. + expect(await deletedIds()).toEqual(["next-pulse"]); + }); +}); From bbef4eea993ffb12881c8e445aef6fe75821025c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 29 Oct 2026 08:56:51 +0100 Subject: [PATCH 06/41] feat(auth): let a single-sign-on account confirm it is you at its provider An account created through OIDC holds no password, passkey or second factor, so the recent-proof gate in front of token minting, MCP connections, exports and the admin reset had nothing to ask it for and the web could only say "sign in again" (#1296). GET /api/auth/oidc/reproof sends the browser back to the configured provider with prompt=login and max_age=0, its own state, nonce and PKCE verifier, and the signed-in account and session sealed into the state cookie. The shared callback branches on that intent: it refuses unless the browser still carries the bound session (checked before the code is redeemed), the ID token names the account's linked issuer and subject, and auth_time lies inside the gate's five-minute window. It then stamps Session.reproofAt exactly as a password re-proof does, creates no session, and returns to the page it left with ?reproof=done|failed. recentProofMethods offers "oidc" only to an account without a second factor whose identity is linked while a provider is configured, and the fresh-factor gate never lists it. GET /api/auth/me/mfa publishes the list as proofMethods, so the passkey card offers what the account can actually give, and register-options takes { method: "oidc" } on a fresh provider re-proof, so such an account can add a passkey as a second way to confirm. Both re-proof dialogs offer the provider and a toast reports the outcome on return. The callback's missing-state branch and the passkey card's enrollment flow move into their own functions to keep both handlers under the size ratchet. --- docs/api/openapi.yaml | 41 ++- messages/de.json | 11 +- messages/en.json | 11 +- messages/es.json | 11 +- messages/fr.json | 11 +- messages/it.json | 11 +- messages/ko.json | 11 +- messages/nl.json | 11 +- messages/pl.json | 11 +- messages/pt-BR.json | 11 +- .../baselines/source-function-lines.json | 6 +- src/__tests__/baselines/test-as-never.json | 5 +- .../openapi-route-coverage-guard.test.ts | 4 + src/__tests__/session-surface-guard.test.ts | 5 +- src/app/api/auth/me/mfa/route.ts | 8 +- src/app/api/auth/oidc/callback/route.ts | 149 +++++++-- src/app/api/auth/oidc/reproof/route.ts | 118 +++++++ .../auth/passkey/register-options/route.ts | 69 +++- src/components/auth/oidc-reproof-return.tsx | 40 +++ src/components/providers.tsx | 2 + .../__tests__/oidc-reproof-choice.test.ts | 78 +++++ .../existing-factor-reauth-dialog.tsx | 66 +++- .../settings/security-section/index.tsx | 6 +- .../security-section/passkey-list-section.tsx | 178 +++++++--- .../security-section/recent-proof-flow.ts | 5 +- src/lib/auth/__tests__/oidc-reproof.test.ts | 281 ++++++++++++++++ src/lib/auth/__tests__/oidc.test.ts | 89 ++++- src/lib/auth/existing-factor-proof.ts | 3 +- src/lib/auth/guards/fresh-mfa.ts | 6 +- src/lib/auth/oidc-callback/state.ts | 30 +- src/lib/auth/oidc-reproof.ts | 158 +++++++++ src/lib/auth/oidc.ts | 21 +- src/lib/auth/second-factor.ts | 35 +- src/lib/openapi/routes/auth.ts | 23 +- tests/integration/oidc-reproof.test.ts | 316 ++++++++++++++++++ 35 files changed, 1654 insertions(+), 187 deletions(-) create mode 100644 src/app/api/auth/oidc/reproof/route.ts create mode 100644 src/components/auth/oidc-reproof-return.tsx create mode 100644 src/components/settings/security-section/__tests__/oidc-reproof-choice.test.ts create mode 100644 src/lib/auth/__tests__/oidc-reproof.test.ts create mode 100644 src/lib/auth/oidc-reproof.ts create mode 100644 tests/integration/oidc-reproof.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 479a40268..7eae195e0 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -2481,10 +2481,13 @@ paths: The body takes the same shapes as POST /api/auth/step-up. On an account with a second factor only `totp`, `webauthn` (a security key) or `passkey` are accepted, because a password has never stood in for the second - factor; on one without, `password` or `passkey`. A method the account cannot use here is refused with 422 - `auth.reproof.too_weak` and `meta.methods`, before any guess is spent. A second-factor or passkey proof also - refreshes the session's second-factor stamp, exactly as signing in again would. Assertions begin at POST - /api/auth/passkey/register-options with `{ method: "passkey" | "webauthn" }`. + factor; on one without, `password` or `passkey`. An account without a second factor whose identity is linked to + the configured identity provider is also offered `oidc` in `meta.methods`; that proof is not posted here but + given by the browser round trip GET /api/auth/oidc/reproof, which stamps the session exactly as a password proof + does. A method the account cannot use here is refused with 422 `auth.reproof.too_weak` and `meta.methods`, + before any guess is spent. A second-factor or passkey proof also refreshes the session's second-factor stamp, + exactly as signing in again would. Assertions begin at POST /api/auth/passkey/register-options with `{ method: + "passkey" | "webauthn" }`. Proofs draw on the account's shared re-proof budget (five per fifteen minutes, 429 once spent). A proof that @@ -4077,7 +4080,9 @@ paths: **Enrollment arm.** Any other body must be a full factor proof (the same shape POST /api/auth/step-up takes): a password, a TOTP code, or a completed passkey / security-key assertion. Only then are registration options - issued. On an account that already has a second factor a password is refused before it is checked, with 422 + issued. An account confirmed at its identity provider (GET /api/auth/oidc/reproof, within five minutes) sends `{ + method: "oidc" }` instead, which takes that re-proof as the proof and, like a password, clears the second-factor + stamp. On an account that already has a second factor a password is refused before it is checked, with 422 `auth.reproof.too_weak` and `meta.methods` naming the proofs that work: a passkey added on the password alone would sign in without the second factor from then on. @@ -4103,6 +4108,7 @@ paths: schema: anyOf: - $ref: "#/components/schemas/PasskeyReauthRequest" + - $ref: "#/components/schemas/PasskeyProviderProofRequest" - $ref: "#/components/schemas/StepUpMintRequest" responses: "200": @@ -28611,6 +28617,17 @@ components: description: The re-proof arm of register-options. Recognised ONLY when `method` is the body's single key — adding any other property routes the request to the enrollment arm, where it will be read as a factor proof and almost certainly refused. + PasskeyProviderProofRequest: + type: object + properties: + method: + type: string + const: oidc + required: + - method + description: Take the session's recent identity-provider re-proof (GET /api/auth/oidc/reproof) as the proof. Accepted + only for an account whose `proofMethods` list `oidc` and only within five minutes of that re-proof; otherwise + 401 `auth.reproof.required` with `meta.methods`. Web only. PasskeyRegisterVerifyRequest: type: object properties: @@ -47232,11 +47249,25 @@ components: $ref: "#/components/schemas/WebauthnMfaCredentialInfo" passkeyNudgeDismissed: type: boolean + proofMethods: + type: array + items: + type: string + enum: + - password + - oidc + - totp + - webauthn + - passkey + description: The ways this account can confirm it is you, the same list a recent-proof refusal names in `meta.methods`. + `password` and `oidc` only on an account without a second factor; `oidc` is a browser round trip to the + identity provider (`GET /api/auth/oidc/reproof`) and is never offered to a token. required: - totp - recoveryCodesRemaining - webauthn - passkeyNudgeDismissed + - proofMethods additionalProperties: false WebauthnMfaCredentialInfo: type: object diff --git a/messages/de.json b/messages/de.json index 0fe8a7223..09da70eff 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6237,14 +6237,17 @@ "password": "Passwort", "totp": "Authenticator-Code", "passkey": "Passkey", - "webauthn": "Sicherheitsschlüssel" + "webauthn": "Sicherheitsschlüssel", + "oidc": "Identitätsanbieter" }, "currentPassword": "Aktuelles Passwort", "authenticatorCode": "Authenticator-Code", "authenticatorPlaceholder": "Sechsstelliger Authenticator-Code", "verifyAndAdd": "Bestätigen und hinzufügen", "factorDescription": "Bestätige eine vorhandene Anmeldemethode, bevor du einen zweiten Faktor hinzufügst.", - "verifyAndContinue": "Bestätigen und fortfahren" + "verifyAndContinue": "Bestätigen und fortfahren", + "oidcHint": "Du meldest dich bei deinem Identitätsanbieter noch einmal an und kommst direkt auf diese Seite zurück. Dann wiederholst du, was du gerade tun wolltest.", + "continueToProvider": "Weiter zur Anmeldung" }, "passkeyName": "Name", "noPasskeys": "Keine Passkeys registriert.", @@ -7517,7 +7520,9 @@ "description": "Damit werden deine Gesundheitsdaten weitergegeben oder exportiert. Bestätige, dass du es bist, um fortzufahren.", "descriptionChange": "Das ändert etwas, das nur du ändern können solltest. Bestätige, dass du es bist, um fortzufahren.", "signInAgain": "Melde dich neu an und versuche es dann noch einmal. Dein Konto hat kein Passwort und keinen Passkey, den du hier bestätigen könntest.", - "failed": "Das hat nicht geklappt. Prüfe deine Eingabe und versuche es noch einmal." + "failed": "Das hat nicht geklappt. Prüfe deine Eingabe und versuche es noch einmal.", + "oidcDone": "Bestätigt. In den nächsten fünf Minuten kannst du weitermachen, ohne erneut gefragt zu werden.", + "oidcFailed": "Dein Identitätsanbieter konnte nicht bestätigen, dass du es bist. Versuche es noch einmal." }, "passwordTitle": "Passwort", "tokenNameLabel": "Token-Name", diff --git a/messages/en.json b/messages/en.json index 3f99d31fe..fab5619d0 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6237,14 +6237,17 @@ "password": "Password", "totp": "Authenticator code", "passkey": "Passkey", - "webauthn": "Security key" + "webauthn": "Security key", + "oidc": "Identity provider" }, "currentPassword": "Current password", "authenticatorCode": "Authenticator code", "authenticatorPlaceholder": "6-digit authenticator code", "verifyAndAdd": "Verify and add", "factorDescription": "Verify an existing sign-in method before adding a second factor.", - "verifyAndContinue": "Verify and continue" + "verifyAndContinue": "Verify and continue", + "oidcHint": "You sign in once more at your identity provider and come straight back to this page. Then do what you were about to do again.", + "continueToProvider": "Continue to sign-in" }, "passkeyName": "Name", "noPasskeys": "No passkeys registered.", @@ -7517,7 +7520,9 @@ "description": "This shares or exports your health data. Confirm it's you to continue.", "descriptionChange": "This changes something only you should be able to change. Confirm it's you to continue.", "signInAgain": "Sign in again, then try once more. Your account has no password or passkey to confirm here.", - "failed": "That did not work. Check what you entered and try again." + "failed": "That did not work. Check what you entered and try again.", + "oidcDone": "Confirmed. For the next five minutes you can go ahead without being asked again.", + "oidcFailed": "Your identity provider could not confirm it's you. Try again." }, "passwordTitle": "Password", "tokenNameLabel": "Token name", diff --git a/messages/es.json b/messages/es.json index 4677c684f..1236f59ef 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6237,14 +6237,17 @@ "password": "Contraseña", "totp": "Código del autenticador", "passkey": "Clave de acceso", - "webauthn": "Llave de seguridad" + "webauthn": "Llave de seguridad", + "oidc": "Proveedor de identidad" }, "currentPassword": "Contraseña actual", "authenticatorCode": "Código del autenticador", "authenticatorPlaceholder": "Código de seis dígitos del autenticador", "verifyAndAdd": "Verificar y añadir", "factorDescription": "Verifica un método de inicio de sesión existente antes de añadir un segundo factor.", - "verifyAndContinue": "Verificar y continuar" + "verifyAndContinue": "Verificar y continuar", + "oidcHint": "Vuelves a iniciar sesión en tu proveedor de identidad y regresas directamente a esta página. Después repite lo que ibas a hacer.", + "continueToProvider": "Continuar al inicio de sesión" }, "passkeyName": "Nombre", "noPasskeys": "No hay claves de acceso registradas.", @@ -7517,7 +7520,9 @@ "description": "Esto comparte o exporta tus datos de salud. Confirma que eres tú para continuar.", "descriptionChange": "Esto cambia algo que solo tú deberías poder cambiar. Confirma que eres tú para continuar.", "signInAgain": "Vuelve a iniciar sesión y inténtalo de nuevo. Tu cuenta no tiene contraseña ni llave de acceso que confirmar aquí.", - "failed": "No ha funcionado. Revisa lo que has introducido e inténtalo de nuevo." + "failed": "No ha funcionado. Revisa lo que has introducido e inténtalo de nuevo.", + "oidcDone": "Confirmado. Durante los próximos cinco minutos puedes continuar sin que se te vuelva a preguntar.", + "oidcFailed": "Tu proveedor de identidad no pudo confirmar que eres tú. Inténtalo de nuevo." }, "passwordTitle": "Contraseña", "tokenNameLabel": "Nombre del token", diff --git a/messages/fr.json b/messages/fr.json index 76a13377a..fe9d0a531 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6237,14 +6237,17 @@ "password": "Mot de passe", "totp": "Code d’authentification", "passkey": "Clé d’accès", - "webauthn": "Clé de sécurité" + "webauthn": "Clé de sécurité", + "oidc": "Fournisseur d'identité" }, "currentPassword": "Mot de passe actuel", "authenticatorCode": "Code d’authentification", "authenticatorPlaceholder": "Code d’authentification à 6 chiffres", "verifyAndAdd": "Vérifier et ajouter", "factorDescription": "Vérifiez une méthode de connexion existante avant d'ajouter un second facteur.", - "verifyAndContinue": "Vérifier et continuer" + "verifyAndContinue": "Vérifier et continuer", + "oidcHint": "Vous vous reconnectez auprès de votre fournisseur d'identité et revenez directement sur cette page. Refaites ensuite ce que vous étiez en train de faire.", + "continueToProvider": "Continuer vers la connexion" }, "passkeyName": "Nom", "noPasskeys": "Aucune clé d’accès enregistrée.", @@ -7517,7 +7520,9 @@ "description": "Cette action partage ou exporte vos données de santé. Confirmez qu'il s'agit bien de vous pour continuer.", "descriptionChange": "Cela modifie quelque chose que vous seul devriez pouvoir modifier. Confirmez qu'il s'agit bien de vous pour continuer.", "signInAgain": "Reconnectez-vous, puis réessayez. Votre compte n'a ni mot de passe ni clé d'accès à confirmer ici.", - "failed": "Cela n'a pas fonctionné. Vérifiez votre saisie et réessayez." + "failed": "Cela n'a pas fonctionné. Vérifiez votre saisie et réessayez.", + "oidcDone": "C'est confirmé. Pendant les cinq prochaines minutes, vous pouvez continuer sans qu'on vous le redemande.", + "oidcFailed": "Votre fournisseur d'identité n'a pas pu confirmer que c'est bien vous. Réessayez." }, "passwordTitle": "Mot de passe", "tokenNameLabel": "Nom du jeton", diff --git a/messages/it.json b/messages/it.json index 0585ad838..490af61d9 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6237,14 +6237,17 @@ "password": "Password", "totp": "Codice dell’app di autenticazione", "passkey": "Passkey", - "webauthn": "Chiave di sicurezza" + "webauthn": "Chiave di sicurezza", + "oidc": "Provider di identità" }, "currentPassword": "Password attuale", "authenticatorCode": "Codice dell’app di autenticazione", "authenticatorPlaceholder": "Codice a 6 cifre dell’app di autenticazione", "verifyAndAdd": "Verifica e aggiungi", "factorDescription": "Verifica un metodo di accesso esistente prima di aggiungere un secondo fattore.", - "verifyAndContinue": "Verifica e continua" + "verifyAndContinue": "Verifica e continua", + "oidcHint": "Accedi di nuovo presso il tuo provider di identità e torni subito a questa pagina. Poi ripeti quello che stavi per fare.", + "continueToProvider": "Continua all'accesso" }, "passkeyName": "Nome", "noPasskeys": "Nessuna passkey registrata.", @@ -7517,7 +7520,9 @@ "description": "Questa azione condivide o esporta i tuoi dati sanitari. Conferma che sei tu per continuare.", "descriptionChange": "Questo modifica qualcosa che solo tu dovresti poter modificare. Conferma che sei tu per continuare.", "signInAgain": "Accedi di nuovo, poi riprova. Il tuo account non ha una password o una passkey da confermare qui.", - "failed": "Non ha funzionato. Controlla quanto inserito e riprova." + "failed": "Non ha funzionato. Controlla quanto inserito e riprova.", + "oidcDone": "Confermato. Per i prossimi cinque minuti puoi procedere senza che ti venga chiesto di nuovo.", + "oidcFailed": "Il tuo provider di identità non ha potuto confermare che sei tu. Riprova." }, "passwordTitle": "Password", "tokenNameLabel": "Nome del token", diff --git a/messages/ko.json b/messages/ko.json index 2a2b86dd3..1437b35ae 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6237,14 +6237,17 @@ "password": "비밀번호", "totp": "인증 앱 코드", "passkey": "패스키", - "webauthn": "보안 키" + "webauthn": "보안 키", + "oidc": "ID 공급자" }, "currentPassword": "현재 비밀번호", "authenticatorCode": "인증 앱 코드", "authenticatorPlaceholder": "6자리 인증 앱 코드", "verifyAndAdd": "확인 후 추가", "factorDescription": "두 번째 인증 수단을 추가하기 전에 기존 로그인 방법을 확인하세요.", - "verifyAndContinue": "확인하고 계속" + "verifyAndContinue": "확인하고 계속", + "oidcHint": "ID 공급자에서 한 번 더 로그인하면 바로 이 페이지로 돌아옵니다. 그런 다음 하려던 작업을 다시 진행하세요.", + "continueToProvider": "로그인으로 계속" }, "passkeyName": "이름", "noPasskeys": "등록된 패스키가 없어요.", @@ -7517,7 +7520,9 @@ "description": "건강 데이터를 공유하거나 내보내는 작업입니다. 계속하려면 본인임을 확인하세요.", "descriptionChange": "본인만 변경할 수 있어야 하는 항목을 변경합니다. 계속하려면 본인임을 확인하세요.", "signInAgain": "다시 로그인한 후 한 번 더 시도하세요. 이 계정에는 여기서 확인할 비밀번호나 패스키가 없습니다.", - "failed": "확인하지 못했습니다. 입력한 내용을 확인하고 다시 시도하세요." + "failed": "확인하지 못했습니다. 입력한 내용을 확인하고 다시 시도하세요.", + "oidcDone": "확인되었습니다. 앞으로 5분 동안은 다시 묻지 않고 계속할 수 있습니다.", + "oidcFailed": "ID 공급자가 본인임을 확인하지 못했습니다. 다시 시도하세요." }, "passwordTitle": "비밀번호", "tokenNameLabel": "토큰 이름", diff --git a/messages/nl.json b/messages/nl.json index e93adec7c..6d62e60fc 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -6237,14 +6237,17 @@ "password": "Wachtwoord", "totp": "Authenticatorcode", "passkey": "Passkey", - "webauthn": "Beveiligingssleutel" + "webauthn": "Beveiligingssleutel", + "oidc": "Identiteitsprovider" }, "currentPassword": "Huidig wachtwoord", "authenticatorCode": "Authenticatorcode", "authenticatorPlaceholder": "Authenticatorcode van 6 cijfers", "verifyAndAdd": "Verifiëren en toevoegen", "factorDescription": "Verifieer een bestaande inlogmethode voordat je een tweede factor toevoegt.", - "verifyAndContinue": "Verifiëren en doorgaan" + "verifyAndContinue": "Verifiëren en doorgaan", + "oidcHint": "Je logt nog een keer in bij je identiteitsprovider en komt meteen terug op deze pagina. Doe daarna opnieuw wat je wilde doen.", + "continueToProvider": "Verder naar inloggen" }, "passkeyName": "Naam", "noPasskeys": "Geen passkeys geregistreerd.", @@ -7517,7 +7520,9 @@ "description": "Hiermee deel of exporteer je je gezondheidsgegevens. Bevestig dat jij het bent om door te gaan.", "descriptionChange": "Hiermee wijzig je iets wat alleen jij zou moeten kunnen wijzigen. Bevestig dat jij het bent om door te gaan.", "signInAgain": "Log opnieuw in en probeer het daarna nog een keer. Je account heeft geen wachtwoord of passkey waarmee je hier kunt bevestigen.", - "failed": "Dat is niet gelukt. Controleer wat je hebt ingevoerd en probeer het opnieuw." + "failed": "Dat is niet gelukt. Controleer wat je hebt ingevoerd en probeer het opnieuw.", + "oidcDone": "Bevestigd. De komende vijf minuten kun je verder zonder dat het opnieuw wordt gevraagd.", + "oidcFailed": "Je identiteitsprovider kon niet bevestigen dat jij het bent. Probeer het opnieuw." }, "passwordTitle": "Wachtwoord", "tokenNameLabel": "Tokennaam", diff --git a/messages/pl.json b/messages/pl.json index 2055df78e..6dae52955 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6237,14 +6237,17 @@ "password": "Hasło", "totp": "Kod z aplikacji uwierzytelniającej", "passkey": "Klucz dostępu", - "webauthn": "Klucz bezpieczeństwa" + "webauthn": "Klucz bezpieczeństwa", + "oidc": "Dostawca tożsamości" }, "currentPassword": "Aktualne hasło", "authenticatorCode": "Kod z aplikacji uwierzytelniającej", "authenticatorPlaceholder": "6-cyfrowy kod z aplikacji uwierzytelniającej", "verifyAndAdd": "Zweryfikuj i dodaj", "factorDescription": "Potwierdź istniejącą metodę logowania, zanim dodasz drugi składnik.", - "verifyAndContinue": "Potwierdź i kontynuuj" + "verifyAndContinue": "Potwierdź i kontynuuj", + "oidcHint": "Zalogujesz się jeszcze raz u swojego dostawcy tożsamości i wrócisz od razu na tę stronę. Potem powtórz to, co chcesz zrobić.", + "continueToProvider": "Przejdź do logowania" }, "passkeyName": "Nazwa", "noPasskeys": "Brak zarejestrowanych kluczy dostępu.", @@ -7517,7 +7520,9 @@ "description": "Ta czynność udostępnia lub eksportuje Twoje dane zdrowotne. Potwierdź, że to Ty, aby kontynuować.", "descriptionChange": "To zmienia coś, co tylko Ty powinieneś móc zmienić. Potwierdź, że to Ty, aby kontynuować.", "signInAgain": "Zaloguj się ponownie, a potem spróbuj jeszcze raz. Twoje konto nie ma hasła ani klucza dostępu do potwierdzenia tutaj.", - "failed": "To się nie udało. Sprawdź wprowadzone dane i spróbuj ponownie." + "failed": "To się nie udało. Sprawdź wprowadzone dane i spróbuj ponownie.", + "oidcDone": "Potwierdzono. Przez następne pięć minut możesz kontynuować bez ponownego pytania.", + "oidcFailed": "Twój dostawca tożsamości nie mógł potwierdzić, że to Ty. Spróbuj ponownie." }, "passwordTitle": "Hasło", "tokenNameLabel": "Nazwa tokena", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index ae8919d20..f7762533c 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -6237,14 +6237,17 @@ "password": "Senha", "totp": "Código do autenticador", "passkey": "Chave de acesso", - "webauthn": "Chave de segurança" + "webauthn": "Chave de segurança", + "oidc": "Provedor de identidade" }, "currentPassword": "Senha atual", "authenticatorCode": "Código do autenticador", "authenticatorPlaceholder": "Código do autenticador de 6 dígitos", "verifyAndAdd": "Verificar e adicionar", "factorDescription": "Verifique um método de entrada existente antes de adicionar um segundo fator.", - "verifyAndContinue": "Verificar e continuar" + "verifyAndContinue": "Verificar e continuar", + "oidcHint": "Você entra mais uma vez no seu provedor de identidade e volta direto para esta página. Depois, repita o que ia fazer.", + "continueToProvider": "Continuar para o login" }, "passkeyName": "Nome", "noPasskeys": "Nenhuma chave de acesso registrada.", @@ -7517,7 +7520,9 @@ "description": "Isto compartilha ou exporta os seus dados de saúde. Confirme que é você para continuar.", "descriptionChange": "Isto altera algo que só você deveria poder alterar. Confirme que é você para continuar.", "signInAgain": "Entre novamente e tente mais uma vez. Sua conta não tem senha nem chave de acesso para confirmar aqui.", - "failed": "Não deu certo. Confira o que você digitou e tente de novo." + "failed": "Não deu certo. Confira o que você digitou e tente de novo.", + "oidcDone": "Confirmado. Nos próximos cinco minutos você pode continuar sem que isso seja pedido de novo.", + "oidcFailed": "Seu provedor de identidade não conseguiu confirmar que é você. Tente novamente." }, "passwordTitle": "Senha", "tokenNameLabel": "Nome do token", diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index f865925fc..e43f8a5c3 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 346, - "total": 112297, + "total": 112215, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 286, @@ -9,7 +9,7 @@ "app/api/admin/settings/route.ts :: apiHandler(arg) #2": 211, "app/api/auth/login/route.ts :: apiHandler(arg)": 262, "app/api/auth/me/route.ts :: apiHandler(arg)": 386, - "app/api/auth/oidc/callback/route.ts :: apiHandler(arg)": 281, + "app/api/auth/oidc/callback/route.ts :: apiHandler(arg)": 258, "app/api/auth/register/route.ts :: apiHandler(arg)": 268, "app/api/coach/about-me/route.ts :: apiHandler(arg) #2": 309, "app/api/cycle/calendar/route.ts :: apiHandler(arg)": 230, @@ -217,7 +217,7 @@ "components/settings/mood-reminder-card.tsx :: MoodReminderCard": 234, "components/settings/report-selection/selection-panel.tsx :: HealthRecordExportPanel": 370, "components/settings/score-section.tsx :: ScoreSection": 383, - "components/settings/security-section/passkey-list-section.tsx :: PasskeyListSection": 410, + "components/settings/security-section/passkey-list-section.tsx :: PasskeyListSection": 390, "components/settings/security-section/security-keys-card.tsx :: SecurityKeysCard": 288, "components/settings/security-section/totp-card.tsx :: TotpCard": 533, "components/settings/security-sessions-card.tsx :: SecuritySessionsCard": 245, diff --git a/src/__tests__/baselines/test-as-never.json b/src/__tests__/baselines/test-as-never.json index 330b6293f..98d2db1c7 100644 --- a/src/__tests__/baselines/test-as-never.json +++ b/src/__tests__/baselines/test-as-never.json @@ -1,7 +1,7 @@ { "$comment": "as never casts in tests, per test file under src/. Maintained by src/__tests__/test-realism-ratchet.test.ts.", - "count": 524, - "total": 5488, + "count": 523, + "total": 5483, "entries": { "__tests__/api/auth/login/mfa-branch.test.ts": 10, "__tests__/api/auth/mfa/verify.test.ts": 9, @@ -337,7 +337,6 @@ "lib/auth/__tests__/mfa-enrollment.test.ts": 8, "lib/auth/__tests__/native-handoff-flow.test.ts": 10, "lib/auth/__tests__/oidc-native-handoff.test.ts": 13, - "lib/auth/__tests__/oidc.test.ts": 5, "lib/auth/__tests__/passkey.test.ts": 7, "lib/auth/__tests__/profile-update-email-limit.test.ts": 5, "lib/auth/__tests__/profile-update.test.ts": 4, diff --git a/src/__tests__/openapi-route-coverage-guard.test.ts b/src/__tests__/openapi-route-coverage-guard.test.ts index d07b2f9c8..de9421d6e 100644 --- a/src/__tests__/openapi-route-coverage-guard.test.ts +++ b/src/__tests__/openapi-route-coverage-guard.test.ts @@ -308,6 +308,10 @@ const UNPUBLISHED: Readonly> = { kind: "adminConsole", methods: ["POST"], }, + // #1296 — the identity-provider re-proof start. A top-level navigation from + // the web's re-proof dialog that ends at the provider's sign-in page; the + // native client never re-proves this way (Bearer has its own step-up). + "/api/auth/oidc/reproof": { kind: "browserHandoff", methods: ["GET"] }, "/api/fitbit/connect": { kind: "browserHandoff", methods: ["GET"] }, "/api/google-health/connect": { kind: "browserHandoff", methods: ["GET"] }, // `/api/nightscout/connect` sat here as a `browserHandoff` and is published diff --git a/src/__tests__/session-surface-guard.test.ts b/src/__tests__/session-surface-guard.test.ts index 7467f1473..62a5b6fd2 100644 --- a/src/__tests__/session-surface-guard.test.ts +++ b/src/__tests__/session-surface-guard.test.ts @@ -102,8 +102,11 @@ describe("S1 — the direct session-resolution set is frozen", () => { // replayed callback redirects instead of minting a second identity, and // (v1.43, #959) so a single-sign-on claim is refused over a live session, // as the claim routes refuse it. + // And the identity-provider re-proof (#1296), which stamps only + // the session the start bound into the sealed state, and only while the + // browser still carries it. "app/api/auth/oidc/callback/route.ts": - "duplicate-callback probe and the claim's live-session refusal on an otherwise anonymous route", + "duplicate-callback probe, the claim's live-session refusal and the re-proof's session binding on an otherwise anonymous route", // Public liveness probe. The session read only decides whether the body // carries operational detail; the status code is computed without it. "app/api/health/route.ts": "admin-only detail on a public probe", diff --git a/src/app/api/auth/me/mfa/route.ts b/src/app/api/auth/me/mfa/route.ts index d283a10d8..703fbf091 100644 --- a/src/app/api/auth/me/mfa/route.ts +++ b/src/app/api/auth/me/mfa/route.ts @@ -22,19 +22,21 @@ import { apiSuccess } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; import { prisma } from "@/lib/db"; import { countRemainingRecoveryCodes } from "@/lib/auth/mfa/recovery-codes"; +import { recentProofMethods } from "@/lib/auth/second-factor"; export const dynamic = "force-dynamic"; export const GET = apiHandler(async () => { const { user } = await requireAuth(); - const [recoveryCodesRemaining, webauthn] = await Promise.all([ + const [recoveryCodesRemaining, webauthn, proofMethods] = await Promise.all([ countRemainingRecoveryCodes(user.id), prisma.webauthnMfaCredential.findMany({ where: { userId: user.id }, select: { id: true, name: true, createdAt: true, lastUsedAt: true }, orderBy: { createdAt: "desc" }, }), + recentProofMethods(user), ]); annotate({ action: { name: "auth.mfa.status" } }); @@ -46,5 +48,9 @@ export const GET = apiHandler(async () => { // The security hub also drives the passkey-upgrade nudge from this one // status read, so it knows whether the user has already dismissed it. passkeyNudgeDismissed: Boolean(user.passkeyUpgradeNudgeDismissed), + // The ways this account can confirm it is you, the list the recent-proof + // refusal names. The passkey card offers exactly these before adding one, + // so an account without a password is not handed a password field. + proofMethods, }); }); diff --git a/src/app/api/auth/oidc/callback/route.ts b/src/app/api/auth/oidc/callback/route.ts index bfe808080..ae12a38b8 100644 --- a/src/app/api/auth/oidc/callback/route.ts +++ b/src/app/api/auth/oidc/callback/route.ts @@ -47,6 +47,11 @@ import { } from "@/lib/managed-profiles/claim-sso"; import { pendingDeletionOf } from "@/lib/account-deletion/pending"; import { isHandoverUsable } from "@/lib/managed-profiles/handover"; +import { + isReproofSession, + recordOidcReproof, + reproofReturnUrl, +} from "@/lib/auth/oidc-reproof"; const LOGIN_ERROR_URL = "/auth/login"; @@ -103,6 +108,20 @@ function deletionPendingRedirect(purgeAt: Date | null): NextResponse { return response; } +/** + * #1296 — a re-proof returns to the page it left, never to the sign-in page: + * the person is still signed in, and the session they came from is the one + * thing that leg may touch. `reason` null is the success. + */ +function reproofRedirect(next: string, reason: string | null): NextResponse { + if (reason) annotate({ meta: { reason } }); + const response = NextResponse.redirect( + reproofReturnUrl(next, reason ? "failed" : "done"), + ); + deleteStateCookie(response); + return response; +} + /** * The state cookie is set with `path: "/api/auth/oidc"`; the delete must * repeat that path (RFC 6265 keys cookies by name+domain+path — a bare @@ -139,6 +158,10 @@ export const GET = apiHandler(async (req: NextRequest) => { const stored = readStoredOidcState(req.cookies.get(OIDC_STATE_COOKIE)?.value); const isNative = stored?.native === true; const isClaim = stored?.intent === "claim"; + // #1296 — a re-proof returns to the page it left (`reproofRedirect`). + const isReproof = stored?.intent === "reproof"; + const reproofBack = (reason: string | null) => + reproofRedirect(stored?.next ?? "/", reason); // Native-aware error redirect: the custom scheme for a decrypted native flow, // the web login page otherwise. Deletes the single-use state cookie either @@ -158,6 +181,7 @@ export const GET = apiHandler(async (req: NextRequest) => { }; const failRedirect = (reason: string): NextResponse => { + if (isReproof) return reproofBack(reason); if (isClaim) return claimRedirect(reason); if (isNative) { const response = NextResponse.redirect( @@ -199,42 +223,11 @@ export const GET = apiHandler(async (req: NextRequest) => { // resolve to the web `oidc_state` fallback: the flow is fundamentally broken // and there is nothing safe to hand the app (spec §1 web-fallback). if (!code || !state) { + if (isReproof) return reproofBack("oidc_state"); return isClaim ? claimRedirect("oidc_state") : errorRedirect("oidc_state"); } - if (!stored) { - // Defence-in-depth idempotency for a duplicated callback. The single-use - // state cookie is set with `path: "/api/auth/oidc"` and DELETED by the - // first callback the moment it succeeds. A second callback for the same - // sign-in (a client/proxy retry, a double navigation, or the - // service-worker double-request this release also fixes) therefore arrives - // with the state cookie already gone — `stored` is null — and, because the - // first callback set the session, it carries a live session cookie. - // Redeeming the now-consumed authorization code again fails at the IdP and - // surfaces `?error=oidc_failed` on a sign-in that actually succeeded. So - // when a VALIDATED session is present, treat this as a duplicate of a - // completed login and send the user into the app instead of erroring. - // - // Fail-closed is preserved for the real attack case: the code is NEVER - // exchanged on this branch, and the session is resolved against the - // database + its expiry (not merely "a cookie is present"). A forged, - // replayed, or expired state from a caller with no valid session gets the - // unchanged `oidc_state` error; a caller who already holds their own valid - // session gains nothing here but a redirect to a page they could already - // reach — no code redemption, no session mint, no identity change. - const existingSession = await validateSessionFromCookieValue( - readSessionCookie(req.cookies)?.value, - ); - if (existingSession) { - annotate({ meta: { reason: "duplicate_callback_authenticated" } }); - const response = NextResponse.redirect(oidcAppUrl("/")); - // The state cookie is already consumed; delete defensively so a stray - // blob can never outlive this response. - deleteStateCookie(response); - return response; - } - return errorRedirect("oidc_state"); - } + if (!stored) return stateMissing(req); // CSRF check — timing-safe, byte-for-byte, mirroring the Withings callback. if ( @@ -242,9 +235,12 @@ export const GET = apiHandler(async (req: NextRequest) => { !timingSafeEqual(Buffer.from(state), Buffer.from(stored.state)) ) { annotate({ meta: { reason: "csrf" } }); + if (isReproof) return reproofBack("oidc_state"); return isClaim ? claimRedirect("oidc_state") : errorRedirect("oidc_state"); } + if (isReproof) return finishReproof(req, stored, config, ip, reproofBack); + // v1.43 (#959) — a claim is never finished over a live session (the claim // routes' rule: a Guardian finishing their own link would otherwise be // signed out of their account and into the person's), and is admitted only @@ -408,3 +404,90 @@ export const GET = apiHandler(async (req: NextRequest) => { return failRedirect("oidc_failed"); } }); + +/** + * #1296 — the re-proof leg. The browser must still carry the session the + * start bound into the state, checked before the code is redeemed; then the + * provider's answer must name this account's linked identity and an + * authentication inside the gate's window. Only then is `reproofAt` stamped. + * No session is created, and every refusal lands back on the page it left. + */ +async function finishReproof( + req: NextRequest, + stored: NonNullable>, + config: NonNullable>, + ip: string, + back: (reason: string | null) => NextResponse, +): Promise { + const live = await validateSessionFromCookieValue( + readSessionCookie(req.cookies)?.value, + ); + if (!isReproofSession(stored, live)) return back("session_mismatch"); + + try { + const metadata = await discoverOidcMetadata(config); + const tokens = await exchangeCodeForTokens({ + metadata, + config, + code: req.nextUrl.searchParams.get("code") as string, + codeVerifier: stored.codeVerifier, + redirectUri: getOidcRedirectUri(), + }); + if (!tokens.id_token) return back("oidc_failed"); + const identity = await verifyIdToken({ + metadata, + config, + idToken: tokens.id_token, + nonce: stored.nonce, + }); + const outcome = await recordOidcReproof({ + userId: stored.reproofUserId as string, + sessionId: stored.reproofSessionId as string, + issuer: metadata.issuer, + identity, + ipAddress: ip, + }); + return back(outcome.ok ? null : outcome.reason); + } catch (err) { + getEvent()?.setError(err); + return back("exchange_or_verify_failed"); + } +} + +/** + * A callback that arrives without its state blob: the duplicate of a sign-in + * that already completed, or a forged, replayed or expired one. + */ +async function stateMissing(req: NextRequest): Promise { + // Defence-in-depth idempotency for a duplicated callback. The single-use + // state cookie is set with `path: "/api/auth/oidc"` and DELETED by the + // first callback the moment it succeeds. A second callback for the same + // sign-in (a client/proxy retry, a double navigation, or the + // service-worker double-request this release also fixes) therefore arrives + // with the state cookie already gone — `stored` is null — and, because the + // first callback set the session, it carries a live session cookie. + // Redeeming the now-consumed authorization code again fails at the IdP and + // surfaces `?error=oidc_failed` on a sign-in that actually succeeded. So + // when a VALIDATED session is present, treat this as a duplicate of a + // completed login and send the user into the app instead of erroring. + // + // Fail-closed is preserved for the real attack case: the code is NEVER + // exchanged on this branch, and the session is resolved against the + // database + its expiry (not merely "a cookie is present"). A forged, + // replayed, or expired state from a caller with no valid session gets the + // unchanged `oidc_state` error; a caller who already holds their own valid + // session gains nothing here but a redirect to a page they could already + // reach — no code redemption, no session mint, no identity change. + const existingSession = await validateSessionFromCookieValue( + readSessionCookie(req.cookies)?.value, + ); + if (existingSession) { + annotate({ meta: { reason: "duplicate_callback_authenticated" } }); + const response = NextResponse.redirect(oidcAppUrl("/")); + // The state cookie is already consumed; delete defensively so a stray + // blob can never outlive this response. + deleteStateCookie(response); + return response; + } + return errorRedirect("oidc_state"); +} diff --git a/src/app/api/auth/oidc/reproof/route.ts b/src/app/api/auth/oidc/reproof/route.ts new file mode 100644 index 000000000..4b9f1d881 --- /dev/null +++ b/src/app/api/auth/oidc/reproof/route.ts @@ -0,0 +1,118 @@ +import { randomBytes } from "node:crypto"; +import { NextRequest, NextResponse } from "next/server"; +import { apiHandler, requireCookieAuth } from "@/lib/api-handler"; +import { getClientIp } from "@/lib/api-response"; +import { annotate, getEvent } from "@/lib/logging/context"; +import { encrypt } from "@/lib/crypto"; +import { shouldEmitSecureCookie } from "@/lib/auth/secure-cookie"; +import { s256Challenge } from "@/lib/mcp/oauth/pkce"; +import { recentProofMethods } from "@/lib/auth/second-factor"; +import { throttleReproof } from "@/lib/auth/existing-factor-proof"; +import { + buildAuthorizationUrl, + discoverOidcMetadata, + getOidcConfig, + getOidcRedirectUri, + sanitizeOidcNextPath, +} from "@/lib/auth/oidc"; +import { + OIDC_STATE_COOKIE, + OIDC_STATE_COOKIE_PATH, + OIDC_STATE_TTL_MS, +} from "@/lib/auth/oidc-cookie"; +import { OIDC_REPROOF_STAGE, reproofReturnUrl } from "@/lib/auth/oidc-reproof"; + +export const dynamic = "force-dynamic"; + +/** + * Start confirming it is you at the identity provider (#1296). + * + * A top-level navigation from the re-proof dialog, cookie session only: a + * Bearer token never re-proves this way, because a stamp on a session row + * means nothing to a token (`requireCookieAuth` reads the session cookie and + * nothing else). Offered only where `recentProofMethods` lists `oidc`, which + * is an account without a second factor whose identity is linked while a + * provider is configured; anything else goes straight back with + * `reproof=failed`. + * + * The round trip asks for a fresh authentication (`prompt=login`, + * `max_age=0`) with its own state, nonce and PKCE verifier, sealed into the + * same single-use cookie the sign-in uses, together with the account and the + * session it confirms. The callback stamps that session's `reproofAt` and + * returns to `next` (a same-origin path) with `reproof=done`. It never mints + * a session. + * + * Draws on the account's shared re-proof budget, like every other proof. + */ +export const GET = apiHandler(async (req: NextRequest) => { + annotate({ action: { name: "auth.oidc.reproof.start" } }); + const { user, session } = await requireCookieAuth(); + + const next = sanitizeOidcNextPath( + req.nextUrl.searchParams.get("next"), + req.url, + ); + const back = (reason: string): NextResponse => { + annotate({ meta: { reason } }); + return NextResponse.redirect(reproofReturnUrl(next, "failed")); + }; + + const config = getOidcConfig(); + if (!config) return back("oidc_disabled"); + + const methods = await recentProofMethods(user); + if (!methods.includes("oidc")) return back("not_offered"); + + const limited = await throttleReproof( + user.id, + getClientIp(req), + OIDC_REPROOF_STAGE, + ); + if (limited) return back("rate_limited"); + + let metadata; + try { + metadata = await discoverOidcMetadata(config); + } catch (err) { + getEvent()?.setError(err); + return back("oidc_failed"); + } + + const state = randomBytes(32).toString("base64url"); + const nonce = randomBytes(32).toString("base64url"); + const codeVerifier = randomBytes(64).toString("base64url"); + + const response = NextResponse.redirect( + buildAuthorizationUrl({ + metadata, + config, + state, + nonce, + codeChallenge: s256Challenge(codeVerifier), + redirectUri: getOidcRedirectUri(), + reauthenticate: true, + }), + ); + response.cookies.set( + OIDC_STATE_COOKIE, + encrypt( + JSON.stringify({ + state, + nonce, + codeVerifier, + next, + intent: "reproof" as const, + reproofUserId: user.id, + reproofSessionId: session.id, + }), + ), + { + httpOnly: true, + secure: shouldEmitSecureCookie(), + sameSite: "lax", + maxAge: Math.floor(OIDC_STATE_TTL_MS / 1000), + path: OIDC_STATE_COOKIE_PATH, + }, + ); + return response; +}); diff --git a/src/app/api/auth/passkey/register-options/route.ts b/src/app/api/auth/passkey/register-options/route.ts index de277bc69..968d9948a 100644 --- a/src/app/api/auth/passkey/register-options/route.ts +++ b/src/app/api/auth/passkey/register-options/route.ts @@ -5,6 +5,8 @@ import { } from "@/lib/auth/passkey"; import { createMfaAuthenticationOptions } from "@/lib/auth/mfa/webauthn"; import { + RECENT_PROOF_MAX_AGE_MS, + REPROOF_REQUIRED_CODE, isSecondFactorProof, recordReproofFailure, refundReproof, @@ -59,6 +61,30 @@ export const POST = apiHandler(async (request: NextRequest) => { return apiSuccess({ ...result, reauth: true }); } + // #1296 — an account whose only credential is its identity provider proves + // itself there (`GET /api/auth/oidc/reproof`), which stamps + // `Session.reproofAt`. `{ method: "oidc" }` asks this route to take that + // stamp as the proof: only where `oidc` is on the account's list (no second + // factor, a linked identity, a configured provider) and only inside the + // five-minute window. Like a password, it clears the MFA stamp rather than + // setting it. + if (Object.keys(body).length === 1 && body.method === "oidc") { + if ( + !(await recentProofMethods(user)).includes("oidc") || + !(await hasRecentReproof(session.id)) + ) { + annotate({ + action: { name: "auth.reproof.required" }, + meta: { stage: "passkey_enroll", method: "oidc" }, + }); + return apiError("Confirm it is you before adding a passkey", 401, { + errorCode: REPROOF_REQUIRED_CODE, + methods: await recentProofMethods(user), + }); + } + return issueRegistrationOptions(user, session.id, null); + } + const parsed = stepUpMintSchema.safeParse(body); if (!parsed.success) return proofRequired(); @@ -101,23 +127,48 @@ export const POST = apiHandler(async (request: NextRequest) => { } await refundReproof(user.id); - // A strong existing factor refreshes the session's MFA stamp. Password proof - // deliberately clears it: registration is authorized by this single-use, - // session-bound challenge but cannot upgrade password-only authentication. + return issueRegistrationOptions( + user, + session.id, + parsed.data.method === "password" ? null : new Date(), + ); +}); + +/** A password or provider re-proof on this session inside the window. */ +async function hasRecentReproof(sessionId: string): Promise { + const row = await prisma.session.findUnique({ + where: { id: sessionId }, + select: { reproofAt: true }, + }); + return ( + row?.reproofAt != null && + Date.now() - row.reproofAt.getTime() <= RECENT_PROOF_MAX_AGE_MS + ); +} + +/** + * A strong existing factor refreshes the session's MFA stamp. A password or + * provider proof deliberately clears it (`mfaVerifiedAt: null`): registration + * is authorized by this single-use, session-bound challenge but cannot upgrade + * first-factor authentication. + */ +async function issueRegistrationOptions( + user: { id: string; email: string | null; username: string }, + sessionId: string, + mfaVerifiedAt: Date | null, +): Promise { await prisma.session.update({ - where: { id: session.id }, - data: { - mfaVerifiedAt: parsed.data.method === "password" ? null : new Date(), - }, + where: { id: sessionId }, + data: { mfaVerifiedAt }, }); const { options, challengeId } = await createRegistrationOptions( user.id, user.email ?? user.username, - session.id, + sessionId, ); annotate({ action: { name: "auth.passkey.register-options" } }); return apiSuccess({ options, challengeId }); -}); +} diff --git a/src/components/auth/oidc-reproof-return.tsx b/src/components/auth/oidc-reproof-return.tsx new file mode 100644 index 000000000..8f9aa15a7 --- /dev/null +++ b/src/components/auth/oidc-reproof-return.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { useEffect } from "react"; + +import { toastWrittenOutcome } from "@/components/outcome/outcome-toast"; +import { useTranslations } from "@/lib/i18n/context"; +import { dropParams, writeUrlState } from "@/lib/url-state"; + +/** + * The outcome a confirmation at the identity provider came back with + * (`?reproof=done|failed`, set by the OIDC callback). Pure, so the parsing is + * testable without a browser. + */ +export function readReproofReturn(search: string): "done" | "failed" | null { + const raw = new URLSearchParams(search).get("reproof"); + return raw === "done" || raw === "failed" ? raw : null; +} + +/** + * Says once how the confirmation at the identity provider went, then takes the + * marker off the address so a reload does not say it again. The browser lands + * on the page it left, which is where the person repeats what they were + * about to do. + */ +export function OidcReproofReturnBridge() { + const { t } = useTranslations(); + useEffect(() => { + const outcome = readReproofReturn(window.location.search); + if (!outcome) return; + writeUrlState(dropParams("reproof"), { mode: "replace", state: "keep" }); + toastWrittenOutcome( + outcome === "done" ? "success" : "failed", + outcome === "done" + ? t("settings.reproof.oidcDone") + : t("settings.reproof.oidcFailed"), + ); + // A later run (a locale switch) finds no marker and does nothing. + }, [t]); + return null; +} diff --git a/src/components/providers.tsx b/src/components/providers.tsx index 1c141b66e..2549b61b0 100644 --- a/src/components/providers.tsx +++ b/src/components/providers.tsx @@ -21,6 +21,7 @@ import { VersionPoller } from "@/components/version-poller"; import { ServiceWorkerRegistrar } from "@/components/service-worker-registrar"; import { SharedRecordGrantLossBridge } from "@/components/layout/shared-record-grant-loss-bridge"; import { RecordSessionFenceBridge } from "@/components/layout/record-session-fence-bridge"; +import { OidcReproofReturnBridge } from "@/components/auth/oidc-reproof-return"; import { useAuth, type AccountAccessStatus } from "@/hooks/use-auth"; import { useRecordSessionTransition } from "@/hooks/use-record-session-transition"; import { isDashboardSnapshotEnabled } from "@/lib/dashboard/snapshot-flag"; @@ -432,6 +433,7 @@ export function Providers({ in its own account, rather than asking every call site to. */} + {children} diff --git a/src/components/settings/security-section/__tests__/oidc-reproof-choice.test.ts b/src/components/settings/security-section/__tests__/oidc-reproof-choice.test.ts new file mode 100644 index 000000000..9131bb3ba --- /dev/null +++ b/src/components/settings/security-section/__tests__/oidc-reproof-choice.test.ts @@ -0,0 +1,78 @@ +/** + * The web side of confirming at the identity provider (#1296): the dialogs + * offer it when the server names it, the round trip starts from and returns + * to the page the person is on, and the outcome marker is read once. + */ +import { describe, expect, it, vi } from "vitest"; + +import { ApiError } from "@/lib/api/api-fetch"; +import { oidcReproofStartUrl } from "../existing-factor-reauth-dialog"; +import { passkeyProofChoices } from "../passkey-list-section"; +import { createRecentProofFlow } from "../recent-proof-flow"; +import { readReproofReturn } from "@/components/auth/oidc-reproof-return"; + +describe("passkeyProofChoices", () => { + it("offers only what the account can give once its list is known", () => { + expect(passkeyProofChoices(["oidc"])).toEqual(["oidc"]); + expect(passkeyProofChoices(["password", "oidc", "passkey"])).toEqual([ + "password", + "oidc", + "passkey", + ]); + expect(passkeyProofChoices(["totp", "something-new"])).toEqual(["totp"]); + }); + + it("falls back to the four body proofs while the list is loading", () => { + expect(passkeyProofChoices(undefined)).toEqual([ + "password", + "totp", + "passkey", + "webauthn", + ]); + }); +}); + +describe("oidcReproofStartUrl", () => { + it("names the current page as the way back, without an old outcome", () => { + const href = oidcReproofStartUrl({ + pathname: "/settings/api", + search: "?tab=mcp&reproof=failed", + }); + const url = new URL(href, "https://healthlog.example.test"); + expect(url.pathname).toBe("/api/auth/oidc/reproof"); + expect(url.searchParams.get("next")).toBe("/settings/api?tab=mcp"); + }); +}); + +describe("readReproofReturn", () => { + it("reads a known outcome off the query string", () => { + expect(readReproofReturn("?tab=mcp&reproof=done")).toBe("done"); + expect(readReproofReturn("?reproof=failed")).toBe("failed"); + }); + + it("ignores an address without a known outcome", () => { + expect(readReproofReturn("")).toBeNull(); + expect(readReproofReturn("?reproof=maybe")).toBeNull(); + }); +}); + +describe("the re-proof dialog flow", () => { + it("opens with the identity provider when that is the account's only way", async () => { + const setMethods = vi.fn(); + const flow = createRecentProofFlow({ + view: { setMethods, setPending: vi.fn(), setError: vi.fn() }, + submit: vi.fn(), + messages: () => ({ signInAgain: "sign in again", failed: "failed" }), + }); + void flow + .run(async () => { + throw new ApiError("Confirm it is you", 401, { + errorCode: "auth.reproof.required", + methods: ["oidc"], + }); + }) + .catch(() => {}); + await vi.waitFor(() => expect(setMethods).toHaveBeenCalledWith(["oidc"])); + flow.cancel(); + }); +}); diff --git a/src/components/settings/security-section/existing-factor-reauth-dialog.tsx b/src/components/settings/security-section/existing-factor-reauth-dialog.tsx index 357e8d4a1..799c30f33 100644 --- a/src/components/settings/security-section/existing-factor-reauth-dialog.tsx +++ b/src/components/settings/security-section/existing-factor-reauth-dialog.tsx @@ -19,7 +19,30 @@ import { useTranslations } from "@/lib/i18n/context"; import { ApiError, apiPost } from "@/lib/api/api-fetch"; import { describePasskeyError } from "@/lib/passkey-errors"; -export type ReauthMethod = "password" | "totp" | "passkey" | "webauthn"; +export type ReauthMethod = + "password" | "oidc" | "totp" | "passkey" | "webauthn"; + +/** + * Where the browser goes to confirm at the identity provider: the re-proof + * start, told to come back to the page it is on now. A `reproof` outcome left + * on the current address from an earlier round trip is dropped, so the page + * does not come back announcing the old one. + */ +export function oidcReproofStartUrl(current: { + pathname: string; + search: string; +}): string { + const params = new URLSearchParams(current.search); + params.delete("reproof"); + const query = params.toString(); + const next = `${current.pathname}${query ? `?${query}` : ""}`; + return `/api/auth/oidc/reproof?${new URLSearchParams({ next }).toString()}`; +} + +/** Leave for the identity provider. The page unloads; nothing resolves. */ +export function startOidcReproof(): void { + window.location.assign(oidcReproofStartUrl(window.location)); +} /** The body the enrollment routes accept as a fresh existing-factor proof. */ export type ExistingFactorProof = @@ -46,23 +69,19 @@ export function isReproofRequired(err: unknown): boolean { const REAUTH_METHODS: readonly ReauthMethod[] = [ "password", + "oidc", "totp", "passkey", "webauthn", ]; /** - * The proofs the refusal says this account can give, or null when it names - * none. On an account with a second factor the server leaves the password out, - * so the dialog does not offer a choice that can only be refused. - */ -/** - * The refusal says the account holds nothing it could re-prove here: an - * account that signs in through an identity provider only, with no password, - * passkey or second factor. A dialog would only offer it a password it does - * not have, so the callers say "sign in again" instead — a fresh sign-in is - * the proof such an account can give (`Session.createdAt` within five - * minutes). + * The refusal says the account holds nothing it could re-prove here: no + * password, passkey or second factor, and no identity provider to confirm at + * (an account whose provider the operator has since switched off). A dialog + * would only offer it a password it does not have, so the callers say "sign + * in again" instead — a fresh sign-in is the proof such an account can give + * (`Session.createdAt` within five minutes). */ export function nothingToReprove(err: unknown): boolean { return ( @@ -72,6 +91,11 @@ export function nothingToReprove(err: unknown): boolean { ); } +/** + * The proofs the refusal says this account can give, or null when it names + * none. On an account with a second factor the server leaves the password out, + * so the dialog does not offer a choice that can only be refused. + */ export function offeredReauthMethods(err: unknown): ReauthMethod[] | null { if (!(err instanceof ApiError)) return null; const raw = err.meta?.methods; @@ -87,7 +111,9 @@ export function offeredReauthMethods(err: unknown): ReauthMethod[] | null { * new factor is added. A passkey or security-key assertion begins at the * cookie-only `/api/auth/passkey/register-options` re-proof arm, which issues * an assertion challenge and nothing else; the resulting proof is handed to - * `onProof`, which retries the enrollment call with it. + * `onProof`, which retries the enrollment call with it. The identity provider + * is not a proof in the body: choosing it leaves the page for the provider and + * comes back to it, and the action is then repeated by hand. */ export function ExistingFactorReauthDialog({ open, @@ -117,6 +143,11 @@ export function ExistingFactorReauthDialog({ async function submit() { setAssertionError(null); + if (method === "oidc") { + setAsserting(true); + startOidcReproof(); + return; + } if (method === "password") { onProof({ method, password: secret }); return; @@ -152,6 +183,7 @@ export function ExistingFactorReauthDialog({ const labels: Record = { password: t("settings.passkeyReauth.methods.password"), + oidc: t("settings.passkeyReauth.methods.oidc"), totp: t("settings.passkeyReauth.methods.totp"), passkey: t("settings.passkeyReauth.methods.passkey"), webauthn: t("settings.passkeyReauth.methods.webauthn"), @@ -223,6 +255,10 @@ export function ExistingFactorReauthDialog({ /> )} + {method === "oidc" && ( +

{t("settings.passkeyReauth.oidcHint")}

+ )} + {shown && (
)} - {t("settings.passkeyReauth.verifyAndContinue")} + {method === "oidc" + ? t("settings.passkeyReauth.continueToProvider") + : t("settings.passkeyReauth.verifyAndContinue")} diff --git a/src/components/settings/security-section/index.tsx b/src/components/settings/security-section/index.tsx index 6008b7d75..646ecb538 100644 --- a/src/components/settings/security-section/index.tsx +++ b/src/components/settings/security-section/index.tsx @@ -20,6 +20,7 @@ interface MfaStatus { recoveryCodesRemaining: number; webauthn: WebauthnKeyInfo[]; passkeyNudgeDismissed: boolean; + proofMethods: string[]; } interface PasskeyInfo { @@ -71,7 +72,10 @@ export function SecuritySection() { totpEnabled={status.totp.enabled} /> - + )} diff --git a/src/components/settings/security-section/passkey-list-section.tsx b/src/components/settings/security-section/passkey-list-section.tsx index 34d9ed2cb..24b860bc5 100644 --- a/src/components/settings/security-section/passkey-list-section.tsx +++ b/src/components/settings/security-section/passkey-list-section.tsx @@ -45,6 +45,11 @@ import { } from "@/lib/api/api-fetch"; import { describePasskeyError } from "@/lib/passkey-errors"; import { describeStepUp } from "./security-keys-card"; +import { + isReproofRequired, + startOidcReproof, + type ReauthMethod, +} from "./existing-factor-reauth-dialog"; import { ReproofCancelledError, useRecentProof } from "./use-recent-proof"; interface PasskeyInfo { @@ -56,17 +61,102 @@ interface PasskeyInfo { lastUsedAt: string | null; } -type ReauthMethod = "password" | "passkey" | "totp" | "webauthn"; - type ExistingFactorProof = | { method: "password"; password: string } | { method: "totp"; code: string } + | { method: "oidc" } | { method: "passkey" | "webauthn"; challengeId: string; credential: unknown; }; +/** What the card offers while the account's own list has not arrived. */ +const FALLBACK_METHODS: readonly ReauthMethod[] = [ + "password", + "totp", + "passkey", + "webauthn", +]; + +/** + * The proofs to offer before adding a passkey: the account's own list when the + * status read has it, so an account without a password is not handed a + * password field, and the four body proofs otherwise. + */ +export function passkeyProofChoices( + proofMethods: readonly string[] | undefined, +): ReauthMethod[] { + if (!proofMethods) return [...FALLBACK_METHODS]; + const known: readonly ReauthMethod[] = [ + "password", + "oidc", + "totp", + "passkey", + "webauthn", + ]; + return proofMethods.filter((m): m is ReauthMethod => + known.includes(m as ReauthMethod), + ); +} + +/** + * Prove an existing factor, then run the registration ceremony. The proof + * finishes before the first registration challenge is requested, so a + * cancelled or failed proof leaves no enrollable challenge behind. + */ +async function addPasskey(method: ReauthMethod, secret: string): Promise { + const webauthn = await import("@simplewebauthn/browser"); + let freshFactor: ExistingFactorProof; + + if (method === "oidc") { + // No secret: the proof is a provider re-proof already on the session. + freshFactor = { method }; + } else if (method === "password") { + freshFactor = { method, password: secret }; + } else if (method === "totp") { + freshFactor = { method, code: secret }; + } else { + const assertionChallenge = await apiPost<{ + options: Parameters< + typeof webauthn.startAuthentication + >[0]["optionsJSON"]; + challengeId: string; + }>("/api/auth/passkey/register-options", { method }); + const credential = await webauthn.startAuthentication({ + optionsJSON: assertionChallenge.options, + }); + freshFactor = { + method, + challengeId: assertionChallenge.challengeId, + credential, + }; + } + + let begun: { + options: Parameters[0]["optionsJSON"]; + challengeId: string; + }; + try { + begun = await apiPost("/api/auth/passkey/register-options", freshFactor); + } catch (err) { + // No provider re-proof on the session yet: go and get one. The page + // unloads and comes back here, so the mutation stays pending rather + // than flashing an error on the way out. + if (method !== "oidc" || !isReproofRequired(err)) throw err; + startOidcReproof(); + return new Promise(() => {}); + } + const { options, challengeId } = begun; + const credential = await webauthn.startRegistration({ + optionsJSON: options, + }); + await apiPost("/api/auth/passkey/register-verify", { + challengeId, + credential, + }); +} + /** * Primary-passkey management: list (name, device, backup, created, last used), * add (registration ceremony), rename, and delete. The passwordless-primary @@ -74,8 +164,11 @@ type ExistingFactorProof = */ export function PasskeyListSection({ isAuthenticated, + proofMethods, }: { isAuthenticated: boolean; + /** `proofMethods` from `GET /api/auth/me/mfa`. */ + proofMethods?: readonly string[]; }) { const { t } = useTranslations(); const queryClient = useQueryClient(); @@ -83,7 +176,10 @@ export function PasskeyListSection({ const [editingId, setEditingId] = useState(null); const [editName, setEditName] = useState(""); const [reauthOpen, setReauthOpen] = useState(false); - const [reauthMethod, setReauthMethod] = useState("password"); + const choices = passkeyProofChoices(proofMethods); + const [reauthMethod, setReauthMethod] = useState( + choices[0] ?? "password", + ); const [reauthSecret, setReauthSecret] = useState(""); const { data: passkeys } = useQuery({ @@ -93,56 +189,18 @@ export function PasskeyListSection({ }); const add = useMutation({ - mutationFn: async ({ + mutationFn: ({ method, secret, }: { method: ReauthMethod; secret: string; - }) => { - const webauthn = await import("@simplewebauthn/browser"); - let freshFactor: ExistingFactorProof; - - if (method === "password") { - freshFactor = { method, password: secret }; - } else if (method === "totp") { - freshFactor = { method, code: secret }; - } else { - const assertionChallenge = await apiPost<{ - options: Parameters< - typeof webauthn.startAuthentication - >[0]["optionsJSON"]; - challengeId: string; - }>("/api/auth/passkey/register-options", { method }); - const credential = await webauthn.startAuthentication({ - optionsJSON: assertionChallenge.options, - }); - freshFactor = { - method, - challengeId: assertionChallenge.challengeId, - credential, - }; - } - - const { options, challengeId } = await apiPost<{ - options: Parameters< - typeof webauthn.startRegistration - >[0]["optionsJSON"]; - challengeId: string; - }>("/api/auth/passkey/register-options", freshFactor); - const credential = await webauthn.startRegistration({ - optionsJSON: options, - }); - await apiPost("/api/auth/passkey/register-verify", { - challengeId, - credential, - }); - }, + }) => addPasskey(method, secret), onSuccess: () => { setMsg(null); setReauthOpen(false); setReauthSecret(""); - setReauthMethod("password"); + setReauthMethod(choices[0] ?? "password"); queryClient.invalidateQueries({ queryKey: queryKeys.passkeys() }); toastWrittenOutcome("success", t("settings.passkeyAddedToast")); }, @@ -195,6 +253,14 @@ export function PasskeyListSection({ }, }); + const methodLabels: Record = { + password: t("settings.passkeyReauth.methods.password"), + oidc: t("settings.passkeyReauth.methods.oidc"), + totp: t("settings.passkeyReauth.methods.totp"), + passkey: t("settings.passkeyReauth.methods.passkey"), + webauthn: t("settings.passkeyReauth.methods.webauthn"), + }; + const DEVICE_TYPE_LABELS: Record = { singleDevice: t("settings.singleDevice"), multiDevice: t("settings.multiDevice"), @@ -350,7 +416,14 @@ export function PasskeyListSection({ onOpenChange={(open) => { if (add.isPending) return; setReauthOpen(open); - if (open) setMsg(null); + if (open) { + setMsg(null); + // The choice the account's list leads with, in case the list + // arrived after the card first rendered. + if (!choices.includes(reauthMethod)) { + setReauthMethod(choices[0] ?? "password"); + } + } }} > @@ -384,14 +457,7 @@ export function PasskeyListSection({ role="group" aria-label={t("settings.passkeyReauth.methodLabel")} > - {( - [ - ["password", t("settings.passkeyReauth.methods.password")], - ["totp", t("settings.passkeyReauth.methods.totp")], - ["passkey", t("settings.passkeyReauth.methods.passkey")], - ["webauthn", t("settings.passkeyReauth.methods.webauthn")], - ] satisfies Array<[ReauthMethod, string]> - ).map(([method, label]) => ( + {choices.map((method) => ( ))}
+ {reauthMethod === "oidc" && ( +

{t("settings.passkeyReauth.oidcHint")}

+ )} + {reauthMethod === "password" && ( ({ + findUser: vi.fn<(args: unknown) => Promise>(), + updateSession: vi.fn<(args: unknown) => Promise>(), + countKeys: vi.fn<(args: unknown) => Promise>(), + countPasskeys: vi.fn<(args: unknown) => Promise>(), +})); +vi.mock("@/lib/db", () => ({ + prisma: { + user: { findUnique: db.findUser }, + session: { update: db.updateSession }, + webauthnMfaCredential: { count: db.countKeys }, + passkey: { count: db.countPasskeys }, + }, +})); +vi.mock("@/lib/auth/audit", () => ({ + auditLog: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/rate-limit", () => ({ + checkRateLimit: vi.fn(), + rateLimitHeaders: vi.fn(() => ({})), + refundRateLimit: vi.fn(), +})); +vi.mock("@/lib/crypto", () => ({ + decrypt: vi.fn((s: string) => s), + encrypt: vi.fn((s: string) => s), +})); + +import { auditLog } from "@/lib/auth/audit"; +import { refundRateLimit } from "@/lib/rate-limit"; +import { + isFirstFactorOnlyProof, + recentProofMethods, +} from "@/lib/auth/second-factor"; +import { + isFreshAuthTime, + isLinkedIdentity, + isReproofSession, + recordOidcReproof, + reproofReturnUrl, +} from "@/lib/auth/oidc-reproof"; +import { readStoredOidcState } from "@/lib/auth/oidc-callback/state"; + +const ISSUER = "https://idp.example.test"; +const OIDC_ENV = { + OIDC_ISSUER_URL: ISSUER, + OIDC_CLIENT_ID: "client", + OIDC_CLIENT_SECRET: "secret", +}; + +function accountRow(overrides: Record = {}) { + return { + passwordHash: null, + oidcIssuer: ISSUER, + oidcSub: "sub-1", + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + Object.assign(process.env, OIDC_ENV); + process.env.NEXT_PUBLIC_APP_URL = "https://healthlog.example.test"; + db.countKeys.mockResolvedValue(0); + db.countPasskeys.mockResolvedValue(0); +}); + +afterEach(() => { + for (const key of Object.keys(OIDC_ENV)) delete process.env[key]; +}); + +describe("recentProofMethods offers the identity provider", () => { + const user = { id: "u1", totpConfirmedAt: null }; + + it("to an account without a second factor whose identity is linked", async () => { + db.findUser.mockResolvedValue(accountRow()); + expect(await recentProofMethods(user, false)).toEqual(["oidc"]); + }); + + it("beside the password and passkey the account also holds", async () => { + db.countPasskeys.mockResolvedValue(1); + db.findUser.mockResolvedValue(accountRow({ passwordHash: "hash" })); + expect(await recentProofMethods(user, false)).toEqual([ + "password", + "oidc", + "passkey", + ]); + }); + + it("not to an account without a linked identity", async () => { + db.findUser.mockResolvedValue( + accountRow({ oidcIssuer: null, oidcSub: null }), + ); + expect(await recentProofMethods(user, false)).toEqual([]); + }); + + it("not when the instance has no provider configured", async () => { + delete process.env.OIDC_CLIENT_SECRET; + db.findUser.mockResolvedValue(accountRow()); + expect(await recentProofMethods(user, false)).toEqual([]); + }); + + it("not to an account with a second factor, where it could not clear the gate", async () => { + db.countKeys.mockResolvedValue(1); + db.findUser.mockResolvedValue(accountRow()); + expect(await recentProofMethods(user, true)).toEqual(["webauthn"]); + }); + + it("and never counts it as a second factor", () => { + expect(isFirstFactorOnlyProof("oidc")).toBe(true); + expect(isFirstFactorOnlyProof("password")).toBe(true); + expect(isFirstFactorOnlyProof("passkey")).toBe(false); + }); +}); + +describe("isFreshAuthTime", () => { + const now = 1_800_000_000_000; + const seconds = (ms: number) => Math.floor(ms / 1000); + + it("accepts an authentication inside the five-minute window", () => { + expect(isFreshAuthTime(seconds(now - 10_000), now)).toBe(true); + expect(isFreshAuthTime(seconds(now - 300_000), now)).toBe(true); + }); + + it("refuses one older than the window", () => { + expect(isFreshAuthTime(seconds(now - 301_000), now)).toBe(false); + expect(isFreshAuthTime(seconds(now - 3_600_000), now)).toBe(false); + }); + + it("refuses a token that does not say when", () => { + expect(isFreshAuthTime(null, now)).toBe(false); + }); + + it("tolerates a provider clock a minute ahead, and no more", () => { + expect(isFreshAuthTime(seconds(now + 30_000), now)).toBe(true); + expect(isFreshAuthTime(seconds(now + 120_000), now)).toBe(false); + }); +}); + +describe("isLinkedIdentity", () => { + it("needs both the issuer and the subject to match", () => { + const account = { oidcIssuer: ISSUER, oidcSub: "sub-1" }; + expect(isLinkedIdentity(account, ISSUER, "sub-1")).toBe(true); + expect(isLinkedIdentity(account, ISSUER, "sub-2")).toBe(false); + expect(isLinkedIdentity(account, "https://other.test", "sub-1")).toBe( + false, + ); + expect( + isLinkedIdentity({ oidcIssuer: null, oidcSub: null }, ISSUER, "sub-1"), + ).toBe(false); + }); +}); + +describe("isReproofSession", () => { + const bound = { reproofUserId: "u1", reproofSessionId: "s1" }; + + it("holds only for the session and account the start bound", () => { + expect( + isReproofSession(bound, { session: { id: "s1" }, user: { id: "u1" } }), + ).toBe(true); + expect( + isReproofSession(bound, { session: { id: "s2" }, user: { id: "u1" } }), + ).toBe(false); + expect( + isReproofSession(bound, { session: { id: "s1" }, user: { id: "u2" } }), + ).toBe(false); + expect(isReproofSession(bound, null)).toBe(false); + expect( + isReproofSession({}, { session: { id: "s1" }, user: { id: "u1" } }), + ).toBe(false); + }); +}); + +describe("recordOidcReproof", () => { + const now = 1_800_000_000_000; + const args = { + userId: "u1", + sessionId: "s1", + issuer: ISSUER, + ipAddress: "203.0.113.5", + nowMs: now, + }; + + it("stamps reproofAt only, refunds the attempt and audits the success", async () => { + db.findUser.mockResolvedValue(accountRow()); + const outcome = await recordOidcReproof({ + ...args, + identity: { sub: "sub-1", authTime: Math.floor(now / 1000) - 5 }, + }); + expect(outcome).toEqual({ ok: true }); + expect(db.updateSession).toHaveBeenCalledWith({ + where: { id: "s1" }, + data: { reproofAt: new Date(now) }, + }); + expect(refundRateLimit).toHaveBeenCalledWith("auth:step-up:u1"); + expect(auditLog).toHaveBeenCalledWith( + "auth.reproof.succeeded", + expect.objectContaining({ details: { method: "oidc" } }), + ); + }); + + it("refuses a stale auth_time without stamping", async () => { + db.findUser.mockResolvedValue(accountRow()); + const outcome = await recordOidcReproof({ + ...args, + identity: { sub: "sub-1", authTime: Math.floor(now / 1000) - 3600 }, + }); + expect(outcome).toEqual({ ok: false, reason: "stale_auth_time" }); + expect(db.updateSession).not.toHaveBeenCalled(); + expect(auditLog).toHaveBeenCalledWith( + "auth.mfa.failed", + expect.objectContaining({ + details: { + stage: "oidc_reproof", + method: "oidc", + reason: "stale_auth_time", + }, + }), + ); + }); + + it("refuses a different subject without stamping", async () => { + db.findUser.mockResolvedValue(accountRow()); + const outcome = await recordOidcReproof({ + ...args, + identity: { sub: "someone-else", authTime: Math.floor(now / 1000) }, + }); + expect(outcome).toEqual({ ok: false, reason: "identity_mismatch" }); + expect(db.updateSession).not.toHaveBeenCalled(); + }); +}); + +describe("the re-proof return and state", () => { + it("returns to the page it left with the outcome attached", () => { + const url = reproofReturnUrl("/settings/api?tab=mcp", "done"); + expect(url.origin).toBe("https://healthlog.example.test"); + expect(url.pathname).toBe("/settings/api"); + expect(url.searchParams.get("tab")).toBe("mcp"); + expect(url.searchParams.get("reproof")).toBe("done"); + }); + + it("reads a reproof state only with both bindings and no other intent's fields", () => { + const base = { state: "s", nonce: "n", codeVerifier: "v", next: "/" }; + const ok = readStoredOidcState( + JSON.stringify({ + ...base, + intent: "reproof", + reproofUserId: "u1", + reproofSessionId: "s1", + }), + ); + expect(ok?.intent).toBe("reproof"); + expect( + readStoredOidcState( + JSON.stringify({ ...base, intent: "reproof", reproofUserId: "u1" }), + ), + ).toBeNull(); + expect( + readStoredOidcState( + JSON.stringify({ + ...base, + intent: "reproof", + reproofUserId: "u1", + reproofSessionId: "s1", + native: true, + appCodeChallenge: "c", + }), + ), + ).toBeNull(); + expect( + readStoredOidcState(JSON.stringify({ ...base, intent: "other" })), + ).toBeNull(); + }); +}); diff --git a/src/lib/auth/__tests__/oidc.test.ts b/src/lib/auth/__tests__/oidc.test.ts index d2f821cdc..d254ca0fe 100644 --- a/src/lib/auth/__tests__/oidc.test.ts +++ b/src/lib/auth/__tests__/oidc.test.ts @@ -37,6 +37,15 @@ const ENV_KEYS = [ ] as const; const ORIGINAL_ENV: Record = {}; +/** + * A verification result carrying only what `verifyIdToken` reads: the claims + * and the header `jose` returns beside them. + */ +type VerifyResult = Awaited>; +function verified(payload: Record): VerifyResult { + return { payload, protectedHeader: { alg: "RS256" } } as VerifyResult; +} + function jsonResponse(body: unknown, ok = true, status = 200): Response { return { ok, @@ -270,6 +279,25 @@ describe("buildAuthorizationUrl", () => { }), ); expect(url.searchParams.get("prompt")).toBe("login"); + expect(url.searchParams.has("max_age")).toBe(false); + }); + + it("asks for a fresh, timed authentication for a re-proof", () => { + configureEnv(); + const url = new URL( + buildAuthorizationUrl({ + metadata: METADATA_DOC, + config: getOidcConfig()!, + state: "state-1", + nonce: "nonce-1", + codeChallenge: "challenge-1", + redirectUri: "https://healthlog.example.com/api/auth/oidc/callback", + reauthenticate: true, + }), + ); + expect(url.searchParams.get("prompt")).toBe("login"); + expect(url.searchParams.get("max_age")).toBe("0"); + expect(url.searchParams.get("code_challenge_method")).toBe("S256"); }); }); @@ -332,16 +360,15 @@ describe("verifyIdToken", () => { beforeEach(() => configureEnv()); it("returns the identity claims on success", async () => { - // Test payloads deliberately omit the full JWTVerifyResult shape. - vi.mocked(mockJwtVerify).mockResolvedValue({ - payload: { + vi.mocked(mockJwtVerify).mockResolvedValue( + verified({ sub: "user-sub-1", email: "person@example.com", email_verified: true, name: "Person", nonce: "nonce-1", - }, - } as never); + }), + ); const config = getOidcConfig()!; const identity = await verifyIdToken({ metadata: METADATA_DOC, @@ -354,13 +381,43 @@ describe("verifyIdToken", () => { email: "person@example.com", emailVerified: true, name: "Person", + authTime: null, }); }); + it("returns auth_time when the provider states it, and null otherwise", async () => { + vi.mocked(mockJwtVerify).mockResolvedValue( + verified({ + sub: "user-sub-1", + nonce: "nonce-1", + auth_time: 1_700_000_000, + }), + ); + const config = getOidcConfig()!; + const stated = await verifyIdToken({ + metadata: METADATA_DOC, + config, + idToken: "id-token", + nonce: "nonce-1", + }); + expect(stated.authTime).toBe(1_700_000_000); + + vi.mocked(mockJwtVerify).mockResolvedValue( + verified({ sub: "user-sub-1", nonce: "nonce-1", auth_time: "recent" }), + ); + const unstated = await verifyIdToken({ + metadata: METADATA_DOC, + config, + idToken: "id-token", + nonce: "nonce-1", + }); + expect(unstated.authTime).toBeNull(); + }); + it("rejects a nonce mismatch", async () => { - vi.mocked(mockJwtVerify).mockResolvedValue({ - payload: { sub: "user-sub-1", nonce: "wrong-nonce" }, - } as never); + vi.mocked(mockJwtVerify).mockResolvedValue( + verified({ sub: "user-sub-1", nonce: "wrong-nonce" }), + ); const config = getOidcConfig()!; await expect( verifyIdToken({ @@ -373,9 +430,7 @@ describe("verifyIdToken", () => { }); it("rejects a token missing sub", async () => { - vi.mocked(mockJwtVerify).mockResolvedValue({ - payload: { nonce: "nonce-1" }, - } as never); + vi.mocked(mockJwtVerify).mockResolvedValue(verified({ nonce: "nonce-1" })); const config = getOidcConfig()!; await expect( verifyIdToken({ @@ -388,9 +443,9 @@ describe("verifyIdToken", () => { }); it("verifies with a closed asymmetric algorithm allowlist and bounded clock tolerance", async () => { - vi.mocked(mockJwtVerify).mockResolvedValue({ - payload: { sub: "user-sub-1", nonce: "nonce-1" }, - } as never); + vi.mocked(mockJwtVerify).mockResolvedValue( + verified({ sub: "user-sub-1", nonce: "nonce-1" }), + ); const config = getOidcConfig()!; await verifyIdToken({ metadata: METADATA_DOC, @@ -411,9 +466,9 @@ describe("verifyIdToken", () => { }); it("pins the JWKS fetch to the issuer's host with a bounded timeout", async () => { - vi.mocked(mockJwtVerify).mockResolvedValue({ - payload: { sub: "user-sub-1", nonce: "nonce-1" }, - } as never); + vi.mocked(mockJwtVerify).mockResolvedValue( + verified({ sub: "user-sub-1", nonce: "nonce-1" }), + ); const config = getOidcConfig()!; await verifyIdToken({ metadata: METADATA_DOC, diff --git a/src/lib/auth/existing-factor-proof.ts b/src/lib/auth/existing-factor-proof.ts index 9f1374e1d..a98a59490 100644 --- a/src/lib/auth/existing-factor-proof.ts +++ b/src/lib/auth/existing-factor-proof.ts @@ -29,6 +29,7 @@ import { import { hasSecondFactorEnrolled, recentProofMethods, + type RecentProofMethod, } from "@/lib/auth/second-factor"; import { verifyPassword } from "@/lib/auth/password"; import { verifyAuthentication } from "@/lib/auth/passkey"; @@ -171,7 +172,7 @@ export async function recordReproofFailure( userId: string, ipAddress: string | null, stage: string, - method: ExistingFactorProof["method"], + method: RecentProofMethod, reason: string, ): Promise { await auditLog("auth.mfa.failed", { diff --git a/src/lib/auth/guards/fresh-mfa.ts b/src/lib/auth/guards/fresh-mfa.ts index 3945a1a69..c452ad953 100644 --- a/src/lib/auth/guards/fresh-mfa.ts +++ b/src/lib/auth/guards/fresh-mfa.ts @@ -3,6 +3,7 @@ import { annotate, getEvent } from "@/lib/logging/context"; import { getSession } from "@/lib/auth/session"; import { hasSecondFactorEnrolled, + isFirstFactorOnlyProof, recentProofMethods, } from "@/lib/auth/second-factor"; import { meterAccountWrite } from "@/lib/auth/write-meter"; @@ -80,9 +81,10 @@ export async function requireFreshMfa( // does, so the browser can open the re-proof dialog instead of printing a // sentence with nowhere to act on it. Only the possession proofs: a // `POST /api/auth/reproof` with one of them stamps `mfaVerifiedAt`, a - // password stamps `reproofAt` only and would leave this gate shut. + // password or a provider sign-in stamps `reproofAt` only and would leave + // this gate shut. const methods = (await recentProofMethods(sessionData.user)).filter( - (method) => method !== "password", + (method) => !isFirstFactorOnlyProof(method), ); annotate({ meta: { step_up_methods: methods.join(",") } }); throw new StepUpRequiredError( diff --git a/src/lib/auth/oidc-callback/state.ts b/src/lib/auth/oidc-callback/state.ts index 1cc56fd3e..b21171579 100644 --- a/src/lib/auth/oidc-callback/state.ts +++ b/src/lib/auth/oidc-callback/state.ts @@ -27,8 +27,17 @@ export interface StoredOidcState { * names instead of signing in, and only when the cookie still carries this * exact binding. Web only: never together with `native`. */ - intent?: "claim"; + intent?: "claim" | "reproof"; claimBinding?: string; + /** + * Set only for `intent: "reproof"` (`GET /api/auth/oidc/reproof`): the + * signed-in account and the session row the round trip confirms. The + * callback stamps nothing unless the browser still carries exactly this + * session and the provider names exactly this account's linked identity. + * Web only, like the claim. + */ + reproofUserId?: string; + reproofSessionId?: string; } function isStoredOidcState(value: unknown): value is StoredOidcState { @@ -50,11 +59,24 @@ function isStoredOidcState(value: unknown): value is StoredOidcState { if (typeof v.appCodeChallenge !== "string") return false; } if (v.intent !== undefined) { - if (v.intent !== "claim") return false; - if (typeof v.claimBinding !== "string" || v.claimBinding.length === 0) { + if (v.native !== undefined) return false; + if (v.intent === "claim") { + if (typeof v.claimBinding !== "string" || v.claimBinding.length === 0) { + return false; + } + } else if (v.intent === "reproof") { + if ( + typeof v.reproofUserId !== "string" || + v.reproofUserId.length === 0 || + typeof v.reproofSessionId !== "string" || + v.reproofSessionId.length === 0 || + v.claimBinding !== undefined + ) { + return false; + } + } else { return false; } - if (v.native !== undefined) return false; } return true; } diff --git a/src/lib/auth/oidc-reproof.ts b/src/lib/auth/oidc-reproof.ts new file mode 100644 index 000000000..a84e03948 --- /dev/null +++ b/src/lib/auth/oidc-reproof.ts @@ -0,0 +1,158 @@ +/** + * Confirming it is you through the identity provider. + * + * An account created by single sign-on holds no password, passkey or second + * factor, so the recent-proof gate (token minting, exports, share links, the + * admin data actions) had nothing to ask it for. This is the proof such an + * account can give: the browser goes back to the provider with + * `prompt=login` and `max_age=0`, and the callback counts the round trip only + * when the ID token says the provider authenticated the person inside the + * gate's own window and names the identity linked to the signed-in account. + * + * It records the proof exactly as a password re-proof does, `Session.reproofAt` + * and nothing else, so it never stands in for a second factor. It creates no + * session and switches none: the callback stamps the session row the start + * bound into the sealed state, and only while the browser still carries it. + * + * Start: `GET /api/auth/oidc/reproof?next=`. Finish: the shared + * `GET /api/auth/oidc/callback` with `intent: "reproof"` in the state blob. + */ +import { prisma } from "@/lib/db"; +import { auditLog } from "@/lib/auth/audit"; +import { annotate } from "@/lib/logging/context"; +import { oidcAppUrl, type OidcIdentity } from "@/lib/auth/oidc"; +import { + RECENT_PROOF_MAX_AGE_MS, + recordReproofFailure, + refundReproof, +} from "@/lib/auth/existing-factor-proof"; + +/** The query parameter the browser comes back with, and its two values. */ +export const OIDC_REPROOF_RESULT_PARAM = "reproof"; +export type OidcReproofResult = "done" | "failed"; + +/** The stage name the shared re-proof bucket and the audit rows carry. */ +export const OIDC_REPROOF_STAGE = "oidc_reproof"; + +/** + * How far in the future `auth_time` may sit and still count: the clock + * tolerance `verifyIdToken` already grants `iat` and `exp`, for an identity + * provider on another box whose clock runs a little ahead. + */ +const AUTH_TIME_FUTURE_TOLERANCE_MS = 60_000; + +/** + * Did the provider authenticate the person inside the gate's window (the five + * minutes every recent-proof surface uses)? A token + * without `auth_time` is refused: with `max_age` in the request the provider + * owes the claim (OpenID Connect Core 2), and without it nothing says the + * person signed in now rather than an hour ago. + */ +export function isFreshAuthTime( + authTime: number | null, + nowMs: number, + maxAgeMs: number = RECENT_PROOF_MAX_AGE_MS, +): boolean { + if (authTime === null) return false; + const at = authTime * 1000; + return nowMs - at <= maxAgeMs && at - nowMs <= AUTH_TIME_FUTURE_TOLERANCE_MS; +} + +/** + * Is the identity the provider named the one linked to this account? Both + * halves, compared exactly: the subject is only unique per issuer. + */ +export function isLinkedIdentity( + account: { oidcIssuer: string | null; oidcSub: string | null }, + issuer: string, + sub: string, +): boolean { + return ( + account.oidcIssuer !== null && + account.oidcSub !== null && + account.oidcIssuer === issuer && + account.oidcSub === sub + ); +} + +/** + * Does the browser still carry the session the round trip was started from? + * Checked before the code is redeemed, so a mismatch spends nothing at the + * provider. + */ +export function isReproofSession( + bound: { reproofUserId?: string; reproofSessionId?: string }, + live: { session: { id: string }; user: { id: string } } | null, +): boolean { + return ( + live !== null && + typeof bound.reproofSessionId === "string" && + typeof bound.reproofUserId === "string" && + live.session.id === bound.reproofSessionId && + live.user.id === bound.reproofUserId + ); +} + +/** Where the browser lands: the page it left, with the outcome attached. */ +export function reproofReturnUrl(next: string, result: OidcReproofResult): URL { + const url = oidcAppUrl(next); + url.searchParams.set(OIDC_REPROOF_RESULT_PARAM, result); + return url; +} + +export type OidcReproofOutcome = + { ok: true } | { ok: false; reason: "identity_mismatch" | "stale_auth_time" }; + +/** + * Check the verified ID token against the account and stamp the session. + * Every refusal is audited like a failed password re-proof; a success gives + * back the attempt the start charged to the shared bucket. + */ +export async function recordOidcReproof(args: { + userId: string; + sessionId: string; + issuer: string; + identity: Pick; + ipAddress: string | null; + nowMs?: number; +}): Promise { + const { userId, sessionId, issuer, identity, ipAddress } = args; + const nowMs = args.nowMs ?? Date.now(); + + const account = await prisma.user.findUnique({ + where: { id: userId }, + select: { oidcIssuer: true, oidcSub: true }, + }); + let refusal: "identity_mismatch" | "stale_auth_time" | null = null; + if (!account || !isLinkedIdentity(account, issuer, identity.sub)) { + refusal = "identity_mismatch"; + } else if (!isFreshAuthTime(identity.authTime, nowMs)) { + refusal = "stale_auth_time"; + } + if (refusal) { + await recordReproofFailure( + userId, + ipAddress, + OIDC_REPROOF_STAGE, + "oidc", + refusal, + ); + return { ok: false, reason: refusal }; + } + + await prisma.session.update({ + where: { id: sessionId }, + data: { reproofAt: new Date(nowMs) }, + }); + await refundReproof(userId); + await auditLog("auth.reproof.succeeded", { + userId, + ipAddress, + details: { method: "oidc" }, + }); + annotate({ + action: { name: "auth.reproof.succeeded" }, + meta: { method: "oidc" }, + }); + return { ok: true }; +} diff --git a/src/lib/auth/oidc.ts b/src/lib/auth/oidc.ts index 1c5b86f56..08ab084a0 100644 --- a/src/lib/auth/oidc.ts +++ b/src/lib/auth/oidc.ts @@ -244,6 +244,13 @@ export function buildAuthorizationUrl(params: { * keyboard now, not a session someone else left open in this browser. */ forceLogin?: boolean; + /** + * Ask the provider to authenticate afresh AND to say when it did: `prompt=login` + * plus `max_age=0` (OpenID Connect Core 3.1.2.1). With `max_age` in the + * request the ID token must carry `auth_time`, which is what the re-proof + * leg checks before it counts the round trip as a proof. + */ + reauthenticate?: boolean; }): string { const url = new URL(params.metadata.authorization_endpoint); url.search = new URLSearchParams({ @@ -255,7 +262,8 @@ export function buildAuthorizationUrl(params: { nonce: params.nonce, code_challenge: params.codeChallenge, code_challenge_method: "S256", - ...(params.forceLogin ? { prompt: "login" } : {}), + ...(params.forceLogin || params.reauthenticate ? { prompt: "login" } : {}), + ...(params.reauthenticate ? { max_age: "0" } : {}), }).toString(); return url.toString(); } @@ -304,6 +312,12 @@ export interface OidcIdentity { email: string | null; emailVerified: boolean | undefined; name: string | null; + /** + * When the provider last authenticated the person (`auth_time`, seconds + * since the epoch), or null when the token does not say. Only the re-proof + * leg reads it, and it refuses a null. + */ + authTime: number | null; } /** @@ -350,6 +364,11 @@ export async function verifyIdToken(params: { ? payload.email_verified : undefined, name: typeof payload.name === "string" ? payload.name : null, + authTime: + typeof payload.auth_time === "number" && + Number.isFinite(payload.auth_time) + ? payload.auth_time + : null, }; } diff --git a/src/lib/auth/second-factor.ts b/src/lib/auth/second-factor.ts index 4eeccf4f0..b41184971 100644 --- a/src/lib/auth/second-factor.ts +++ b/src/lib/auth/second-factor.ts @@ -1,4 +1,5 @@ import { prisma } from "@/lib/db"; +import { isOidcConfigured } from "@/lib/auth/oidc"; /** * Does this account have a SECOND FACTOR at all? @@ -40,11 +41,25 @@ export async function hasSecondFactorEnrolled(user: { } /** - * A proof the recent-proof gate can ask for. `password` and `passkey` on an - * account without a second factor; `totp`, `webauthn` and `passkey` on one - * with a second factor, where a password alone has never been enough. + * A proof the recent-proof gate can ask for. `password`, `oidc` and `passkey` + * on an account without a second factor; `totp`, `webauthn` and `passkey` on + * one with a second factor, where a password alone has never been enough. + * + * `oidc` is a fresh sign-in at the account's linked identity provider + * (`GET /api/auth/oidc/reproof`), not a body proof: it stamps + * `Session.reproofAt` exactly as a password does, and like a password it never + * stands in for a second factor. */ -export type RecentProofMethod = "password" | "totp" | "webauthn" | "passkey"; +export type RecentProofMethod = + "password" | "oidc" | "totp" | "webauthn" | "passkey"; + +/** + * The proofs that stamp only `Session.reproofAt` and so never clear a gate that + * asks for a second factor (`requireFreshMfa`). + */ +export function isFirstFactorOnlyProof(method: RecentProofMethod): boolean { + return method === "password" || method === "oidc"; +} /** * The proofs that would satisfy `assertRecentCookieProof` for this @@ -63,15 +78,21 @@ export async function recentProofMethods( prisma.passkey.count({ where: { userId: user.id } }), prisma.user.findUnique({ where: { id: user.id }, - select: { passwordHash: true }, + select: { passwordHash: true, oidcIssuer: true, oidcSub: true }, }), ]); const methods: RecentProofMethod[] = []; if (hasSecond) { if (user.totpConfirmedAt) methods.push("totp"); if (keys > 0) methods.push("webauthn"); - } else if (credential?.passwordHash) { - methods.push("password"); + } else { + if (credential?.passwordHash) methods.push("password"); + // A linked identity is only a way to confirm while the instance still + // talks to a provider: with the provider group unset the round trip has + // nowhere to go. + if (credential?.oidcIssuer && credential.oidcSub && isOidcConfigured()) { + methods.push("oidc"); + } } if (passkeys > 0) methods.push("passkey"); return methods; diff --git a/src/lib/openapi/routes/auth.ts b/src/lib/openapi/routes/auth.ts index 78a4217fb..05a213b7e 100644 --- a/src/lib/openapi/routes/auth.ts +++ b/src/lib/openapi/routes/auth.ts @@ -342,6 +342,11 @@ const mfaStatusResponse = z recoveryCodesRemaining: z.number().int(), webauthn: z.array(webauthnCredentialInfo), passkeyNudgeDismissed: z.boolean(), + proofMethods: z + .array(z.enum(["password", "oidc", "totp", "webauthn", "passkey"])) + .describe( + "The ways this account can confirm it is you, the same list a recent-proof refusal names in `meta.methods`. `password` and `oidc` only on an account without a second factor; `oidc` is a browser round trip to the identity provider (`GET /api/auth/oidc/reproof`) and is never offered to a token.", + ), }) .meta({ id: "MfaStatusResponse" }); @@ -599,6 +604,14 @@ const changePasswordRequest = changePasswordSchema.meta({ "Password rotation. `newPassword` and `confirmPassword` must match, and `newPassword` must differ from the current one. The strength floor is 12 characters plus a scored check against the account's own username and email.", }); +const passkeyProviderProofRequest = z + .object({ method: z.literal("oidc") }) + .meta({ + id: "PasskeyProviderProofRequest", + description: + "Take the session's recent identity-provider re-proof (GET /api/auth/oidc/reproof) as the proof. Accepted only for an account whose `proofMethods` list `oidc` and only within five minutes of that re-proof; otherwise 401 `auth.reproof.required` with `meta.methods`. Web only.", + }); + const passkeyReauthRequest = z .object({ method: z @@ -1130,7 +1143,7 @@ export const authPaths: NonNullable = { summary: "Re-prove a credential on a browser session (cookie only)", description: "Stamps the calling browser session as recently re-proved, so for the next five minutes it may run the actions that ask for a fresh proof: GET /api/export/full-backup, POST /api/export/encrypted, POST /api/share-links, POST /api/mcp/tokens, POST /api/tokens/measurements, POST /api/tokens/workouts, POST /api/tokens/documents, and the admin backup download, upload and restore, the admin data wipe and the admin password reset. Those answer 401 `auth.reproof.required` with `meta.methods` (the proofs this account can give) when the session signed in or re-proved more than five minutes ago. A `totp`, `webauthn` or `passkey` proof here also clears the routes that ask for a fresh second factor (the managed-profile acts, the second-factor and passkey changes, the password change, account deletion, the data wipe and key rotation): on a browser session those answer 401 `auth.stepup.required` with `meta.methods` naming the possession proofs the account holds, and the web re-proves here and retries.\n\n" + - 'The body takes the same shapes as POST /api/auth/step-up. On an account with a second factor only `totp`, `webauthn` (a security key) or `passkey` are accepted, because a password has never stood in for the second factor; on one without, `password` or `passkey`. A method the account cannot use here is refused with 422 `auth.reproof.too_weak` and `meta.methods`, before any guess is spent. A second-factor or passkey proof also refreshes the session\'s second-factor stamp, exactly as signing in again would. Assertions begin at POST /api/auth/passkey/register-options with `{ method: "passkey" | "webauthn" }`.\n\n' + + 'The body takes the same shapes as POST /api/auth/step-up. On an account with a second factor only `totp`, `webauthn` (a security key) or `passkey` are accepted, because a password has never stood in for the second factor; on one without, `password` or `passkey`. An account without a second factor whose identity is linked to the configured identity provider is also offered `oidc` in `meta.methods`; that proof is not posted here but given by the browser round trip GET /api/auth/oidc/reproof, which stamps the session exactly as a password proof does. A method the account cannot use here is refused with 422 `auth.reproof.too_weak` and `meta.methods`, before any guess is spent. A second-factor or passkey proof also refreshes the session\'s second-factor stamp, exactly as signing in again would. Assertions begin at POST /api/auth/passkey/register-options with `{ method: "passkey" | "webauthn" }`.\n\n' + "Proofs draw on the account's shared re-proof budget (five per fifteen minutes, 429 once spent). A proof that does not verify is 401 `auth.reproof.failed` and is audited. Bearer callers use POST /api/auth/step-up instead.", requestBody: { required: true, @@ -2454,7 +2467,7 @@ export const authPaths: NonNullable = { description: "Cookie-only — a Bearer token cannot enroll a sign-in credential on this surface at any scope. The route has TWO arms and the body decides which.\n\n" + '**Re-proof arm.** A body of exactly `{ method: "passkey" }` or `{ method: "webauthn" }` begins an assertion ceremony WITHOUT beginning enrollment, and answers with `reauth: true` alongside the options. Use it to obtain the assertion the second arm needs.\n\n' + - "**Enrollment arm.** Any other body must be a full factor proof (the same shape POST /api/auth/step-up takes): a password, a TOTP code, or a completed passkey / security-key assertion. Only then are registration options issued. On an account that already has a second factor a password is refused before it is checked, with 422 `auth.reproof.too_weak` and `meta.methods` naming the proofs that work: a passkey added on the password alone would sign in without the second factor from then on.\n\n" + + '**Enrollment arm.** Any other body must be a full factor proof (the same shape POST /api/auth/step-up takes): a password, a TOTP code, or a completed passkey / security-key assertion. Only then are registration options issued. An account confirmed at its identity provider (GET /api/auth/oidc/reproof, within five minutes) sends `{ method: "oidc" }` instead, which takes that re-proof as the proof and, like a password, clears the second-factor stamp. On an account that already has a second factor a password is refused before it is checked, with 422 `auth.reproof.too_weak` and `meta.methods` naming the proofs that work: a passkey added on the password alone would sign in without the second factor from then on.\n\n' + "Which factor you prove changes the session, and the direction is not the obvious one: a strong proof stamps the session second-factor-verified, while a PASSWORD proof deliberately CLEARS that stamp. Registering with a password is authorised by the single-use, session-bound challenge alone and must not silently upgrade password-only authentication into something the rest of the app reads as a second factor.\n\n" + "Every refusal on the proof path is the same 401 with the same prose — a missing content type, an unparseable body, a body that is not a valid proof shape, and a wrong password are indistinguishable to the caller. A `totp` proof counts only from an authenticator whose enrollment was confirmed.\n\n" + "Each proof on the enrollment arm draws on the account's shared re-proof budget (five per fifteen minutes, the same budget as POST /api/auth/step-up), and every failed proof writes an `auth.mfa.failed` audit row. Once the budget is spent the answer is 429 until the window passes, even for a correct proof.", @@ -2462,7 +2475,11 @@ export const authPaths: NonNullable = { required: true, content: { "application/json": { - schema: z.union([passkeyReauthRequest, stepUpMintSchema]), + schema: z.union([ + passkeyReauthRequest, + passkeyProviderProofRequest, + stepUpMintSchema, + ]), }, }, }, diff --git a/tests/integration/oidc-reproof.test.ts b/tests/integration/oidc-reproof.test.ts new file mode 100644 index 000000000..7d8beb9e7 --- /dev/null +++ b/tests/integration/oidc-reproof.test.ts @@ -0,0 +1,316 @@ +/** + * Confirming at the identity provider (#1296), against real Postgres. + * + * An account created by single sign-on holds no password, passkey or second + * factor, so the recent-proof gate in front of token minting had nothing to + * ask it for. The browser now goes back to the provider with a fresh login + * request, and the callback stamps the signed-in session's `reproofAt` when + * the ID token names this account's linked identity and an authentication + * inside the window. The provider's network legs are stubbed; the state blob, + * the session binding, the stamp, the gate and the passkey enrollment are + * real. + */ +import { NextRequest } from "next/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +process.env.API_TOKEN_HMAC_KEY ??= + "test-hmac-key-oidc-reproof-32-bytes-minimum-1234567890abcd"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const ISSUER = "https://idp.example.test"; +const METADATA = { + issuer: ISSUER, + authorization_endpoint: `${ISSUER}/authorize`, + token_endpoint: `${ISSUER}/token`, + jwks_uri: `${ISSUER}/jwks`, +}; + +vi.mock("@/lib/auth/oidc", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + discoverOidcMetadata: vi.fn(async () => METADATA), + exchangeCodeForTokens: vi.fn(async () => ({ id_token: "id-token" })), + verifyIdToken: vi.fn(), + }; +}); + +const oidc = await import("@/lib/auth/oidc"); + +const USER_ID = "user-oidc-reproof"; +const SUB = "idp-sub-reproof"; +const HOUR = 60 * 60 * 1000; +const OIDC_ENV = { + OIDC_ISSUER_URL: ISSUER, + OIDC_CLIENT_ID: "client", + OIDC_CLIENT_SECRET: "secret", +}; + +type RouteFn = (request: NextRequest) => Promise; + +async function startSession(createdAgoMs: number): Promise { + const now = Date.now(); + const session = await getPrismaClient().session.create({ + data: { + userId: USER_ID, + createdAt: new Date(now - createdAgoMs), + expiresAt: new Date(now + HOUR), + }, + }); + cookieJar.set("healthlog_session", session.id); + return session.id; +} + +/** The provider says it authenticated `sub` this many seconds ago. */ +function providerSays(sub: string, secondsAgo: number) { + vi.mocked(oidc.verifyIdToken).mockResolvedValue({ + sub, + email: "sso@example.test", + emailVerified: true, + name: null, + authTime: Math.floor(Date.now() / 1000) - secondsAgo, + }); +} + +async function startReproof(next: string) { + const { GET } = await import("@/app/api/auth/oidc/reproof/route"); + const res = await (GET as unknown as RouteFn)( + new NextRequest( + `http://localhost:3000/api/auth/oidc/reproof?next=${encodeURIComponent(next)}`, + ), + ); + return res as Response & { cookies?: unknown }; +} + +/** Run the start, then come back through the callback with the browser's cookies. */ +async function roundTrip(next: string, sessionCookie: string) { + const started = await startReproof(next); + expect(started.status).toBe(307); + const authorize = new URL(started.headers.get("location")!); + const stateCookie = /oidc_auth_state=([^;]+)/.exec( + started.headers.get("set-cookie") ?? "", + )?.[1]; + expect(stateCookie).toBeTruthy(); + + const { GET } = await import("@/app/api/auth/oidc/callback/route"); + const res = await (GET as unknown as RouteFn)( + new NextRequest( + `http://localhost:3000/api/auth/oidc/callback?code=abc&state=${authorize.searchParams.get("state")}`, + { + headers: { + cookie: `oidc_auth_state=${stateCookie}; healthlog_session=${sessionCookie}`, + }, + }, + ), + ); + return { authorize, res }; +} + +async function mintMeasurementToken() { + const { POST } = await import("@/app/api/tokens/measurements/route"); + return (POST as unknown as RouteFn)( + new NextRequest("http://localhost:3000/api/tokens/measurements", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ name: "shortcut" }), + }), + ); +} + +async function reproofAtOf(sessionId: string) { + const row = await getPrismaClient().session.findUniqueOrThrow({ + where: { id: sessionId }, + select: { reproofAt: true, mfaVerifiedAt: true }, + }); + return row; +} + +beforeEach(async () => { + vi.clearAllMocks(); + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); + Object.assign(process.env, OIDC_ENV); + process.env.APP_URL = "http://localhost:3000"; + process.env.NEXT_PUBLIC_APP_URL = "http://localhost:3000"; + await getPrismaClient().user.create({ + data: { + id: USER_ID, + username: "sso-only", + email: "sso@example.test", + role: "USER", + oidcIssuer: ISSUER, + oidcSub: SUB, + onboardingCompletedAt: new Date(), + }, + }); +}); + +afterEach(() => { + for (const key of Object.keys(OIDC_ENV)) delete process.env[key]; +}); + +describe("confirming at the identity provider", () => { + it("is what the gate offers an account with nothing else to prove", async () => { + await startSession(HOUR); + const refused = await mintMeasurementToken(); + expect(refused.status).toBe(401); + const body = (await refused.json()) as { + meta?: { errorCode?: string; methods?: string[] }; + }; + expect(body.meta?.errorCode).toBe("auth.reproof.required"); + expect(body.meta?.methods).toEqual(["oidc"]); + }); + + it("asks the provider for a fresh, timed sign-in with state, nonce and PKCE", async () => { + await startSession(HOUR); + const started = await startReproof("/settings/api"); + const authorize = new URL(started.headers.get("location")!); + expect(authorize.origin + authorize.pathname).toBe( + METADATA.authorization_endpoint, + ); + expect(authorize.searchParams.get("prompt")).toBe("login"); + expect(authorize.searchParams.get("max_age")).toBe("0"); + expect(authorize.searchParams.get("state")).toBeTruthy(); + expect(authorize.searchParams.get("nonce")).toBeTruthy(); + expect(authorize.searchParams.get("code_challenge_method")).toBe("S256"); + }); + + it("stamps the session, returns to the page and lets the token mint", async () => { + const sessionId = await startSession(HOUR); + providerSays(SUB, 10); + const { res } = await roundTrip("/settings/api?tab=tokens", sessionId); + expect(res.status).toBe(307); + const back = new URL(res.headers.get("location")!); + expect(back.pathname).toBe("/settings/api"); + expect(back.searchParams.get("tab")).toBe("tokens"); + expect(back.searchParams.get("reproof")).toBe("done"); + + const row = await reproofAtOf(sessionId); + expect(row.reproofAt).not.toBeNull(); + expect(row.mfaVerifiedAt).toBeNull(); + // No session was created or switched. + expect(await getPrismaClient().session.count()).toBe(1); + expect( + await getPrismaClient().auditLog.count({ + where: { userId: USER_ID, action: "auth.reproof.succeeded" }, + }), + ).toBe(1); + + const minted = await mintMeasurementToken(); + expect(minted.status).toBe(201); + }); + + it("then lets the account enroll a passkey", async () => { + const sessionId = await startSession(HOUR); + const { POST } = + await import("@/app/api/auth/passkey/register-options/route"); + const begin = () => + (POST as unknown as RouteFn)( + new NextRequest( + "http://localhost:3000/api/auth/passkey/register-options", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ method: "oidc" }), + }, + ), + ); + + const before = await begin(); + expect(before.status).toBe(401); + expect( + ((await before.json()) as { meta?: { methods?: string[] } }).meta + ?.methods, + ).toEqual(["oidc"]); + + providerSays(SUB, 10); + await roundTrip("/settings/security", sessionId); + + const after = await begin(); + expect(after.status).toBe(200); + const begun = (await after.json()) as { + data: { challengeId: string; options: { challenge: string } }; + }; + expect(begun.data.challengeId).toBeTruthy(); + expect(begun.data.options.challenge).toBeTruthy(); + }); + + it("refuses a provider sign-in older than the window", async () => { + const sessionId = await startSession(HOUR); + providerSays(SUB, 60 * 60); + const { res } = await roundTrip("/settings/api", sessionId); + expect( + new URL(res.headers.get("location")!).searchParams.get("reproof"), + ).toBe("failed"); + expect((await reproofAtOf(sessionId)).reproofAt).toBeNull(); + expect((await mintMeasurementToken()).status).toBe(401); + expect( + await getPrismaClient().auditLog.count({ + where: { userId: USER_ID, action: "auth.mfa.failed" }, + }), + ).toBe(1); + }); + + it("refuses a different subject at the same provider", async () => { + const sessionId = await startSession(HOUR); + providerSays("someone-else", 10); + const { res } = await roundTrip("/settings/api", sessionId); + expect( + new URL(res.headers.get("location")!).searchParams.get("reproof"), + ).toBe("failed"); + expect((await reproofAtOf(sessionId)).reproofAt).toBeNull(); + expect((await mintMeasurementToken()).status).toBe(401); + }); + + it("stamps nothing and spends nothing when the browser comes back with another session", async () => { + const sessionId = await startSession(HOUR); + const other = await getPrismaClient().session.create({ + data: { userId: USER_ID, expiresAt: new Date(Date.now() + HOUR) }, + }); + providerSays(SUB, 10); + const { res } = await roundTrip("/settings/api", other.id); + expect( + new URL(res.headers.get("location")!).searchParams.get("reproof"), + ).toBe("failed"); + expect(oidc.exchangeCodeForTokens).not.toHaveBeenCalled(); + expect((await reproofAtOf(sessionId)).reproofAt).toBeNull(); + expect((await reproofAtOf(other.id)).reproofAt).toBeNull(); + }); + + it("is not offered to an account with a second factor", async () => { + await getPrismaClient().user.update({ + where: { id: USER_ID }, + data: { totpConfirmedAt: new Date() }, + }); + await startSession(HOUR); + const started = await startReproof("/settings/api"); + const back = new URL(started.headers.get("location")!); + expect(back.origin).toBe("http://localhost:3000"); + expect(back.searchParams.get("reproof")).toBe("failed"); + expect(oidc.discoverOidcMetadata).not.toHaveBeenCalled(); + }); +}); From 2490a2d21878194ebac53ca28042603633320994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 29 Oct 2026 09:00:04 +0100 Subject: [PATCH 07/41] fix(i18n): mirror the language to the account only when signed in The provider sent PUT /api/auth/me/locale on mount and on every switch, signed in or not. Signed out the route answers 401, the once-per-tab marker was never set, and every signed-out page load repeated the refused write. The mirror moves out of the provider into LocaleServerSyncBridge, which watches the account payload without fetching it and writes only once an account is known. The choice made on the sign-in page therefore reaches the account right after signing in, and the marker is keyed on the account as well as the language, so another account signing in on the same tab is written for too. The browser test stub no longer treats the PUT as ambient, which is what lets its test see a stray write. --- ...locale-server-sync-bridge.browser.test.tsx | 71 +++++++++++++++++++ .../layout/locale-server-sync-bridge.tsx | 39 ++++++++++ src/components/providers.tsx | 2 + .../locale-first-paint-single-write.test.tsx | 9 ++- src/lib/i18n/context.tsx | 26 ++++--- tests/browser/fetch-stub.ts | 15 ++-- 6 files changed, 134 insertions(+), 28 deletions(-) create mode 100644 src/components/layout/__tests__/locale-server-sync-bridge.browser.test.tsx create mode 100644 src/components/layout/locale-server-sync-bridge.tsx diff --git a/src/components/layout/__tests__/locale-server-sync-bridge.browser.test.tsx b/src/components/layout/__tests__/locale-server-sync-bridge.browser.test.tsx new file mode 100644 index 000000000..86ea74cbf --- /dev/null +++ b/src/components/layout/__tests__/locale-server-sync-bridge.browser.test.tsx @@ -0,0 +1,71 @@ +/** + * The language reaches the account only when there is an account. + * + * The provider used to PUT the locale on every mount, signed in or not. A + * signed-out page got a 401, the once-per-tab marker was never set, and every + * signed-out page load sent the refused write again. The write now belongs to + * `LocaleServerSyncBridge`, which waits for the account payload without + * fetching it and then sends the choice once. + */ +import { describe, expect, it } from "vitest"; + +import { queryKeys } from "@/lib/query-keys"; + +import { authUser, renderWithApp } from "../../../../tests/browser/render"; +import { stubFetch } from "../../../../tests/browser/fetch-stub"; +import { LocaleServerSyncBridge } from "../locale-server-sync-bridge"; + +const LOCALE_PUT = "/api/auth/me/locale"; + +async function settle(): Promise { + await new Promise((resolve) => setTimeout(resolve, 150)); +} + +describe("LocaleServerSyncBridge", () => { + it("sends nothing while nobody is signed in, and never asks who is", async () => { + const fetchStub = stubFetch().on("PUT", LOCALE_PUT, { data: null }); + await renderWithApp(, { + user: null, + locale: "de", + }); + await settle(); + expect(fetchStub.calls("PUT", LOCALE_PUT)).toHaveLength(0); + expect(fetchStub.calls("GET", "/api/auth/me")).toHaveLength(0); + }); + + it("sends the choice made while signed out once an account appears", async () => { + const fetchStub = stubFetch().on("PUT", LOCALE_PUT, { data: null }); + const { queryClient } = await renderWithApp(, { + user: null, + locale: "de", + }); + await settle(); + expect(fetchStub.calls("PUT", LOCALE_PUT)).toHaveLength(0); + + queryClient.setQueryData( + queryKeys.authMe(), + authUser({ id: "account-signed-in-later" }), + ); + await expect.poll(() => fetchStub.calls("PUT", LOCALE_PUT).length).toBe(1); + expect(fetchStub.calls("PUT", LOCALE_PUT)[0].body).toEqual({ + locale: "de", + }); + }); + + it("writes once per tab for the same account and language", async () => { + const fetchStub = stubFetch().on("PUT", LOCALE_PUT, { data: null }); + const first = await renderWithApp(, { + user: { id: "account-already-synced" }, + locale: "en", + }); + await expect.poll(() => fetchStub.calls("PUT", LOCALE_PUT).length).toBe(1); + await first.screen.unmount(); + + await renderWithApp(, { + user: { id: "account-already-synced" }, + locale: "en", + }); + await settle(); + expect(fetchStub.calls("PUT", LOCALE_PUT)).toHaveLength(1); + }); +}); diff --git a/src/components/layout/locale-server-sync-bridge.tsx b/src/components/layout/locale-server-sync-bridge.tsx new file mode 100644 index 000000000..23bb093f7 --- /dev/null +++ b/src/components/layout/locale-server-sync-bridge.tsx @@ -0,0 +1,39 @@ +"use client"; + +import { useEffect } from "react"; +import { useQuery } from "@tanstack/react-query"; + +import { fetchMe } from "@/hooks/use-auth"; +import { persistLocaleToServer, useTranslations } from "@/lib/i18n/context"; +import { queryKeys } from "@/lib/query-keys"; + +/** + * Keeps `User.locale` in step with the language on screen, for a signed-in + * account only, so the proactive Coach nudge and the other cookie-blind + * background paths write in the language the person reads. + * + * It watches the account payload without fetching it (`enabled: false`): the + * shell already reads `/api/auth/me` on every signed-in page, and a signed-out + * page has no account to write to. Fetching here would trade the refused + * locale PUT for a refused account read on every signed-out load. Same key + * and same fetcher as `useAuth`, so the cache entry is shared, not shadowed. + * + * Fires when an account appears (right after signing in, carrying the choice + * made on the sign-in page) and on every switch; `persistLocaleToServer` + * skips a value this tab already wrote for this account. + */ +export function LocaleServerSyncBridge() { + const { locale } = useTranslations(); + const { data: account } = useQuery({ + queryKey: queryKeys.authMe(), + queryFn: fetchMe, + enabled: false, + }); + const accountId = account?.id ?? null; + + useEffect(() => { + if (accountId) persistLocaleToServer(locale, accountId); + }, [locale, accountId]); + + return null; +} diff --git a/src/components/providers.tsx b/src/components/providers.tsx index 2549b61b0..de8dbec17 100644 --- a/src/components/providers.tsx +++ b/src/components/providers.tsx @@ -22,6 +22,7 @@ import { ServiceWorkerRegistrar } from "@/components/service-worker-registrar"; import { SharedRecordGrantLossBridge } from "@/components/layout/shared-record-grant-loss-bridge"; import { RecordSessionFenceBridge } from "@/components/layout/record-session-fence-bridge"; import { OidcReproofReturnBridge } from "@/components/auth/oidc-reproof-return"; +import { LocaleServerSyncBridge } from "@/components/layout/locale-server-sync-bridge"; import { useAuth, type AccountAccessStatus } from "@/hooks/use-auth"; import { useRecordSessionTransition } from "@/hooks/use-record-session-transition"; import { isDashboardSnapshotEnabled } from "@/lib/dashboard/snapshot-flag"; @@ -434,6 +435,7 @@ export function Providers({ + {children} diff --git a/src/lib/i18n/__tests__/locale-first-paint-single-write.test.tsx b/src/lib/i18n/__tests__/locale-first-paint-single-write.test.tsx index 7ca105bff..1ee834ca9 100644 --- a/src/lib/i18n/__tests__/locale-first-paint-single-write.test.tsx +++ b/src/lib/i18n/__tests__/locale-first-paint-single-write.test.tsx @@ -63,8 +63,11 @@ describe("first paint carries the resolved locale, not a default", () => { /prev\.locale === locale \? \{ locale, messages: loaded \} : prev/, ); - // And the persistence effect keys on the locale alone, so one mount - // with one locale means one write. - expect(src).toMatch(/persistLocaleToServer\(locale\);\s*\}, \[locale\]\);/); + // And the provider writes nothing to the server itself: the mirror lives + // in `LocaleServerSyncBridge`, keyed on the locale and the signed-in + // account, so a signed-out mount sends nothing and one account with one + // locale means one write (its browser test pins both). + expect(src).toMatch(/export function persistLocaleToServer\(/); + expect(src).not.toMatch(/persistLocaleToServer\(locale/); }); }); diff --git a/src/lib/i18n/context.tsx b/src/lib/i18n/context.tsx index 5fd96c77d..13e9734af 100644 --- a/src/lib/i18n/context.tsx +++ b/src/lib/i18n/context.tsx @@ -125,16 +125,23 @@ function persistLocale(newLocale: Locale) { // the server-set cookie the route emits in reply: Safari's ITP caps the // script-written copy at 7 days, and a Set-Cookie is exempt. // -// Once per tab session and per locale, not on every mount: every page load -// mounts the provider, and every authenticated write is charged to the +// Once per tab session, account and locale, not on every mount: every page +// load mounts the provider, and every authenticated write is charged to the // account's shared write bucket, so a mount-time PUT spent one write per page // view on a value that had not changed. A switch still writes at once (the // locale differs from the one recorded), and a new tab writes once. +// +// Only for a signed-in account. The route answers 401 to nobody, the marker +// is then never set, and every signed-out page load used to repeat the +// refused PUT. Signed out, the choice stays in localStorage and the cookie; +// `LocaleServerSyncBridge` calls this once an account is known, so the +// choice made before signing in reaches the column right after. const LOCALE_SYNCED_KEY = "healthlog-locale-synced"; -function persistLocaleToServer(newLocale: Locale) { +export function persistLocaleToServer(newLocale: Locale, accountId: string) { + const marker = `${accountId}:${newLocale}`; try { - if (sessionStorage.getItem(LOCALE_SYNCED_KEY) === newLocale) return; + if (sessionStorage.getItem(LOCALE_SYNCED_KEY) === marker) return; } catch { // Storage unavailable: write as before. } @@ -148,7 +155,7 @@ function persistLocaleToServer(newLocale: Locale) { .then((res) => { if (!res.ok) return; try { - sessionStorage.setItem(LOCALE_SYNCED_KEY, newLocale); + sessionStorage.setItem(LOCALE_SYNCED_KEY, marker); } catch { // Storage unavailable: the next mount writes again. } @@ -238,15 +245,6 @@ export function I18nProvider({ }; }, [locale]); - // v1.25.0 — keep `User.locale` in step with the committed UI locale. Runs on - // the first mount of a tab session (backfilling the column for users who set - // their language before this landed) and on every switch, so the proactive - // Coach nudge and other cookie-blind background paths render in the language - // the user reads. - useEffect(() => { - persistLocaleToServer(locale); - }, [locale]); - const setLocale = useCallback((newLocale: Locale) => { if (!(locales as readonly string[]).includes(newLocale)) return; diff --git a/tests/browser/fetch-stub.ts b/tests/browser/fetch-stub.ts index 5bcbe4fd2..99482907a 100644 --- a/tests/browser/fetch-stub.ts +++ b/tests/browser/fetch-stub.ts @@ -14,10 +14,10 @@ * state. A request no route matches fails with a 404 envelope and is recorded * too, so an unexpected call is visible rather than hanging. * - * One request is ambient: `I18nProvider` mirrors the active locale to - * `PUT /api/auth/me/locale` on every mount, as it does in the app. The stub - * answers it and leaves it out of `requests`, so "nothing was sent" means - * nothing the component under test sent. + * Nothing is ambient: the locale mirror (`PUT /api/auth/me/locale`) belongs + * to `LocaleServerSyncBridge` in the app's providers, which `renderWithApp` + * does not mount, so "nothing was sent" means nothing the component under + * test sent. * * `setup.ts` restores the real `fetch` after every test. */ @@ -72,9 +72,6 @@ export interface FetchStub { let realFetch: typeof fetch | null = null; -/** Sent by the providers on every mount, not by the component under test. */ -const AMBIENT = [{ method: "PUT", path: "/api/auth/me/locale" }]; - function matches(route: Route, method: string, path: string): boolean { if (route.method !== method) return false; const bare = path.split("?")[0]; @@ -123,10 +120,6 @@ export function stubFetch(): FetchStub { window.fetch = async (input, init) => { const request = await record(input, init); - const ambient = AMBIENT.some( - (a) => a.method === request.method && a.path === request.path, - ); - if (ambient) return envelope({ data: request.body ?? null }); requests.push(request); // The newest route wins, so a test can override a default it set up. const route = [...routes] From 9eed407ccc5749fcfadb7b093d957f528ebd7247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 29 Oct 2026 09:06:17 +0100 Subject: [PATCH 08/41] feat(measurements): fold Google Health and Oura heart rate like the exports The dense hourly fold only took Apple Health and Health Connect samples, so heart rate from the provider syncs that store one row per sample stayed raw forever. One self-hosted account held about 25 million Google Health pulse rows after a full-history backfill, and every reader and fold that falls back to the raw table paid for it (#1297). The fold now walks a map of sources to the dense types each stores sample by sample: every dense type for Apple Health and Health Connect, heart rate for Google Health and Oura. Their daily SpO2 values are no intraday stream and stay as they are; Fitbit does not sync intraday heart rate, and WHOOP, Withings and Polar write dense types only once per recovery, night or spot check. Each source folds into its own hourly stats rows, and the DAY rollup is still taken from the raw samples before they go. The fold deletes the samples it averages, and the provider syncs write first-write-wins on the externalId, so a backfill or full sync would have put every folded sample back. Google Health would even have re-keyed an hourly row into a sample that fell on its anchor through the natural-key rescue. The folded_window rule now has one hour rule per fold source, and both syncs drop the samples of an hour the fold already took before they write. Recent samples cost no lookup. The boot discovery of the fold probes each account and source for the first raw sample it would take instead of grouping every such row, and the one-time fold repair stays on the two sources it was written for. --- .../baselines/source-file-lines.json | 4 +- src/lib/google-health/mapping.md | 2 +- src/lib/google-health/sync-core.ts | 36 ++-- src/lib/jobs/dense-intraday-hourly-rebuild.ts | 6 +- src/lib/jobs/dense-intraday-retention.ts | 68 ++++--- src/lib/jobs/measurement-fold-repair.ts | 8 +- .../dense-intraday-drain-exemption.test.ts | 2 +- .../dense-intraday-retention.test.ts | 8 +- .../__tests__/folded-window.test.ts | 93 +++++++++ .../dense-intraday-hourly-rebuild.ts | 2 +- .../measurements/dense-intraday-retention.ts | 63 +----- .../measurements/dense-intraday-sources.ts | 99 ++++++++++ src/lib/measurements/folded-window.ts | 88 +++++++-- src/lib/oura/sync-heart-rate.ts | 23 ++- tests/integration/provider-dense-fold.test.ts | 185 ++++++++++++++++++ 15 files changed, 543 insertions(+), 144 deletions(-) create mode 100644 src/lib/measurements/dense-intraday-sources.ts create mode 100644 tests/integration/provider-dense-fold.test.ts diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index d86b28d50..31b681f6f 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 84, - "total": 94733, + "total": 94686, "entries": { "app/medications/page-client.tsx": 839, "components/charts/mood-chart.tsx": 1166, @@ -71,7 +71,7 @@ "lib/managed-profiles/handover.ts": 986, "lib/mcp/record-search.ts": 1467, "lib/mcp/rich-reads.ts": 1467, - "lib/measurements/dense-intraday-retention.ts": 1057, + "lib/measurements/dense-intraday-retention.ts": 1010, "lib/measurements/import-apple-health-export.ts": 1722, "lib/medications/scheduling/cadence.ts": 976, "lib/medications/scheduling/recurrence.ts": 1100, diff --git a/src/lib/google-health/mapping.md b/src/lib/google-health/mapping.md index eb958bf6b..0d92bef67 100644 --- a/src/lib/google-health/mapping.md +++ b/src/lib/google-health/mapping.md @@ -89,7 +89,7 @@ Scope: `googlehealth.health_metrics_and_measurements.readonly`. | `daily-heart-rate-variability` | `dailyHeartRateVariability.averageHeartRateVariabilityMilliseconds` | `HRV_RMSSD` | ms | `hrv` | daily | Google documents the daily average as RMSSD, so it lands in `HRV_RMSSD`, apart from the SDNN `HEART_RATE_VARIABILITY`. | | `daily-resting-heart-rate` | `dailyRestingHeartRate.beatsPerMinute` (int64 string) | `RESTING_HEART_RATE` | bpm | `rhr` | daily | | | `daily-respiratory-rate` | `dailyRespiratoryRate.breathsPerMinute` (number) | `RESPIRATORY_RATE` | breaths/min | `resp_rate` | daily | `respiratory-rate` does NOT exist in the catalogue. Schema is `{date, breathsPerMinute}` — the earlier `dailyRespiratoryRateBpm` leaf never existed. | -| `heart-rate` | `heartRate.beatsPerMinute` (int64 string) | `PULSE` | bpm | `hr` | sample | Intraday spot HR; per-minute volume — consider the 14-day-max rollup if PULSE should stay coarse. | +| `heart-rate` | `heartRate.beatsPerMinute` (int64 string) | `PULSE` | bpm | `hr` | sample | Intraday spot HR, one row per sample; older than 90 days it folds to hourly means (`dense-intraday-sources.ts`). | | `blood-glucose` | `bloodGlucose.bloodGlucoseMilligramsPerDeciliter` (number) | `BLOOD_GLUCOSE` | mg/dL | `glucose` | sample | Already in the canonical mg/dL storage unit. Meal/timing/specimen enums not stored. | | `core-body-temperature` | `coreBodyTemperature.temperatureCelsius` (number) | `BODY_TEMPERATURE` | celsius | `core_temp` | sample | Core slot — distinct from SKIN_TEMPERATURE / WRIST_TEMPERATURE. | | `daily-sleep-temperature-derivations` | `dailySleepTemperatureDerivations.nightlyTemperatureCelsius` (number) | `WRIST_TEMPERATURE` | celsius | `wrist_temp` | daily | ABSOLUTE nightly skin temperature (mean over the night's samples) — the schema refutes the earlier "deviation-only" premise. Baseline/stddev derivations not stored. | diff --git a/src/lib/google-health/sync-core.ts b/src/lib/google-health/sync-core.ts index 44f200e28..7553f1268 100644 --- a/src/lib/google-health/sync-core.ts +++ b/src/lib/google-health/sync-core.ts @@ -36,6 +36,7 @@ import { type MeasurementReconciliationVerdict, } from "@/lib/measurements/reconcile-external-measurement"; import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; +import { withoutFoldedSamples } from "@/lib/measurements/folded-window"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { recordSyncFailure, type FailureKind } from "@/lib/integrations/status"; import { collapseToTypeDayKeys } from "@/lib/rollups/measurement-rollups"; @@ -1128,7 +1129,7 @@ export async function upsertGoogleHealthMeasurements( // before they can become rows. A refused reading is dropped and tallied on // the ambient event, never clamped and never written for a later reader to // discover. - const { kept: readings, dropped: naturalKeyTwins } = collapseNaturalKeyTwins( + const { kept: collapsed, dropped: naturalKeyTwins } = collapseNaturalKeyTwins( dropImplausibleMeasurements("googleHealth", incoming, (r) => ({ type: r.type, value: r.value, @@ -1137,14 +1138,18 @@ export async function upsertGoogleHealthMeasurements( if (naturalKeyTwins > 0) { annotate({ meta: { "googleHealth.naturalKeyTwins": naturalKeyTwins } }); } + // A sample of an hour the dense fold already took is not written again. + const { kept: readings } = await withoutFoldedSamples( + prisma, + userId, + "GOOGLE_HEALTH", + collapsed, + ); if (readings.length === 0) return { imported: 0, touched: [], inserted: [] }; // Probe EVERY existing row (live AND tombstoned) for the batch's externalIds - // in a single query. The full unique index guarantees at most one row per - // `(type, externalId)` key, and a tombstoned match must take the UPDATE - // branch (which clears `deletedAt`) — treating it as absent would plan an - // insert that `skipDuplicates` silently drops against the tombstone's key, - // permanently wedging the key (see TOMBSTONES RESURRECT above). + // in one query: a tombstoned match must take the UPDATE branch, or the + // insert is silently dropped against its key (see TOMBSTONES RESURRECT). const externalIds = readings.map((r) => r.externalId); interface ProbedRow { id: string; @@ -1162,10 +1167,8 @@ export async function upsertGoogleHealthMeasurements( source: "GOOGLE_HEALTH", externalId: { in: externalIds }, }, - // The payload fields ride along so the update branch can skip a no-op - // overwrite: the 24 h overlap re-fetches every recent row hourly, and an - // unconditional write would bump `syncVersion` (iOS delta churn) on rows - // whose values did not change. + // The payload rides along so the update branch can skip a no-op + // overwrite (the hourly 24 h overlap would bump `syncVersion` for nothing). select: { id: true, type: true, @@ -1193,9 +1196,8 @@ export async function upsertGoogleHealthMeasurements( ]), ); } catch (err) { - // A probe failure must not strand the whole batch; fall back to treating - // every row as fresh (`skipDuplicates` absorbs the collision on the full - // unique index — those rows simply retry on the next sync tick). + // A probe failure must not strand the batch: every row is treated as + // fresh, and `skipDuplicates` absorbs the collisions until the next tick. getEvent()?.addWarning(`google-health: row probe failed: ${err}`); } @@ -1225,11 +1227,9 @@ export async function upsertGoogleHealthMeasurements( const key = `${type} ${r.externalId}`; const existingRow = rowByKey.get(key); if (existingRow) { - // Skip the no-op overwrite: a LIVE row whose payload matches the - // re-fetched reading exactly gains nothing from a write, and the - // unconditional `syncVersion` bump churned the DB + every iOS delta pull - // hourly for the whole 24 h overlap window. A TOMBSTONED row always - // updates — the write is what resurrects it (`deletedAt: null`). + // Skip the no-op overwrite of a LIVE row whose payload matches exactly + // (it would churn `syncVersion` and every iOS delta pull). A TOMBSTONED + // row always updates: the write is what resurrects it. const unchanged = existingRow.deletedAt === null && existingRow.value === r.value && diff --git a/src/lib/jobs/dense-intraday-hourly-rebuild.ts b/src/lib/jobs/dense-intraday-hourly-rebuild.ts index e5a8c0f8a..b58219802 100644 --- a/src/lib/jobs/dense-intraday-hourly-rebuild.ts +++ b/src/lib/jobs/dense-intraday-hourly-rebuild.ts @@ -30,10 +30,8 @@ import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { DENSE_INTRADAY_RETENTION_ENABLED } from "@/lib/jobs/dense-intraday-retention"; -import { - DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, -} from "@/lib/measurements/dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_DAYS } from "@/lib/measurements/dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "@/lib/measurements/dense-intraday-sources"; import { runDenseIntradayHourlyRebuild } from "@/lib/measurements/dense-intraday-hourly-rebuild"; import { DENSE_INTRADAY_HOURLY_REBUILD_QUEUE } from "./dense-intraday-hourly-rebuild-queue"; diff --git a/src/lib/jobs/dense-intraday-retention.ts b/src/lib/jobs/dense-intraday-retention.ts index 27b7322ae..2ced5d3f5 100644 --- a/src/lib/jobs/dense-intraday-retention.ts +++ b/src/lib/jobs/dense-intraday-retention.ts @@ -18,16 +18,18 @@ * `src/lib/jobs/reminder-worker.ts` so pg-boss provisions it at boot; an * unregistered queue silently never drains. */ -import { dropPendingDeletion } from "@/lib/account-deletion/active-account"; +import { ACTIVE_ACCOUNT } from "@/lib/account-deletion/active-account"; import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { runDenseIntradayRetention, - DENSE_INTRADAY_RETENTION_TYPES, DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_FOLD_SOURCES, } from "@/lib/measurements/dense-intraday-retention"; +import { + DENSE_INTRADAY_FOLD_SOURCES, + DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE, +} from "@/lib/measurements/dense-intraday-sources"; import { DENSE_INTRADAY_RETENTION_QUEUE } from "./dense-intraday-retention-queue"; /** @@ -133,8 +135,8 @@ export async function runDenseIntradayRetentionForUser( /** * Boot-time discovery. Finds every user holding at least one LIVE - * per-sample dense-tier row (a row of a `DENSE_INTRADAY_FOLD_SOURCES` source - * and a `DENSE_INTRADAY_RETENTION_TYPES` type whose externalId does NOT start + * per-sample dense-tier row (a row of a source and type the fold takes, + * `DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE`, whose externalId does NOT start * with the daily-stats prefix, that is not tombstoned, AND whose * `measuredAt` is older than the retention window) and enqueues one * retention job per account. @@ -161,36 +163,40 @@ export async function enqueueBootTimeDenseIntradayRetention(): Promise<{ } try { - // The dense-tier type set is a closed compile-time list of enum - // members; splice-free — Prisma binds the `type IN (...)` array as - // parameters. - const types = Array.from(DENSE_INTRADAY_RETENTION_TYPES); - if (types.length === 0) { - return { enqueued: 0, skipped: 0, error: null }; - } - const windowStart = new Date( Date.now() - DENSE_INTRADAY_RETENTION_DAYS * 24 * 60 * 60 * 1000, ); - // `groupBy` compiles to a server-side `GROUP BY` — Prisma's - // `distinct` dedups in the client AFTER pulling every matching row, - // an instance-wide scan of every out-of-window dense-tier row just - // to list user ids. - const users = await dropPendingDeletion( - prisma, - await prisma.measurement.groupBy({ - by: ["userId"], - where: { - source: { in: [...DENSE_INTRADAY_FOLD_SOURCES] }, - type: { in: types }, - deletedAt: null, - NOT: { externalId: { startsWith: "stats:" } }, - measuredAt: { lt: windowStart }, - }, - }), - (u) => u.userId, - ); + // One probe per account and source for the first raw sample the fold + // would take, which stops at that row. A grouped count over every such + // row read the whole out-of-window history of every account to list + // their ids, and on an account with tens of millions of heart-rate rows + // that is most of the table (#1297). + const accounts = await prisma.user.findMany({ + where: ACTIVE_ACCOUNT, + select: { id: true }, + orderBy: { createdAt: "asc" }, + }); + const users: Array<{ userId: string }> = []; + for (const { id } of accounts) { + for (const source of DENSE_INTRADAY_FOLD_SOURCES) { + const raw = await prisma.measurement.findFirst({ + where: { + userId: id, + source, + type: { in: [...DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE[source]] }, + deletedAt: null, + NOT: { externalId: { startsWith: "stats:" } }, + measuredAt: { lt: windowStart }, + }, + select: { id: true }, + }); + if (raw) { + users.push({ userId: id }); + break; + } + } + } if (users.length === 0) { return { enqueued: 0, skipped: 0, error: null }; diff --git a/src/lib/jobs/measurement-fold-repair.ts b/src/lib/jobs/measurement-fold-repair.ts index c9c42e3b0..8a8738e41 100644 --- a/src/lib/jobs/measurement-fold-repair.ts +++ b/src/lib/jobs/measurement-fold-repair.ts @@ -116,12 +116,11 @@ import { hourOfDayForUserTz, } from "@/lib/measurements/consolidation-tz"; import { - DENSE_INTRADAY_FOLD_SOURCES, DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, deriveDailyRestingFromPulse, hourlyStatsExternalId, } from "@/lib/measurements/dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "@/lib/measurements/dense-intraday-sources"; import { absorbIntoFold, foldConstituentHorizon, @@ -160,6 +159,9 @@ interface RepairStep { foldedAfterMs: number; } +/** Sources the dense fold took when this repair was written; later ones never split a window. */ +const REPAIRED_DENSE_SOURCES = ["APPLE_HEALTH", "HEALTH_CONNECT"] as const; + /** Fixed order, so a resume position (`step`) means the same on every run. */ export const FOLD_REPAIR_STEPS: readonly RepairStep[] = [ ...[...HIGH_FREQUENCY_MEAN_TYPES].map((type): RepairStep => ({ @@ -168,7 +170,7 @@ export const FOLD_REPAIR_STEPS: readonly RepairStep[] = [ grain: "day", foldedAfterMs: CONSOLIDATION_GRACE_CUTOFF_HOURS * HOUR_MS, })), - ...DENSE_INTRADAY_FOLD_SOURCES.flatMap((source) => + ...REPAIRED_DENSE_SOURCES.flatMap((source) => [...DENSE_INTRADAY_RETENTION_TYPES].map((type): RepairStep => ({ type, source, diff --git a/src/lib/measurements/__tests__/dense-intraday-drain-exemption.test.ts b/src/lib/measurements/__tests__/dense-intraday-drain-exemption.test.ts index fb0311aa7..5878092f0 100644 --- a/src/lib/measurements/__tests__/dense-intraday-drain-exemption.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-drain-exemption.test.ts @@ -26,7 +26,7 @@ import { CUMULATIVE_HK_TYPES, HIGH_FREQUENCY_MEAN_TYPES, } from "../apple-health-mapping"; -import { DENSE_INTRADAY_RETENTION_TYPES } from "../dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "../dense-intraday-sources"; describe("dense intra-day retention — drain exemption (WX-E)", () => { it("EXEMPTS HEART_RATE_VARIABILITY from the destructive daily-mean drain", () => { diff --git a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts index 2d76f13ae..6c7c88ba3 100644 --- a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts @@ -21,8 +21,8 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({ import { runDenseIntradayRetention, DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, } from "../dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "../dense-intraday-sources"; import type { PerSampleRow } from "../drain-per-sample-cumulative"; import { foldBoundary } from "../consolidation-tz"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; @@ -161,7 +161,7 @@ describe("runDenseIntradayRetention — retention bound", () => { } }); - it("scans exactly the dense-tier types, once per fold source, Apple Health first", async () => { + it("scans each fold source's dense types, once per source, Apple Health first", async () => { const { mock, findManyMeasurement } = buildPrismaMock({}); await runDenseIntradayRetention(mock, { log: () => {} }); @@ -172,9 +172,13 @@ describe("runDenseIntradayRetention — retention bound", () => { const types = Array.from(DENSE_INTRADAY_RETENTION_TYPES); // One walk per source, each scoped to its one source: two sources // never share a mean, and manual or Withings rows are never scanned. + // The provider syncs fold only what they store sample by sample: heart + // rate, never their daily SpO2 (#1297). expect(scans).toEqual([ ...types.map((type) => `APPLE_HEALTH:${type}`), ...types.map((type) => `HEALTH_CONNECT:${type}`), + "GOOGLE_HEALTH:PULSE", + "OURA:PULSE", ]); }); }); diff --git a/src/lib/measurements/__tests__/folded-window.test.ts b/src/lib/measurements/__tests__/folded-window.test.ts index d83a3d464..4b36a5b98 100644 --- a/src/lib/measurements/__tests__/folded-window.test.ts +++ b/src/lib/measurements/__tests__/folded-window.test.ts @@ -14,6 +14,7 @@ import { findCompactionTombstones, findFoldedWindowDuplicates, isFoldedWindowCandidate, + withoutFoldedSamples, } from "../folded-window"; const DAY = 86_400_000; @@ -332,3 +333,95 @@ describe("Health Connect", () => { ]); }); }); + +describe("provider syncs (#1297)", () => { + const old = new Date("2026-03-01T10:15:00.000Z"); + const hour = "stats:HKQuantityTypeIdentifierHeartRate:2026-03-01T10"; + + it("takes the heart rate Google Health and Oura store per sample, and nothing else of theirs", () => { + for (const source of ["GOOGLE_HEALTH", "OURA"]) { + expect(isFoldedWindowCandidate(raw("PULSE", old, { source }), NOW)).toBe( + true, + ); + // A nightly or daily SpO2 value is no intraday stream: never folded. + expect( + isFoldedWindowCandidate(raw("OXYGEN_SATURATION", old, { source }), NOW), + ).toBe(false); + } + // Neither stores intraday heart rate as samples. + for (const source of ["FITBIT", "WHOOP", "WITHINGS"]) { + expect(isFoldedWindowCandidate(raw("PULSE", old, { source }), NOW)).toBe( + false, + ); + } + }); + + it("drops the samples of an hour the fold took, under the write's own source", async () => { + // The externalId read finds the hour's row; the reach read finds nothing. + const findMany = vi.fn( + async (args: { + where: { source: string; externalId: { in?: string[] } }; + }) => + args.where.source === "GOOGLE_HEALTH" && + args.where.externalId.in?.includes(hour) + ? [{ type: "PULSE", externalId: hour }] + : [], + ); + const findUnique = vi.fn(async () => ({ timezone: "UTC" })); + const rows = [ + { type: "PULSE", externalId: "hr:a", measuredAt: old, value: 61 }, + { + type: "PULSE", + externalId: "hr:b", + measuredAt: new Date("2026-03-01T12:15:00.000Z"), + value: 62, + }, + { + type: "PULSE", + externalId: "hr:c", + measuredAt: new Date(NOW.getTime() - DAY), + value: 63, + }, + ]; + const { kept, dropped } = await withoutFoldedSamples( + { + measurement: { findMany }, + user: { findUnique }, + } as unknown as Parameters[0], + "u1", + "GOOGLE_HEALTH", + rows, + { now: NOW }, + ); + expect(dropped).toBe(1); + expect(kept.map((row) => row.externalId)).toEqual(["hr:b", "hr:c"]); + for (const call of findMany.mock.calls) { + expect(call[0].where.source).toBe("GOOGLE_HEALTH"); + } + }); + + it("asks nothing for a write of recent samples", async () => { + const findMany = vi.fn(); + const findUnique = vi.fn(); + const rows = [ + { + type: "PULSE", + externalId: "heartrate:x", + measuredAt: new Date(NOW.getTime() - 2 * 3_600_000), + }, + ]; + const { kept } = await withoutFoldedSamples( + { + measurement: { findMany }, + user: { findUnique }, + } as unknown as Parameters[0], + "u1", + "OURA", + rows, + { now: NOW }, + ); + expect(kept).toEqual(rows); + expect(findMany).not.toHaveBeenCalled(); + expect(findUnique).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/measurements/dense-intraday-hourly-rebuild.ts b/src/lib/measurements/dense-intraday-hourly-rebuild.ts index 0c49f4c61..6f34f3d0a 100644 --- a/src/lib/measurements/dense-intraday-hourly-rebuild.ts +++ b/src/lib/measurements/dense-intraday-hourly-rebuild.ts @@ -72,11 +72,11 @@ import { localDayWindow } from "@/lib/tz/local-day"; import { meanBucketValue } from "./consolidate-daily-mean"; import { DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, adoptOrMintHourlyRows, bucketRowsByLocalHour, hourlyStatsExternalId, } from "./dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "./dense-intraday-sources"; import { bucketSpan, enqueueRollupRecompute, diff --git a/src/lib/measurements/dense-intraday-retention.ts b/src/lib/measurements/dense-intraday-retention.ts index fd7a1babc..89abd6362 100644 --- a/src/lib/measurements/dense-intraday-retention.ts +++ b/src/lib/measurements/dense-intraday-retention.ts @@ -56,7 +56,6 @@ import { FOLD_TRANSACTION_OPTIONS, } from "@/lib/export/restore-lock"; import type { - MeasurementSource, MeasurementType, Prisma, PrismaClient, @@ -78,6 +77,11 @@ import { percentile } from "@/lib/insights/strain-score"; import type { PerSampleRow } from "./drain-per-sample-cumulative"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; import { localDayWindow } from "@/lib/tz/local-day"; +import { + DENSE_INTRADAY_FOLD_SOURCES, + DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE, + type DenseIntradayFoldSource, +} from "./dense-intraday-sources"; import { absorbIntoFold, foldConstituentHorizon, @@ -121,40 +125,6 @@ import { * they did against the raw stream's successor rows before. */ -/** - * The dense-tier types. These are the high-frequency intra-day signals that - * accumulate raw per-sample rows the destructive daily-mean drain never - * touches: - * - * - `HEART_RATE_VARIABILITY` + `PULSE` — the daytime shape the Stress - * engine reads (v1.10.0 / v1.18.11). - * - `OXYGEN_SATURATION` — Apple Watch's Blood-Oxygen app samples SpO2 - * periodically through the day and overnight; the readings map to - * `aggregation: "latest"` and sit in NEITHER `CUMULATIVE_HK_TYPES` nor - * `HIGH_FREQUENCY_MEAN_TYPES`, so every sample piled up raw forever. - * Folded here to hourly-mean rows with the daily min/max preserved on - * the rollup tier (the lowest sample is the overnight-desaturation - * nadir — clinically the meaningful figure, not the mean). - * - * Every member is deliberately EXEMPT from the destructive daily-mean drain - * (`HIGH_FREQUENCY_MEAN_TYPES`); the disjointness from that set is the - * load-bearing invariant a regression test pins — a type in both would be - * folded by two drains at once and corrupt the value. - * - * `RESPIRATORY_RATE` is deliberately NOT here: it is already a member of - * `HIGH_FREQUENCY_MEAN_TYPES`, so the nightly mean-consolidation already - * collapses it to a daily-mean row. Adding it here would double-fold it. - * - * One fold grain per tier: every member folds HOURLY. A per-type grain - * split would fork the write path and the rebuild for no reader benefit. - */ -export const DENSE_INTRADAY_RETENTION_TYPES: ReadonlySet = - new Set([ - "HEART_RATE_VARIABILITY", - "PULSE", - "OXYGEN_SATURATION", - ]); - /** * Retention bound (days). Per-sample dense-tier rows older than this window * fold to hourly-mean rows and the raw rows are deleted; rows inside the @@ -173,23 +143,6 @@ export const DENSE_INTRADAY_RETENTION_TYPES: ReadonlySet = */ export const DENSE_INTRADAY_RETENTION_DAYS = 90; -/** - * The sources whose raw dense samples fold (v1.42). Apple Health from the - * start; Health Connect because its importer writes the samples of the last - * 90 days raw, exactly like an Apple Health sync, and without a fold every - * new export would add another quarter of minute-by-minute heart rate that - * never ages into hourly means. Each source folds into its OWN `stats:` rows, - * so an hour two sources both covered keeps one mean per source, the same - * way the raw samples kept their source. - */ -export const DENSE_INTRADAY_FOLD_SOURCES = [ - "APPLE_HEALTH", - "HEALTH_CONNECT", -] as const satisfies readonly MeasurementSource[]; - -export type DenseIntradayFoldSource = - (typeof DENSE_INTRADAY_FOLD_SOURCES)[number]; - /** The stats externalId prefix marks an already-collapsed row. */ const DAILY_STATS_PREFIX = "stats:"; @@ -590,7 +543,7 @@ export async function runDenseIntradayRetention( await runConsolidation({ prismaClient, options: { ...options, cutoffHours }, - types: DENSE_INTRADAY_RETENTION_TYPES, + types: DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE[source], hkIdentifierForType, dailyStatsExternalId, statsPrefix: DAILY_STATS_PREFIX, @@ -1034,8 +987,8 @@ export async function runDenseIntradayRetention( }, }); - // One walk per source, Apple Health first so its behaviour and its order - // are exactly what they were before Health Connect folded too. A walk that + // One walk per source, in the map's order (Apple Health first, so its + // behaviour is what it was before other sources folded too). A walk that // stopped early stops the pass: the next run resumes where it left off. for (const source of DENSE_INTRADAY_FOLD_SOURCES) { const walk = await walkSource(source); diff --git a/src/lib/measurements/dense-intraday-sources.ts b/src/lib/measurements/dense-intraday-sources.ts new file mode 100644 index 000000000..2db64daa9 --- /dev/null +++ b/src/lib/measurements/dense-intraday-sources.ts @@ -0,0 +1,99 @@ +/** + * What the dense hourly fold (`dense-intraday-retention.ts`) takes: the + * dense-tier types, and for each source the types it stores one row per + * intraday sample of. The `folded_window` ingest guard (`folded-window.ts`) + * reads the same map, so a source's raw samples are refused at ingest for + * exactly the hours the fold has taken from it. + */ +import type { + MeasurementSource, + MeasurementType, +} from "@/generated/prisma/client"; + +/** + * The dense-tier types. These are the high-frequency intra-day signals that + * accumulate raw per-sample rows the destructive daily-mean drain never + * touches: + * + * - `HEART_RATE_VARIABILITY` + `PULSE` — the daytime shape the Stress + * engine reads. + * - `OXYGEN_SATURATION` — Apple Watch's Blood-Oxygen app samples SpO2 + * periodically through the day and overnight; the readings map to + * `aggregation: "latest"` and sit in NEITHER `CUMULATIVE_HK_TYPES` nor + * `HIGH_FREQUENCY_MEAN_TYPES`, so every sample piled up raw forever. + * Folded here to hourly-mean rows with the daily min/max preserved on + * the rollup tier (the lowest sample is the overnight-desaturation + * nadir — clinically the meaningful figure, not the mean). + * + * Every member is deliberately EXEMPT from the destructive daily-mean drain + * (`HIGH_FREQUENCY_MEAN_TYPES`); the disjointness from that set is the + * load-bearing invariant a regression test pins — a type in both would be + * folded by two drains at once and corrupt the value. + * + * `RESPIRATORY_RATE` is deliberately NOT here: it is already a member of + * `HIGH_FREQUENCY_MEAN_TYPES`, so the nightly mean-consolidation already + * collapses it to a daily-mean row. Adding it here would double-fold it. + * + * One fold grain per tier: every member folds HOURLY. A per-type grain + * split would fork the write path and the rebuild for no reader benefit. + */ +export const DENSE_INTRADAY_RETENTION_TYPES: ReadonlySet = + new Set([ + "HEART_RATE_VARIABILITY", + "PULSE", + "OXYGEN_SATURATION", + ]); + +/** + * The sources whose raw dense samples fold, and which dense types of each. + * A source belongs here only for a type it stores sample by sample through + * the day; a type it stores as one value a day (a nightly SpO2 average, a + * recovery's HRV) is no intraday stream, and folding that row would only + * move it to another hour. + * + * - Apple Health and Health Connect: every dense type. Both write the + * samples of an export raw, and without the fold every new export would + * add another quarter of minute-by-minute heart rate. + * - Google Health: heart rate, one row per sample from the `heart-rate` + * collection (SpO2 comes from the daily collection). A full-history + * backfill brought tens of millions of these onto one account (#1297). + * - Oura: heart rate, a reading every few minutes from the `heartrate` + * collection (SpO2 is the nightly average). + * + * Fitbit does not sync intraday heart rate, and WHOOP, Withings and Polar + * write dense types only as one value per recovery, night or spot check, so + * none of them is here. + * + * Each source folds into its OWN `stats:` rows, so an hour two sources both + * covered keeps one mean per source, the same way the raw samples kept their + * source. The walk runs in this order. + */ +export const DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE = { + APPLE_HEALTH: DENSE_INTRADAY_RETENTION_TYPES, + HEALTH_CONNECT: DENSE_INTRADAY_RETENTION_TYPES, + GOOGLE_HEALTH: new Set(["PULSE"]), + OURA: new Set(["PULSE"]), +} as const satisfies Partial< + Record> +>; + +export type DenseIntradayFoldSource = + keyof typeof DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE; + +/** Every source the fold walks, in walk order. */ +export const DENSE_INTRADAY_FOLD_SOURCES = Object.keys( + DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE, +) as readonly DenseIntradayFoldSource[]; + +/** Whether the fold takes `type` from `source`. */ +export function isDenseFoldedSample( + source: MeasurementSource, + type: MeasurementType, +): boolean { + const types = ( + DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE as Partial< + Record> + > + )[source]; + return types?.has(type) ?? false; +} diff --git a/src/lib/measurements/folded-window.ts b/src/lib/measurements/folded-window.ts index 9fa41a6e1..eb84dfb2c 100644 --- a/src/lib/measurements/folded-window.ts +++ b/src/lib/measurements/folded-window.ts @@ -47,12 +47,14 @@ * of tombstones does not use that reach; a tombstone it misses stays class B, * which keeps a row longer, never loses one. * - * Health Connect folds its raw heart-rate history the same way under its own - * source (the importer writes samples of the last 90 days raw, and the - * nightly pass folds them once they age out), so the hour rule covers both - * sources and a covering row is always looked up under the sample's own - * source. The day rule stays Apple Health only: the nightly mean - * consolidation reads no other source. + * Every source the dense fold walks folds under its own source, for the + * types it stores sample by sample (`DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE`): + * Health Connect from its importer, and Google Health and Oura from their + * syncs, whose backfills walk years of heart rate (#1297). So there is one + * hour rule per source, and a covering row is always looked up under the + * sample's own source. The provider syncs ask the same question before they + * write (`withoutFoldedSamples`). The day rule stays Apple Health only: the + * nightly mean consolidation reads no other source. */ import type { MeasurementSource, @@ -71,11 +73,14 @@ import { hourOfDayForUserTz, } from "@/lib/measurements/consolidation-tz"; import { - DENSE_INTRADAY_FOLD_SOURCES, DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, hourlyStatsExternalId, } from "@/lib/measurements/dense-intraday-retention"; +import { + DENSE_INTRADAY_FOLD_SOURCES, + DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE, + DENSE_INTRADAY_RETENTION_TYPES, +} from "@/lib/measurements/dense-intraday-sources"; /** The per-entry status reason the batch result carries for such a row. */ export type FoldedWindowReason = "folded_window"; @@ -88,33 +93,38 @@ const HOUR_MS = 3_600_000; * `reachMs` is how far from a sample the anchor of its window can sit in any * zone (see the module comment), for the timezone-change check. */ -export const FOLDED_WINDOW_RULES = [ - { +export const FOLDED_WINDOW_RULES: readonly FoldedWindowRule[] = [ + // One hour rule per source the dense fold walks, with that source's types. + ...DENSE_INTRADAY_FOLD_SOURCES.map((source): FoldedWindowRule => ({ window: "hour", - sources: DENSE_INTRADAY_FOLD_SOURCES, - types: DENSE_INTRADAY_RETENTION_TYPES, + sources: [source], + types: DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE[source], foldedAfterMs: DENSE_INTRADAY_RETENTION_DAYS * 24 * HOUR_MS, reachMs: HOUR_MS / 2, - }, + })), { window: "day", - sources: ["APPLE_HEALTH"] as const, + sources: ["APPLE_HEALTH"], types: HIGH_FREQUENCY_MEAN_TYPES, foldedAfterMs: CONSOLIDATION_GRACE_CUTOFF_HOURS * HOUR_MS, reachMs: 14 * HOUR_MS, }, -] as const; +]; -type FoldedWindowRule = (typeof FOLDED_WINDOW_RULES)[number]; +interface FoldedWindowRule { + window: "hour" | "day"; + sources: readonly MeasurementSource[]; + types: ReadonlySet; + foldedAfterMs: number; + reachMs: number; +} function ruleFor( row: Pick, ): FoldedWindowRule | null { return ( FOLDED_WINDOW_RULES.find( - (rule) => - (rule.sources as readonly MeasurementSource[]).includes(row.source) && - rule.types.has(row.type), + (rule) => rule.sources.includes(row.source) && rule.types.has(row.type), ) ?? null ); } @@ -409,6 +419,46 @@ export async function findFoldedWindowDuplicates( ); } +/** + * The rows of one provider write without the raw samples whose hour the + * dense fold has already taken from that source (#1297). + * + * The provider syncs write first-write-wins on `(user, type, source, + * externalId)`, and the fold deletes the samples it averages, so a backfill + * or a full sync that walks old history would find no row for a folded + * sample and insert it again beside the hourly mean that already accounts + * for it. Google Health would even re-key the hourly row: a sample that fell + * on the hour's anchor shares its instant, and the sync's natural-key rescue + * then overwrote the mean with the sample. Rows too young to have been folded + * cost no query, so the hourly syncs never ask. + */ +export async function withoutFoldedSamples< + T extends { type: string; externalId?: string | null; measuredAt: Date }, +>( + client: LookupClient, + userId: string, + source: MeasurementSource, + rows: readonly T[], + options: { now?: Date } = {}, +): Promise<{ kept: T[]; dropped: number }> { + const folded = await findFoldedWindowDuplicates( + client, + userId, + rows.map((row) => ({ + type: row.type as MeasurementType, + source, + externalId: row.externalId ?? null, + measuredAt: row.measuredAt, + })), + options, + ); + if (folded.size === 0) return { kept: [...rows], dropped: 0 }; + return { + kept: rows.filter((_, index) => !folded.has(index)), + dropped: folded.size, + }; +} + /** A tombstone as the classifier reads it. */ export interface TombstoneCandidateRow extends FoldedWindowCandidateRow { deletedAt: Date | null; diff --git a/src/lib/oura/sync-heart-rate.ts b/src/lib/oura/sync-heart-rate.ts index 9c0a9a1f4..f2b8ff4b0 100644 --- a/src/lib/oura/sync-heart-rate.ts +++ b/src/lib/oura/sync-heart-rate.ts @@ -11,14 +11,16 @@ * Writes are first-write-wins: a sample is final once it is in Oura's cloud, * and it is keyed on its own instant (`heartrate:`), so a re-read of the * same range is an insert that `skipDuplicates` drops. A row the person - * deleted keeps its tombstone; the re-read does not bring it back. Oura's - * samples go raw, like Google Health's and Fitbit's minute stream: the dense - * hourly fold covers only the export sources (`DENSE_INTRADAY_FOLD_SOURCES`). + * deleted keeps its tombstone; the re-read does not bring it back. Samples + * older than the retention window are folded into hourly means like Google + * Health's (`DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE`), and a re-read of a folded + * hour writes nothing (`withoutFoldedSamples`). */ import { prisma } from "@/lib/db"; import type { MeasurementType } from "@/generated/prisma/client"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; +import { withoutFoldedSamples } from "@/lib/measurements/folded-window"; import { collapseToTypeDayKeys } from "@/lib/rollups/measurement-rollups"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; import { forEachHeartRatePage } from "./client"; @@ -76,10 +78,17 @@ export async function writeOuraHeartRatePage( const m = mapHeartRateSample(s); if (m) mapped.push(m); } - const kept = dropImplausibleMeasurements("oura", mapped, (m) => ({ - type: m.type, - value: m.value, - })); + // A sample of an hour the dense fold already took is not written again: it + // was deleted with the fold, so `skipDuplicates` would not stop it. + const { kept } = await withoutFoldedSamples( + prisma, + userId, + "OURA", + dropImplausibleMeasurements("oura", mapped, (m) => ({ + type: m.type, + value: m.value, + })), + ); let inserted = 0; for (let i = 0; i < kept.length; i += OURA_HEART_RATE_WRITE_CHUNK) { const chunk = kept.slice(i, i + OURA_HEART_RATE_WRITE_CHUNK); diff --git a/tests/integration/provider-dense-fold.test.ts b/tests/integration/provider-dense-fold.test.ts new file mode 100644 index 000000000..d36c21a52 --- /dev/null +++ b/tests/integration/provider-dense-fold.test.ts @@ -0,0 +1,185 @@ +/** + * The dense hourly fold over the provider syncs that store heart rate one row + * per sample, and their ingest guard, against Postgres (#1297). + * + * - Google Health and Oura heart rate older than the retention window folds + * to hourly `stats:` rows under its own source, the raw rows go, and the + * DAY rollup keeps the figures it had from the raw samples. + * - A daily SpO2 value of the same sources is no intraday stream and stays. + * - Writing the same old samples again, as a backfill or a full sync does, + * inserts nothing and leaves every hourly mean as it was, including the + * hour whose anchor a sample fell on (Google Health's natural-key rescue + * used to re-key that row back into the sample). + * - Samples inside the window still write as before. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { upsertGoogleHealthMeasurements } from "@/lib/google-health/sync-core"; +import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; +import { + TouchedDays, + writeOuraHeartRatePage, +} from "@/lib/oura/sync-heart-rate"; + +import { makeUser } from "../kit/factories"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const DAY_MS = 86_400_000; +const MIN_MS = 60_000; + +beforeEach(async () => { + await truncateAllTables(prisma); +}); + +/** Noon UTC, `days` ago: every sample below stays inside that UTC day. */ +function dayStart(days: number): number { + const now = Date.now(); + return Math.floor(now / DAY_MS) * DAY_MS - days * DAY_MS; +} + +/** Samples of 10:00-11:59 UTC, one every 10 minutes, 10:30 included. */ +function samples(base: number): Array<{ at: Date; bpm: number }> { + const out: Array<{ at: Date; bpm: number }> = []; + for (let m = 0; m < 120; m += 10) { + out.push({ + at: new Date(base + 10 * 60 * MIN_MS + m * MIN_MS), + bpm: 50 + ((m * 7) % 45), + }); + } + return out; +} + +const gh = (s: { at: Date; bpm: number }) => ({ + type: "PULSE", + value: s.bpm, + unit: "bpm", + measuredAt: s.at, + externalId: `${s.at.getTime()}:hr`, +}); + +async function liveRows(userId: string, source: "GOOGLE_HEALTH" | "OURA") { + return prisma.measurement.findMany({ + where: { userId, source, deletedAt: null }, + select: { type: true, value: true, externalId: true, measuredAt: true }, + orderBy: [{ type: "asc" }, { measuredAt: "asc" }], + }); +} + +describe("dense fold of provider heart rate (#1297)", () => { + it("folds old Google Health heart rate to hourly means, and a re-sync writes none of it back", async () => { + const user = await makeUser({ timezone: "UTC" }); + const old = samples(dayStart(120)); + const recent = samples(dayStart(5)); + await upsertGoogleHealthMeasurements(user.id, old.map(gh)); + // A nightly SpO2 average: one value a day, never folded. + await prisma.measurement.create({ + data: { + userId: user.id, + type: "OXYGEN_SATURATION", + value: 96, + unit: "%", + source: "GOOGLE_HEALTH", + measuredAt: new Date(dayStart(120) + 7 * 3_600_000), + externalId: "2026-01-01:spo2", + }, + }); + const dayBefore = await prisma.measurementRollup.findFirstOrThrow({ + where: { userId: user.id, type: "PULSE", granularity: "DAY" }, + }); + + const summary = await runDenseIntradayRetention(prisma, { + userId: user.id, + log: () => {}, + }); + expect(summary.totals.perSampleRowsSoftDeleted).toBeGreaterThan(0); + + const folded = await liveRows(user.id, "GOOGLE_HEALTH"); + const pulse = folded.filter((r) => r.type === "PULSE"); + // Two hours, two hourly means; no raw sample of them is left. + expect(pulse).toHaveLength(2); + for (const row of pulse) { + expect(row.externalId).toMatch( + /^stats:HKQuantityTypeIdentifierHeartRate:\d{4}-\d{2}-\d{2}T1[01]$/, + ); + } + const firstHour = old.slice(0, 6).map((s) => s.bpm); + expect(pulse[0].value).toBeCloseTo( + firstHour.reduce((a, b) => a + b, 0) / firstHour.length, + 6, + ); + expect(folded.filter((r) => r.type === "OXYGEN_SATURATION")).toHaveLength( + 1, + ); + // The DAY bucket keeps the raw figures. + const dayAfter = await prisma.measurementRollup.findFirstOrThrow({ + where: { userId: user.id, type: "PULSE", granularity: "DAY" }, + }); + expect(dayAfter.count).toBe(dayBefore.count); + expect(dayAfter.minValue).toBe(dayBefore.minValue); + expect(dayAfter.maxValue).toBe(dayBefore.maxValue); + expect(dayAfter.mean).toBeCloseTo(dayBefore.mean, 9); + + // A backfill walks the same history again. + const again = await upsertGoogleHealthMeasurements( + user.id, + [...old, ...recent].map(gh), + ); + expect(again.imported).toBe(recent.length); + const afterResync = await liveRows(user.id, "GOOGLE_HEALTH"); + expect(afterResync.filter((r) => r.type === "PULSE")).toHaveLength( + 2 + recent.length, + ); + // The hourly means are untouched, the 10:30 one included. + expect( + afterResync.filter((r) => r.externalId?.startsWith("stats:")), + ).toEqual(pulse); + }); + + it("folds old Oura heart rate the same way, and a re-read writes none of it back", async () => { + const user = await makeUser({ timezone: "UTC" }); + const old = samples(dayStart(150)).map((s) => ({ + bpm: s.bpm, + timestamp: s.at.toISOString(), + source: "awake", + })); + expect( + await writeOuraHeartRatePage(user.id, old, new TouchedDays(), { + emitArrivals: false, + }), + ).toBe(old.length); + + await runDenseIntradayRetention(prisma, { + userId: user.id, + log: () => {}, + }); + const folded = await liveRows(user.id, "OURA"); + expect(folded).toHaveLength(2); + expect(folded.every((r) => r.externalId?.startsWith("stats:"))).toBe(true); + + expect( + await writeOuraHeartRatePage(user.id, old, new TouchedDays(), { + emitArrivals: false, + }), + ).toBe(0); + expect(await liveRows(user.id, "OURA")).toEqual(folded); + }); + + it("leaves heart rate inside the window raw", async () => { + const user = await makeUser({ timezone: "UTC" }); + const recent = samples(dayStart(10)); + await upsertGoogleHealthMeasurements(user.id, recent.map(gh)); + await runDenseIntradayRetention(prisma, { + userId: user.id, + log: () => {}, + }); + expect(await liveRows(user.id, "GOOGLE_HEALTH")).toHaveLength( + recent.length, + ); + }); +}); From bf0142a62672f672259868a1be09648e90fc1c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 29 Oct 2026 09:07:37 +0100 Subject: [PATCH 09/41] fix(ai): give background work the High room on a model that always reasons Background jobs ask for medium reasoning at most, and status cards ask for none. A model that cannot be switched down (GLM behind a vLLM gateway, for instance) ignores both, spends the whole room thinking and returns only reasoning, so the daily briefing and the status cards fail as reasoning_only and the chain pauses. The Coach on High works because it leaves 12k tokens to think in. The chain runner now marks every call of a job walk as unattended, in one place rather than at each call site. The gateway and local clients give such a call at least the High thinking room once its model has answered with reasoning alone. The room has to ride on the #1267 repeat with none: one call may send two requests, and a once-a-day job meets an expired lesson every morning, so a room that waited for the unswitchable lesson would never reach the first briefing of the day. It is a ceiling, not a charge: a model that does stop at none never touches it. Calls a person is reading, the Coach included, keep the room of their own level, and the requested effort is unchanged. The room calculation both clients shared is now one function. Reported by @Tio61 in #1276. --- .../gateway-unswitchable-reasoning.test.ts | 95 ++++++++++++++++++- .../provider-runner-background-bench.test.ts | 26 +++++ src/lib/ai/local-client.ts | 12 +-- src/lib/ai/openai-client.ts | 17 +--- src/lib/ai/provider-runner.ts | 11 ++- src/lib/ai/reasoning/answerless.ts | 48 ++++++++++ src/lib/ai/types.ts | 8 ++ 7 files changed, 191 insertions(+), 26 deletions(-) diff --git a/src/lib/ai/__tests__/gateway-unswitchable-reasoning.test.ts b/src/lib/ai/__tests__/gateway-unswitchable-reasoning.test.ts index 8e415bd80..329363eab 100644 --- a/src/lib/ai/__tests__/gateway-unswitchable-reasoning.test.ts +++ b/src/lib/ai/__tests__/gateway-unswitchable-reasoning.test.ts @@ -45,16 +45,22 @@ interface SeenRequest { * `always`: the thought fills the ceiling whatever the effort (gpt-oss, GLM). * `answers-unreasoned`: the same, except asked with `none` it answers * (Gemma 4 on Ollama). + * `answers-with-room`: ignores the effort and answers once the ceiling leaves + * it the `high` thinking room (GLM 5.3 behind a gateway, #1276). */ -let mode: "always" | "answers-unreasoned" = "always"; +let mode: "always" | "answers-unreasoned" | "answers-with-room" = "always"; let server: http.Server; let origin: string; let seen: SeenRequest[] = []; const ANSWER = JSON.stringify({ summary: "Steady." }); -function reply(effort: unknown) { - const answers = mode === "answers-unreasoned" && effort === "none"; +function reply(effort: unknown, maxTokens: unknown) { + const answers = + (mode === "answers-unreasoned" && effort === "none") || + (mode === "answers-with-room" && + typeof maxTokens === "number" && + maxTokens >= 12_000); return { id: "chatcmpl-1279", object: "chat.completion", @@ -87,7 +93,14 @@ beforeAll(async () => { maxTokens: body.max_tokens ?? body.max_completion_tokens, }); res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify(reply(body.reasoning_effort))); + res.end( + JSON.stringify( + reply( + body.reasoning_effort, + body.max_tokens ?? body.max_completion_tokens, + ), + ), + ); }); }); await new Promise((r) => server.listen(0, "127.0.0.1", r)); @@ -210,3 +223,77 @@ describe.each([ }); }, ); + +/** A daily briefing call as the chain runner hands it to a client. */ +function briefingParams(unattended: boolean): CompletionParams { + return { + ...singleUserTurn({ + system: "You write the daily briefing.", + user: "Snapshot: …", + maxTokens: 300, + timeoutMs: 30_000, + responseFormat: "json", + }), + reasoning: { effort: "medium", summaries: false }, + ...(unattended ? { unattended: true } : {}), + }; +} + +describe.each([ + ["the OpenAI-compatible gateway", gateway], + ["the Local client", local], +] as const)( + "#1276 — %s, background work for a model that cannot be switched down", + (_, make) => { + it("gives the #1267 repeat the High room, and answers on the first call", async () => { + mode = "answers-with-room"; + const result = await make().generateCompletion(briefingParams(true)); + + expect(result.content).toBe(ANSWER); + // Two requests, the call's budget: medium at its own room, then the + // repeat with `none` and the High room the reply showed it needs. + expect(seen).toEqual([ + { reasoningEffort: "medium", maxTokens: 4396 }, + { reasoningEffort: "none", maxTokens: 12_300 }, + ]); + // Every later background call starts with the High room. + seen = []; + await make().generateCompletion(briefingParams(true)); + expect(seen).toEqual([{ reasoningEffort: "none", maxTokens: 12_300 }]); + }); + + it("gives a status card, which asks no level, the same room", async () => { + mode = "answers-with-room"; + const result = await make().generateCompletion({ + ...statusParams(), + unattended: true, + }); + expect(result.content).toBe(ANSWER); + expect(seen.map((s) => s.maxTokens)).toEqual([300, 12_300]); + }); + + it("keeps the High room once even `none` is learned useless, and stays at one request", async () => { + const err = (await make() + .generateCompletion(briefingParams(true)) + .catch((e) => e)) as Record; + expect(err.kind).toBe("reasoning_only"); + expect(seen.map((s) => s.maxTokens)).toEqual([4396, 12_300]); + seen = []; + await make() + .generateCompletion(briefingParams(true)) + .catch(() => {}); + expect(seen).toEqual([{ reasoningEffort: "medium", maxTokens: 12_300 }]); + }); + + it("leaves a call a person is reading at the room of its level", async () => { + await make() + .generateCompletion(briefingParams(false)) + .catch(() => {}); + seen = []; + await make() + .generateCompletion(briefingParams(false)) + .catch(() => {}); + expect(seen).toEqual([{ reasoningEffort: "medium", maxTokens: 4396 }]); + }); + }, +); diff --git a/src/lib/ai/__tests__/provider-runner-background-bench.test.ts b/src/lib/ai/__tests__/provider-runner-background-bench.test.ts index e17e9c558..c7efb8110 100644 --- a/src/lib/ai/__tests__/provider-runner-background-bench.test.ts +++ b/src/lib/ai/__tests__/provider-runner-background-bench.test.ts @@ -272,3 +272,29 @@ describe("#1279: background work and a benched provider", () => { expect((err as AllProvidersFailedError).tokensSpent).toBe(5400); }); }); + +describe("#1276 — the runner marks background calls", () => { + class Recorder implements AIProvider { + readonly type = "openai-compatible" as const; + seen: CompletionParams[] = []; + async generateCompletion(p: CompletionParams): Promise { + this.seen.push(p); + return { content: "{}", tokensUsed: 1 } as CompletionResult; + } + } + + it.each([ + ["job", true], + ["coach", undefined], + ] as const)("a %s walk carries unattended: %s", async (surface, expected) => { + const instance = new Recorder(); + await walk( + surface, + `u-unattended-${surface}`, + "openai-compatible", + instance, + createInMemoryProviderHealthLedger(), + ); + expect(instance.seen[0]?.unattended).toBe(expected); + }); +}); diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index 6af86a14c..4fbe52d6b 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -27,11 +27,10 @@ import { noteReasoningSeen, refusesReasoningField, } from "./reasoning/dialect-cache"; -import { uncontrolledThinkingRoom } from "./reasoning/levels"; import { askAgainUnreasoned, knownReasoner, - reasonsUnasked, + uncontrolledCallThinkingRoom, repeatUnreasoned, } from "./reasoning/answerless"; import { @@ -149,11 +148,10 @@ export class LocalOpenAICompatibleClient implements AIProvider { // `none` sent for Off or after the #1267 lesson must not leave it the // bare answer budget to think in. A call without a level gets the `low` // room once the model has shown it reasons unasked. - const thinkingTokens = params.reasoning - ? uncontrolledThinkingRoom(params.reasoning.effort) - : reasonsUnasked(this.reasoningDialectKey()) - ? uncontrolledThinkingRoom("off") - : 0; + const thinkingTokens = uncontrolledCallThinkingRoom( + this.reasoningDialectKey(), + params, + ); const body = JSON.stringify({ model: this.config.model, messages, diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index feb6efd5c..1e144ac2c 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -21,15 +21,12 @@ import { import { askAgainUnreasoned, knownReasoner, - reasonsUnasked, + uncontrolledCallThinkingRoom, repeatUnreasoned, } from "./reasoning/answerless"; import { RequestHops } from "./request-hops"; import { learnFromRefusal } from "./wire-refusal"; -import { - REASONING_THINKING_BUDGET, - uncontrolledThinkingRoom, -} from "./reasoning/levels"; +import { REASONING_THINKING_BUDGET } from "./reasoning/levels"; import { answerOf, isOpenRouterEndpoint, @@ -573,19 +570,15 @@ export class OpenAIClient implements AIProvider { * included, and a `none` sent for the person's Off or after the #1267 * lesson must not leave it the bare answer budget to think in. Without a * level of its own, a call gets the `low` room once the model has shown it - * reasons unasked. + * reasons unasked, and background work the `high` room once the model has + * answered with reasoning alone (`unattendedThinkingRoom`). */ private thinkingRoom( params: CompletionParams, wire: OpenAIReasoningWire, ): number { if (this.isGateway) { - if (params.reasoning) { - return uncontrolledThinkingRoom(params.reasoning.effort); - } - return reasonsUnasked(this.reasoningDialectKey()) - ? uncontrolledThinkingRoom("off") - : 0; + return uncontrolledCallThinkingRoom(this.reasoningDialectKey(), params); } if (wire.kind === "none" || wire.effort === "none") return 0; const level = levelOfWireEffort(wire.effort); diff --git a/src/lib/ai/provider-runner.ts b/src/lib/ai/provider-runner.ts index 684853b5c..2aafb360d 100644 --- a/src/lib/ai/provider-runner.ts +++ b/src/lib/ai/provider-runner.ts @@ -743,14 +743,17 @@ export async function runStreamingRawCompletionWithFallback( onDelta: (delta: string) => void; }, ): Promise { + // #1276 — a job walk is background work; stamped here, not per call site. + const params: CompletionParams = + args.surface === "job" ? { ...args.params, unattended: true } : args.params; return runRawChain( args, // Prefer the provider's streaming wire; fall back to the buffered call for // providers (cloud clients today) that do not implement streaming. (candidate) => candidate.instance.generateCompletionStream - ? candidate.instance.generateCompletionStream(args.params, args.onDelta) - : candidate.instance.generateCompletion(args.params), + ? candidate.instance.generateCompletionStream(params, args.onDelta) + : candidate.instance.generateCompletion(params), (candidate) => ({ ai_chain_streamed: Boolean(candidate.instance.generateCompletionStream), }), @@ -789,7 +792,9 @@ export interface RunRawWithFallbackResult { export async function runRawCompletionWithFallback( args: ChainWalkArgs, ): Promise { + const params: CompletionParams = + args.surface === "job" ? { ...args.params, unattended: true } : args.params; return runRawChain(args, (candidate) => - candidate.instance.generateCompletion(args.params), + candidate.instance.generateCompletion(params), ); } diff --git a/src/lib/ai/reasoning/answerless.ts b/src/lib/ai/reasoning/answerless.ts index bcf2a2e18..492a94e3f 100644 --- a/src/lib/ai/reasoning/answerless.ts +++ b/src/lib/ai/reasoning/answerless.ts @@ -36,6 +36,11 @@ import { rememberReasoningDialect, type ReasoningDialectKey, } from "./dialect-cache"; +import { + REASONING_THINKING_BUDGET, + uncontrolledThinkingRoom, + type ReasoningLevel, +} from "./levels"; /** * What the client does next with an answerless reply. @@ -137,6 +142,49 @@ export function reasonsUnasked(key: ReasoningDialectKey): boolean { ); } +/** + * #1276 — the thinking room of an unattended call (`CompletionParams.unattended`) + * to a model that has answered with reasoning alone (`reasonsUnasked`): at + * least the `high` room the Coach gets on High. Background work asks for + * `medium` at most, and a model that cannot be switched down (GLM behind + * vLLM) spends all of it thinking and answers nothing, the daily briefing's + * `reasoning_only`. The room applies from the #1267 repeat with `none` on: + * one call may send two requests (`request-hops.ts`), and a once-a-day job + * meets an expired lesson every morning, so the room has to ride on the + * repeat or the first job of the day never gets it. A ceiling, not a charge: + * a model that does stop at `none` never touches it, one that does not + * thinks as long as it does either way and reaches the answer. Every call a + * person is reading keeps `room`. + */ +export function unattendedThinkingRoom( + key: ReasoningDialectKey, + room: number, + unattended: boolean | undefined, +): number { + return unattended && reasonsUnasked(key) + ? Math.max(room, REASONING_THINKING_BUDGET.high) + : room; +} + +/** + * #1276 — the thinking room of one call on a wire whose reasoning switch the + * client cannot verify (a gateway, a local server): by the call's level, not + * by what the wire says, the `low` room for a call without a level once the + * model has shown it reasons unasked, and the `high` room for background work + * once it has answered with reasoning alone (`unattendedThinkingRoom`). + */ +export function uncontrolledCallThinkingRoom( + key: ReasoningDialectKey, + params: { reasoning?: { effort: ReasoningLevel }; unattended?: boolean }, +): number { + const room = params.reasoning + ? uncontrolledThinkingRoom(params.reasoning.effort) + : reasonsUnasked(key) + ? uncontrolledThinkingRoom("off") + : 0; + return unattendedThinkingRoom(key, room, params.unattended); +} + /** * #1276 — true when the model has shown it reasons, for a request that did * (`asked`) or did not ask it to: a lesson that says it reasons unasked, or a diff --git a/src/lib/ai/types.ts b/src/lib/ai/types.ts index 4dcbdc1bf..f4a78a27a 100644 --- a/src/lib/ai/types.ts +++ b/src/lib/ai/types.ts @@ -383,6 +383,14 @@ export interface CompletionParams { * client sends exactly what it sent before the field existed. */ reasoning?: { effort: ReasoningLevel; summaries: boolean }; + /** + * #1276 — no person reads this reply as it streams: the call runs as + * background work. Set by the chain runner for every `"job"` walk, never by + * a call site. The gateway and local clients give such a call the `high` + * thinking room once its model has answered with reasoning alone + * (`unattendedThinkingRoom`); nothing else reads it. + */ + unattended?: boolean; /** * v1.41.2 — a stable key for calls that resend the same prompt prefix (the * rounds of one Coach turn, the turns of one conversation). Providers that From acf58923f7d15ba10fa926834943c682d62ff3b2 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:42:54 +0000 Subject: [PATCH 10/41] test(ai): pin the step-up when gpt-6.1-sol refuses effort none gpt-6.1-sol, a rung on the default list, answers reasoning "off" (effort none) with 400 unsupported_value and lists low as its lowest effort (seen 2026-10-24). The client must stay on the slug, retry with low and remember it, not walk on as if the model were retired. --- src/lib/ai/__tests__/codex-reasoning.test.ts | 43 ++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/lib/ai/__tests__/codex-reasoning.test.ts b/src/lib/ai/__tests__/codex-reasoning.test.ts index ac022be44..a4609130e 100644 --- a/src/lib/ai/__tests__/codex-reasoning.test.ts +++ b/src/lib/ai/__tests__/codex-reasoning.test.ts @@ -137,6 +137,49 @@ describe("Codex reasoning request", () => { expect(body.include).toEqual([]); expect(result.reasoning?.downgradedTo).toBeUndefined(); }); + + it("on a slug that refuses none, steps up to its lowest effort and stays on the slug", async () => { + // gpt-6.1-sol, a rung on the default list, refuses "none" (seen 2026-10-24). + // That refusal is about the effort, not the model: no walk to the next slug. + const refusal = JSON.stringify({ + error: { + message: + "Unsupported value: 'none' is not supported with the 'gpt-6.1-sol' model. Supported values are: 'low', 'medium', 'high', 'xhigh', and 'max'.", + type: "invalid_request_error", + param: "reasoning.effort", + code: "unsupported_value", + }, + }); + const mock = vi + .fn() + .mockResolvedValueOnce(errorResponse(400, refusal)) + .mockImplementation(async () => + sseResponse(fixture("codex-after-tool.sse")), + ); + vi.stubGlobal("fetch", mock); + const c = new CodexClient({ + accessToken: "test-token", + accountId: "acct-test", + onTokenRefresh: vi + .fn() + .mockResolvedValue({ accessToken: "x", accountId: "acct-test" }), + slugChain: ["gpt-6.1-sol", "gpt-6-sol"], + }); + + await c.generateCompletion( + params({ reasoning: { effort: "off", summaries: false } }), + ); + expect(sentBody(mock, 0).reasoning).toEqual({ effort: "none" }); + expect(sentBody(mock, 1).model).toBe("gpt-6.1-sol"); + expect(sentBody(mock, 1).reasoning).toEqual({ effort: "low" }); + + await c.generateCompletion( + params({ reasoning: { effort: "off", summaries: false } }), + ); + expect(mock).toHaveBeenCalledTimes(3); + expect(sentBody(mock, 2).model).toBe("gpt-6.1-sol"); + expect(sentBody(mock, 2).reasoning).toEqual({ effort: "low" }); + }); }); describe("Codex reasoning stream", () => { From 69d46f0add7ff20a4ecd2fad9f6844f667e78d77 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:09:02 +0100 Subject: [PATCH 11/41] test(ai): pin the walk past a retired model on the default Codex list OpenAI retired the 5.x line from Codex for ChatGPT sign-in in steps: GPT-5.2 and GPT-5.3-Codex on 2026-05-26, GPT-5.4 and GPT-5.4 mini on 2026-08-31, and GPT-5.5 on 2026-10-14. The backend answers a retired model with 400 model_not_found and "not supported when using Codex with a ChatGPT account". Pin that exact refusal as a reason to try the next slug, pin that an account refused the head of the default list walks on to the next rung and caches it, and keep the long-retired models off the default list. --- .../ai/__tests__/codex-slug-fallback.test.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/lib/ai/__tests__/codex-slug-fallback.test.ts b/src/lib/ai/__tests__/codex-slug-fallback.test.ts index 860cbfd39..6ac75d8a3 100644 --- a/src/lib/ai/__tests__/codex-slug-fallback.test.ts +++ b/src/lib/ai/__tests__/codex-slug-fallback.test.ts @@ -91,6 +91,48 @@ describe("Codex slug fallback chain", () => { clearCodexSlugCache(); }); + it("an account without the head of the default list walks on to the next rung", async () => { + const origModel = process.env.CODEX_MODEL; + const origChain = process.env.CODEX_MODEL_FALLBACK_CHAIN; + delete process.env.CODEX_MODEL; + delete process.env.CODEX_MODEL_FALLBACK_CHAIN; + try { + // The exact refusal a ChatGPT sign-in gets for a retired model. + const mockFetch = vi + .fn() + .mockResolvedValueOnce( + rejectionResponse( + 400, + '{"detail":"The \'gpt-5.6-terra\' model is not supported when using Codex with a ChatGPT account.","error":{"type":"invalid_request_error","code":"model_not_found"}}', + ), + ) + .mockResolvedValueOnce(sseSuccessResponse()); + vi.stubGlobal("fetch", mockFetch); + + const client = new CodexClient({ + accessToken: "test", + accountId: "acct", + onTokenRefresh, + }); + await client.generateCompletion( + singleUserTurn({ system: "s", user: "u" }), + ); + + expect(client.getLastDiagnostics()?.attempted).toEqual([ + "gpt-5.6-terra", + "gpt-6-sol", + ]); + expect(client.getLastDiagnostics()?.workingSlug).toBe("gpt-6-sol"); + expect(getCachedCodexSlug()).toBe("gpt-6-sol"); + } finally { + if (origModel === undefined) delete process.env.CODEX_MODEL; + else process.env.CODEX_MODEL = origModel; + if (origChain === undefined) + delete process.env.CODEX_MODEL_FALLBACK_CHAIN; + else process.env.CODEX_MODEL_FALLBACK_CHAIN = origChain; + } + }); + it("first slug succeeds — uses head-of-chain and caches it", async () => { const mockFetch = vi.fn().mockResolvedValue(sseSuccessResponse()); vi.stubGlobal("fetch", mockFetch); @@ -340,6 +382,14 @@ describe("Codex slug fallback chain", () => { ).toBe(true); expect(helper(400, '{"error":{"code":"model_not_found"}}')).toBe(true); expect(helper(400, "model gpt-5 does not exist")).toBe(true); + // The refusal a ChatGPT sign-in got for gpt-5.5 after its retirement + // (seen 2026-10-24). + expect( + helper( + 400, + '{"detail":"The \'gpt-5.5\' model is not supported when using Codex with a ChatGPT account.","error":{"type":"invalid_request_error","code":"model_not_found"}}', + ), + ).toBe(true); expect(helper(404, "anything")).toBe(true); expect(helper(500, "internal error")).toBe(false); expect(helper(429, "rate limited")).toBe(false); @@ -360,6 +410,17 @@ describe("Codex slug fallback chain", () => { "gpt-5.6-sol", "gpt-5.5", ]); + // OpenAI retired these from Codex for ChatGPT sign-in: 5.2 and 5.3-Codex + // on 2026-05-26, 5.4 and 5.4 mini on 2026-08-31. A retired rung costs + // every account that reaches it a refused request. + for (const retired of [ + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.3-codex", + "gpt-5.2", + ]) { + expect(__test.DEFAULT_SLUG_FALLBACK_CHAIN).not.toContain(retired); + } }); it("DEFAULT_SLUG_FALLBACK_CHAIN holds only slugs every family gate recognises", () => { From 0b32b567c09a2a54b003e33c3cb65dedcb35abf1 Mon Sep 17 00:00:00 2001 From: haritowa <6107211+haritowa@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:09:23 +0100 Subject: [PATCH 12/41] docs(ai): record the live check of the GPT-6 rungs and the retired 5.x line From a HealthLog container with its stored ChatGPT sign-in, gpt-6.1-sol, gpt-6-sol and gpt-6-luna each answered text and read a lab-report image, and the account's own model list offered them with image input. gpt-6.1-sol refuses effort none and names low as its lowest. The spec said the gpt-6 rungs had never been probed live, and still listed gpt-5.4, gpt-5.4 mini, gpt-5.3-codex and gpt-5.2 as accepted, all of which OpenAI has retired for ChatGPT sign-in, with the exact refusal the backend sends. Also note why a fixed chain goes stale: once GPT-5.5 retired, every model on the chain shipped at the time was retired with it. --- docs/codex-protocol-spec.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/codex-protocol-spec.md b/docs/codex-protocol-spec.md index de74b0198..67b45d4bb 100644 --- a/docs/codex-protocol-spec.md +++ b/docs/codex-protocol-spec.md @@ -631,10 +631,10 @@ Confirmed-rejected slugs (despite the source-tree references in the table above) Confirmed-accepted slugs on a ChatGPT Plus subscription (per `available_in_plans` in `models.json`): - `gpt-5.6-terra` — HealthLog's default (first chain entry) as of 2026-10-24; the backend rejected `gpt-5.5` and everything below it that day. `gpt-5.6-luna` and `gpt-5.6-sol` are its siblings on the same line -- `gpt-6-sol`, `gpt-6.1-sol`, `gpt-6-luna`, `gpt-6-astra` — the gpt-6 line, listed in `models.json` since 2026-09-22 (`gpt-6.1-sol` the catalog default since 2026-09-29). The catalog names `gpt-6-sol` as the upgrade of `gpt-5.6-terra` and `gpt-5.6-sol`, and `gpt-6-luna` as that of `gpt-5.6-luna`. Every gpt-6 slug lists `input_modalities: ["text", "image"]` and the same reasoning presets as the gpt-5.6 line (`low` to `max`, sol and astra also `ultra`). The presets omit `none`, as they do for `gpt-5.5`, which a live probe on 2026-10-06 showed accepts it; HealthLog sends `none` for "off" on the gpt-6 line too and learns a refusal per slug. The gpt-6 slugs were not probed against a live backend when they joined the chain -- `gpt-5.5` — the default from 2026-06 until 2026-10-24 -- `gpt-5.4` / `gpt-5.4-mini` / `gpt-5.2` — accepted per plan -- `gpt-5.3-codex` — was the default (verified 2026-05-09); the backend rotated off the `-codex` line on 2026-06-02 and this slug is now only accepted intermittently, so it sits on a late chain rung +- `gpt-6-sol`, `gpt-6.1-sol`, `gpt-6-luna`, `gpt-6-astra` — the gpt-6 line, listed in `models.json` since 2026-09-22 (`gpt-6.1-sol` the catalog default since 2026-09-29). The catalog names `gpt-6-sol` as the upgrade of `gpt-5.6-terra` and `gpt-5.6-sol`, and `gpt-6-luna` as that of `gpt-5.6-luna`. Every gpt-6 slug lists `input_modalities: ["text", "image"]` and the same reasoning presets as the gpt-5.6 line (`low` to `max`, sol and astra also `ultra`). The presets omit `none`, as they do for `gpt-5.5`, which a live probe on 2026-10-06 showed accepts it; HealthLog sends `none` for "off" on the gpt-6 line too and learns a refusal per slug. A live check on 2026-10-24 found the account's own list (`GET /backend-api/codex/models?client_version=…`) offering `gpt-6.1-sol`, `gpt-6-astra`, `gpt-6-sol`, `gpt-6-luna` and the `gpt-5.6` line, all with `input_modalities: ["text", "image"]`. From a HealthLog container with its stored sign-in, `gpt-6.1-sol`, `gpt-6-sol` and `gpt-6-luna` each answered text and read a lab-report image correctly. `gpt-6.1-sol` refuses `reasoning.effort: "none"` and names `low` as its lowest effort, which the client learns and steps up to on the same slug; the other two accept `none` +- `gpt-5.5` — the default from 2026-06 until 2026-10-24, kept as the last chain rung while plans differ + +Retired for ChatGPT sign-in (OpenAI's Codex models page): `gpt-5.2` and `gpt-5.3-codex` on 2026-05-26, `gpt-5.4` and `gpt-5.4-mini` on 2026-08-31, `gpt-5.5` on 2026-10-14 at 19:00 UTC. The backend answers each of them with 400 `model_not_found`, "The '' model is not supported when using Codex with a ChatGPT account.", which the client reads as a reason to try the next slug. The `codex-rs/tui/src/model_migration.rs` references to `gpt-5-codex` etc. are _migration prompts_ shown to existing users — they are NOT the slugs sent on the wire after migration. The wire slugs come from `models.json`. Always cross-check `models.json` (`gh api repos/openai/codex/contents/codex-rs/models-manager/models.json`) before picking a default. @@ -662,6 +662,8 @@ Every default rung must be a model family the reasoning, sampling and vision gat The shipped constant in `src/lib/ai/codex-slug-chain.ts` is authoritative; when the upstream rotates again, that file moves first and this listing follows. +A fixed chain goes stale. From 2026-10-14, when GPT-5.5 retired, every model on the chain HealthLog shipped then was retired, and every Codex request answered 503 "all configured Codex slugs were rejected" until the chain was updated or `CODEX_MODEL` was set by hand. Reading the account's live list (`GET /backend-api/codex/models`, as the Codex CLI does) would avoid this. + The chain is overridable via the env var `CODEX_MODEL_FALLBACK_CHAIN` (comma-separated). The first env-var-named slug supersedes `CODEX_MODEL`; if both are set, `CODEX_MODEL` is folded into position 0 and de-duplicated. **Trigger condition** (when to walk to the next slug): From ef141f457b98323acba51b54aebedb9b80f1e83c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 29 Oct 2026 09:09:35 +0100 Subject: [PATCH 13/41] test(i18n): lower the version-stamp baseline for the provider The locale effect and its dated comment left the i18n provider with the previous commit, so the ratchet records one stamp fewer. --- src/__tests__/baselines/version-stamp-comments.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index f319e154c..c44b1da77 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", - "count": 2897, - "total": 9236, + "count": 2896, + "total": 9235, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -2067,7 +2067,6 @@ "lib/google-health/sync-metrics.ts": 1, "lib/i18n/__tests__/relative-time-updated.test.ts": 2, "lib/i18n/config.ts": 1, - "lib/i18n/context.tsx": 1, "lib/i18n/relative-time.ts": 4, "lib/i18n/shared-resolve.ts": 1, "lib/i18n/source-labels.ts": 1, From 107db1bb55e51ef44d29b1dc13d08c7a74e7d968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 29 Oct 2026 09:10:26 +0100 Subject: [PATCH 14/41] feat(admin): let an admin delete another account through the grace window There was no way for an admin to remove someone else's account; the users list only offered the undo for a deletion the account itself or a Guardian had asked for. DELETE /api/admin/users/{id} takes the account's own deletion path, now shared as requestAccountDeletion: with a grace window the account is locked at once (credentials revoked, record untouched) and erased by the purge job when the window ends, and the existing undo gives it back until then; with ACCOUNT_DELETION_GRACE_DAYS=0 it is erased in the request. The route is cookie-only behind requireAdmin and the recent-proof gate, takes the typed DELETE_ACCOUNT confirmation, and refuses the caller's own account, a managed profile (its Guardians delete it), an account already pending deletion, the last admin who can sign in and the last Guardian of a managed profile, the last three decided under their locks inside the transaction. Both the admin and the account get an audit row. The users list says who asked for a pending deletion, now including an admin, and flags managed profiles. Each other account's row carries a delete button whose confirmation names the purge date, or says the erasure is immediate when the server keeps no window. --- docs/api/openapi.yaml | 5 +- messages/de.json | 11 +- messages/en.json | 11 +- messages/es.json | 11 +- messages/fr.json | 11 +- messages/it.json | 11 +- messages/ko.json | 11 +- messages/nl.json | 11 +- messages/pl.json | 11 +- messages/pt-BR.json | 11 +- .../destructive-control-guard.test.ts | 11 + .../managed-profile-contract.test.ts | 18 +- .../openapi-route-coverage-guard.test.ts | 2 +- .../recent-proof-surface-guard.test.ts | 1 + .../step-up-web-caller-guard.test.ts | 1 + src/app/api/admin/users/[id]/route.ts | 141 +++++++++- .../api/admin/users/__tests__/route.test.ts | 2 + src/app/api/admin/users/route.ts | 12 +- src/app/api/settings/account/route.ts | 74 ++--- .../user-management-responsive.test.tsx | 17 ++ src/components/admin/_shared.tsx | 10 +- src/components/admin/delete-user-control.tsx | 136 +++++++++ .../admin/pending-deletion-controls.tsx | 16 +- .../admin/user-management-section.tsx | 18 +- src/lib/account-deletion/request.ts | 81 ++++++ src/lib/openapi/error-codes.ts | 7 +- src/lib/validations/admin.ts | 8 + tests/integration/admin-user-deletion.test.ts | 264 ++++++++++++++++++ 28 files changed, 828 insertions(+), 95 deletions(-) create mode 100644 src/components/admin/delete-user-control.tsx create mode 100644 src/lib/account-deletion/request.ts create mode 100644 tests/integration/admin-user-deletion.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 7eae195e0..39ecc4374 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -38543,8 +38543,9 @@ components: `upstream_error`, `timeout`, `connection_failed` and the per-provider additions) are enumerated in each `/test` operation instead, where the differences can be stated; and a 401 raised by a route checking a credential of its own carries the generic `http.unauthenticated`. account — - account.deletion.no_guardian, account.deletion.not_pending; activity — activity.calendar.future_date, - activity.calendar.too_old; ai — ai.budget.exceeded, ai.provider.none, ai.record.notPermitted, + account.deletion.already_pending, account.deletion.no_guardian, account.deletion.not_pending; activity — + activity.calendar.future_date, activity.calendar.too_old; admin — admin.users.delete.managed_profile, + admin.users.delete.self; ai — ai.budget.exceeded, ai.provider.none, ai.record.notPermitted, ai.unavailable, ai.usage.aboveCeiling, ai.usage.defaultAboveCeiling, ai.usage.invalid, ai.usage.levelTooLow, ai.usage.rateLimited; ai_provider — ai_provider.invalid, ai_provider.no_fields; aiRuns — aiRuns.failed, aiRuns.notFound, aiRuns.readerBusy, aiRuns.timedOut, aiRuns.workerUnavailable; diff --git a/messages/de.json b/messages/de.json index 09da70eff..cfd221364 100644 --- a/messages/de.json +++ b/messages/de.json @@ -8110,7 +8110,16 @@ "undoDeletionBody": "{name} hat die Löschung dieses Kontos verlangt; es wird gelöscht, wenn die Frist endet. Rückgängig machen behält das Konto und alles darin so, wie es war. {name} kann sich wieder anmelden, aber API-Tokens, Freigabe-Links und vertrauenswürdige Geräte bleiben widerrufen und müssen neu eingerichtet werden.", "undoDeletionBodyGuardian": "Eine betreuende Person hat das verwaltete Profil {name} gelöscht; es wird gelöscht, wenn die Frist endet. Rückgängig machen behält das Profil und alles darin so, wie es war, und die betreuenden Personen erreichen es wieder. Seine Freigabe-Links und Tokens bleiben widerrufen und müssen neu eingerichtet werden.", "undoDeletionDone": "Löschung von {name} rückgängig gemacht", - "undoDeletionFailed": "Löschung konnte nicht rückgängig gemacht werden" + "undoDeletionFailed": "Löschung konnte nicht rückgängig gemacht werden", + "undoDeletionBodyAdmin": "Ein Admin hat das Konto {name} gelöscht; es wird gelöscht, wenn die Frist endet. Rückgängig machen behält das Konto und alles darin so, wie es war. {name} kann sich wieder anmelden, aber API-Tokens, Freigabe-Links und vertrauenswürdige Geräte bleiben widerrufen und müssen neu eingerichtet werden.", + "deleteUser": "Konto löschen", + "deleteUserTitle": "Das Konto {name} löschen?", + "deleteUserBodyGrace": "{name} wird sofort überall abgemeldet und kann sich nicht mehr anmelden. Das Konto und alles darin wird am {date} gelöscht. Bis dahin kannst du die Löschung hier rückgängig machen.", + "deleteUserBodyNow": "{name} und alles im Konto wird sofort gelöscht. Dieser Server hat keine Frist eingestellt, die Löschung lässt sich also nicht rückgängig machen.", + "deleteUserConfirm": "Konto löschen", + "deleteUserScheduled": "Das Konto {name} ist gesperrt und wird am {date} gelöscht", + "deleteUserDone": "Das Konto {name} wurde gelöscht", + "deleteUserFailed": "Das Konto konnte nicht gelöscht werden" }, "invites": { "title": "Einladungen", diff --git a/messages/en.json b/messages/en.json index fab5619d0..308ec10b3 100644 --- a/messages/en.json +++ b/messages/en.json @@ -8110,7 +8110,16 @@ "undoDeletionBody": "{name} asked to delete this account, and it is erased when the grace window ends. Undoing keeps the account and everything in it as it was. {name} can sign in again, but API tokens, share links and trusted devices stay revoked and have to be set up again.", "undoDeletionBodyGuardian": "A guardian deleted the managed profile {name}, and it is erased when the grace window ends. Undoing keeps the profile and everything in it as it was, and its guardians can reach it again. Its share links and tokens stay revoked and have to be set up again.", "undoDeletionDone": "Deletion undone for {name}", - "undoDeletionFailed": "Could not undo the deletion" + "undoDeletionFailed": "Could not undo the deletion", + "undoDeletionBodyAdmin": "An admin deleted the account {name}, and it is erased when the grace window ends. Undoing keeps the account and everything in it as it was. {name} can sign in again, but API tokens, share links and trusted devices stay revoked and have to be set up again.", + "deleteUser": "Delete account", + "deleteUserTitle": "Delete the account {name}?", + "deleteUserBodyGrace": "{name} is signed out everywhere at once and can no longer sign in. The account and everything in it is erased on {date}. Until then you can undo the deletion here.", + "deleteUserBodyNow": "{name} and everything in the account is erased right away. This server keeps no grace window, so the deletion cannot be undone.", + "deleteUserConfirm": "Delete account", + "deleteUserScheduled": "The account {name} is locked and will be erased on {date}", + "deleteUserDone": "The account {name} was deleted", + "deleteUserFailed": "Could not delete the account" }, "invites": { "title": "Invites", diff --git a/messages/es.json b/messages/es.json index 1236f59ef..150c8db31 100644 --- a/messages/es.json +++ b/messages/es.json @@ -8110,7 +8110,16 @@ "undoDeletionBody": "{name} pidió eliminar esta cuenta, y se borra cuando termina el plazo de gracia. Deshacerlo conserva la cuenta y todo su contenido tal como estaba. {name} puede volver a iniciar sesión, pero los tokens de API, los enlaces compartidos y los dispositivos de confianza siguen revocados y hay que configurarlos de nuevo.", "undoDeletionBodyGuardian": "Una persona responsable eliminó el perfil gestionado {name}, y se borra cuando termina el plazo de gracia. Deshacerlo conserva el perfil y todo su contenido tal como estaba, y las personas responsables vuelven a tener acceso. Sus enlaces compartidos y tokens siguen revocados y hay que configurarlos de nuevo.", "undoDeletionDone": "Eliminación de {name} deshecha", - "undoDeletionFailed": "No se pudo deshacer la eliminación" + "undoDeletionFailed": "No se pudo deshacer la eliminación", + "undoDeletionBodyAdmin": "Un administrador eliminó la cuenta {name}, y se borra cuando termina el plazo de gracia. Deshacerlo conserva la cuenta y todo su contenido tal como estaba. {name} puede volver a iniciar sesión, pero los tokens de API, los enlaces compartidos y los dispositivos de confianza siguen revocados y hay que configurarlos de nuevo.", + "deleteUser": "Eliminar cuenta", + "deleteUserTitle": "¿Eliminar la cuenta {name}?", + "deleteUserBodyGrace": "{name} cierra sesión en todas partes al instante y ya no puede iniciar sesión. La cuenta y todo su contenido se borran el {date}. Hasta entonces puedes deshacer la eliminación aquí.", + "deleteUserBodyNow": "{name} y todo el contenido de la cuenta se borran de inmediato. Este servidor no tiene plazo de gracia, así que la eliminación no se puede deshacer.", + "deleteUserConfirm": "Eliminar cuenta", + "deleteUserScheduled": "La cuenta {name} está bloqueada y se borrará el {date}", + "deleteUserDone": "Se eliminó la cuenta {name}", + "deleteUserFailed": "No se pudo eliminar la cuenta" }, "invites": { "title": "Invitaciones", diff --git a/messages/fr.json b/messages/fr.json index fe9d0a531..7665a38d8 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -8110,7 +8110,16 @@ "undoDeletionBody": "{name} a demandé la suppression de ce compte, qui sera effacé à la fin du délai de grâce. Annuler conserve le compte et tout son contenu tels quels. {name} peut se reconnecter, mais les jetons d’API, les liens de partage et les appareils de confiance restent révoqués et doivent être configurés à nouveau.", "undoDeletionBodyGuardian": "Une personne responsable a supprimé le profil géré {name}, qui sera effacé à la fin du délai de grâce. Annuler conserve le profil et tout son contenu tels quels, et les personnes responsables y ont de nouveau accès. Ses liens de partage et jetons restent révoqués et doivent être configurés à nouveau.", "undoDeletionDone": "Suppression de {name} annulée", - "undoDeletionFailed": "Impossible d’annuler la suppression" + "undoDeletionFailed": "Impossible d’annuler la suppression", + "undoDeletionBodyAdmin": "Un administrateur a supprimé le compte {name}, qui sera effacé à la fin du délai de grâce. Annuler conserve le compte et tout son contenu tels quels. {name} peut se reconnecter, mais les jetons d’API, les liens de partage et les appareils de confiance restent révoqués et doivent être configurés à nouveau.", + "deleteUser": "Supprimer le compte", + "deleteUserTitle": "Supprimer le compte {name} ?", + "deleteUserBodyGrace": "{name} est déconnecté partout immédiatement et ne peut plus se connecter. Le compte et tout son contenu seront effacés le {date}. D’ici là, vous pouvez annuler la suppression ici.", + "deleteUserBodyNow": "{name} et tout le contenu du compte sont effacés immédiatement. Ce serveur n’a pas de délai de grâce, la suppression ne peut donc pas être annulée.", + "deleteUserConfirm": "Supprimer le compte", + "deleteUserScheduled": "Le compte {name} est verrouillé et sera effacé le {date}", + "deleteUserDone": "Le compte {name} a été supprimé", + "deleteUserFailed": "Impossible de supprimer le compte" }, "invites": { "title": "Invitations", diff --git a/messages/it.json b/messages/it.json index 490af61d9..85988398b 100644 --- a/messages/it.json +++ b/messages/it.json @@ -8110,7 +8110,16 @@ "undoDeletionBody": "{name} ha chiesto di eliminare questo account, che viene cancellato alla fine del periodo di tolleranza. Annullare mantiene l’account e tutto il suo contenuto com’era. {name} può accedere di nuovo, ma i token API, i link di condivisione e i dispositivi attendibili restano revocati e vanno configurati di nuovo.", "undoDeletionBodyGuardian": "Una persona responsabile ha eliminato il profilo gestito {name}, che viene cancellato alla fine del periodo di tolleranza. Annullare mantiene il profilo e tutto il suo contenuto com’era, e le persone responsabili possono di nuovo accedervi. I suoi link di condivisione e token restano revocati e vanno configurati di nuovo.", "undoDeletionDone": "Eliminazione di {name} annullata", - "undoDeletionFailed": "Impossibile annullare l’eliminazione" + "undoDeletionFailed": "Impossibile annullare l’eliminazione", + "undoDeletionBodyAdmin": "Un amministratore ha eliminato l’account {name}, che viene cancellato alla fine del periodo di tolleranza. Annullare mantiene l’account e tutto il suo contenuto com’era. {name} può accedere di nuovo, ma i token API, i link di condivisione e i dispositivi attendibili restano revocati e vanno configurati di nuovo.", + "deleteUser": "Elimina account", + "deleteUserTitle": "Eliminare l’account {name}?", + "deleteUserBodyGrace": "{name} viene disconnesso ovunque subito e non può più accedere. L’account e tutto il suo contenuto vengono cancellati il {date}. Fino ad allora puoi annullare l’eliminazione da qui.", + "deleteUserBodyNow": "{name} e tutto il contenuto dell’account vengono cancellati subito. Questo server non ha un periodo di tolleranza, quindi l’eliminazione non si può annullare.", + "deleteUserConfirm": "Elimina account", + "deleteUserScheduled": "L’account {name} è bloccato e verrà cancellato il {date}", + "deleteUserDone": "L’account {name} è stato eliminato", + "deleteUserFailed": "Impossibile eliminare l’account" }, "invites": { "title": "Inviti", diff --git a/messages/ko.json b/messages/ko.json index 1437b35ae..b46410178 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -8085,7 +8085,16 @@ "undoDeletionBody": "{name} 님이 이 계정의 삭제를 요청했고, 유예 기간이 끝나면 지워져요. 취소하면 계정과 그 안의 모든 내용이 그대로 남아요. {name} 님은 다시 로그인할 수 있지만, API 토큰, 공유 링크, 신뢰하는 기기는 계속 취소된 상태라 다시 설정해야 해요.", "undoDeletionBodyGuardian": "보호자가 관리 프로필 {name}을(를) 삭제했고, 유예 기간이 끝나면 지워져요. 취소하면 프로필과 그 안의 모든 내용이 그대로 남고, 보호자들이 다시 접근할 수 있어요. 공유 링크와 토큰은 계속 취소된 상태라 다시 설정해야 해요.", "undoDeletionDone": "{name} 삭제를 취소했어요", - "undoDeletionFailed": "삭제를 취소하지 못했어요" + "undoDeletionFailed": "삭제를 취소하지 못했어요", + "undoDeletionBodyAdmin": "관리자가 {name} 계정을 삭제했고, 유예 기간이 끝나면 지워져요. 취소하면 계정과 그 안의 모든 내용이 그대로 남아요. {name} 님은 다시 로그인할 수 있지만, API 토큰, 공유 링크, 신뢰하는 기기는 계속 취소된 상태라 다시 설정해야 해요.", + "deleteUser": "계정 삭제", + "deleteUserTitle": "{name} 계정을 삭제할까요?", + "deleteUserBodyGrace": "{name} 님은 즉시 모든 곳에서 로그아웃되고 더 이상 로그인할 수 없어요. 계정과 그 안의 모든 내용은 {date}에 지워져요. 그때까지는 여기서 삭제를 취소할 수 있어요.", + "deleteUserBodyNow": "{name} 님과 계정 안의 모든 내용이 바로 지워져요. 이 서버에는 유예 기간이 없어서 삭제를 취소할 수 없어요.", + "deleteUserConfirm": "계정 삭제", + "deleteUserScheduled": "{name} 계정이 잠겼고 {date}에 지워져요", + "deleteUserDone": "{name} 계정을 삭제했어요", + "deleteUserFailed": "계정을 삭제하지 못했어요" }, "invites": { "title": "초대", diff --git a/messages/nl.json b/messages/nl.json index 6d62e60fc..968571fce 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -8110,7 +8110,16 @@ "undoDeletionBody": "{name} heeft gevraagd dit account te verwijderen; het wordt gewist als de bedenktijd afloopt. Ongedaan maken houdt het account en alles erin zoals het was. {name} kan weer inloggen, maar API-tokens, deellinks en vertrouwde apparaten blijven ingetrokken en moeten opnieuw worden ingesteld.", "undoDeletionBodyGuardian": "Een verzorger heeft het beheerde profiel {name} verwijderd; het wordt gewist als de bedenktijd afloopt. Ongedaan maken houdt het profiel en alles erin zoals het was, en de verzorgers kunnen het weer bereiken. De deellinks en tokens blijven ingetrokken en moeten opnieuw worden ingesteld.", "undoDeletionDone": "Verwijdering van {name} ongedaan gemaakt", - "undoDeletionFailed": "Verwijdering kon niet ongedaan worden gemaakt" + "undoDeletionFailed": "Verwijdering kon niet ongedaan worden gemaakt", + "undoDeletionBodyAdmin": "Een beheerder heeft het account {name} verwijderd; het wordt gewist als de bedenktijd afloopt. Ongedaan maken houdt het account en alles erin zoals het was. {name} kan weer inloggen, maar API-tokens, deellinks en vertrouwde apparaten blijven ingetrokken en moeten opnieuw worden ingesteld.", + "deleteUser": "Account verwijderen", + "deleteUserTitle": "Het account {name} verwijderen?", + "deleteUserBodyGrace": "{name} wordt meteen overal uitgelogd en kan niet meer inloggen. Het account en alles erin wordt op {date} gewist. Tot dan kun je de verwijdering hier ongedaan maken.", + "deleteUserBodyNow": "{name} en alles in het account wordt meteen gewist. Deze server heeft geen bedenktijd, dus de verwijdering kan niet ongedaan worden gemaakt.", + "deleteUserConfirm": "Account verwijderen", + "deleteUserScheduled": "Het account {name} is vergrendeld en wordt op {date} gewist", + "deleteUserDone": "Het account {name} is verwijderd", + "deleteUserFailed": "Het account kon niet worden verwijderd" }, "invites": { "title": "Uitnodigingen", diff --git a/messages/pl.json b/messages/pl.json index 6dae52955..7681100be 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -8110,7 +8110,16 @@ "undoDeletionBody": "{name} poprosił o usunięcie tego konta; zostanie ono usunięte po upływie okresu karencji. Cofnięcie zachowuje konto i całą jego zawartość bez zmian. {name} może znów się zalogować, ale tokeny API, linki udostępniania i zaufane urządzenia pozostają unieważnione i trzeba je skonfigurować od nowa.", "undoDeletionBodyGuardian": "Osoba opiekująca się profilem usunęła profil zarządzany {name}; zostanie on usunięty po upływie okresu karencji. Cofnięcie zachowuje profil i całą jego zawartość bez zmian, a osoby opiekujące się nim znów mają do niego dostęp. Jego linki udostępniania i tokeny pozostają unieważnione i trzeba je skonfigurować od nowa.", "undoDeletionDone": "Cofnięto usunięcie {name}", - "undoDeletionFailed": "Nie udało się cofnąć usunięcia" + "undoDeletionFailed": "Nie udało się cofnąć usunięcia", + "undoDeletionBodyAdmin": "Administrator usunął konto {name}; zostanie ono usunięte po upływie okresu karencji. Cofnięcie zachowuje konto i całą jego zawartość bez zmian. {name} może znów się zalogować, ale tokeny API, linki udostępniania i zaufane urządzenia pozostają unieważnione i trzeba je skonfigurować od nowa.", + "deleteUser": "Usuń konto", + "deleteUserTitle": "Usunąć konto {name}?", + "deleteUserBodyGrace": "{name} zostanie od razu wszędzie wylogowany i nie będzie mógł się już zalogować. Konto i cała jego zawartość zostaną usunięte {date}. Do tego czasu możesz tutaj cofnąć usunięcie.", + "deleteUserBodyNow": "{name} i cała zawartość konta zostaną usunięte od razu. Ten serwer nie ma okresu karencji, więc usunięcia nie da się cofnąć.", + "deleteUserConfirm": "Usuń konto", + "deleteUserScheduled": "Konto {name} jest zablokowane i zostanie usunięte {date}", + "deleteUserDone": "Konto {name} zostało usunięte", + "deleteUserFailed": "Nie udało się usunąć konta" }, "invites": { "title": "Zaproszenia", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index f7762533c..fedfb8d60 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -8110,7 +8110,16 @@ "undoDeletionBody": "{name} pediu para excluir esta conta, e ela é apagada quando o prazo de carência termina. Desfazer mantém a conta e tudo o que há nela como estava. {name} pode entrar de novo, mas os tokens de API, os links de compartilhamento e os dispositivos confiáveis continuam revogados e precisam ser configurados de novo.", "undoDeletionBodyGuardian": "Um guardião excluiu o perfil gerenciado {name}, e ele é apagado quando o prazo de carência termina. Desfazer mantém o perfil e tudo o que há nele como estava, e os guardiões voltam a ter acesso. Os links de compartilhamento e tokens continuam revogados e precisam ser configurados de novo.", "undoDeletionDone": "Exclusão de {name} desfeita", - "undoDeletionFailed": "Não foi possível desfazer a exclusão" + "undoDeletionFailed": "Não foi possível desfazer a exclusão", + "undoDeletionBodyAdmin": "Um administrador excluiu a conta {name}, e ela é apagada quando o prazo de carência termina. Desfazer mantém a conta e tudo o que há nela como estava. {name} pode entrar de novo, mas os tokens de API, os links de compartilhamento e os dispositivos confiáveis continuam revogados e precisam ser configurados de novo.", + "deleteUser": "Excluir conta", + "deleteUserTitle": "Excluir a conta {name}?", + "deleteUserBodyGrace": "{name} sai da conta em todos os lugares na hora e não consegue mais entrar. A conta e tudo o que há nela são apagados em {date}. Até lá, você pode desfazer a exclusão aqui.", + "deleteUserBodyNow": "{name} e tudo o que há na conta são apagados na hora. Este servidor não tem prazo de carência, então a exclusão não pode ser desfeita.", + "deleteUserConfirm": "Excluir conta", + "deleteUserScheduled": "A conta {name} está bloqueada e será apagada em {date}", + "deleteUserDone": "A conta {name} foi excluída", + "deleteUserFailed": "Não foi possível excluir a conta" }, "invites": { "title": "Convites", diff --git a/src/__tests__/destructive-control-guard.test.ts b/src/__tests__/destructive-control-guard.test.ts index 7d647f30f..75f36fd43 100644 --- a/src/__tests__/destructive-control-guard.test.ts +++ b/src/__tests__/destructive-control-guard.test.ts @@ -679,6 +679,17 @@ const REGISTRY: DestructiveEntry[] = [ recovery: "permanent", confirm: ["AlertDialog"], }, + { + file: "components/admin/delete-user-control.tsx", + destroys: + "another user's whole account, locked at once and erased when the deletion grace window ends (or at once when the window is 0)", + triggers: ["components/admin/user-management-section.tsx"], + // The undo route exists and the row offers it until the purge date + // (`pending-deletion-controls.tsx`); with the window at 0 there is + // nothing to undo, and the dialog says so. + recovery: "restorable", + confirm: ["ConfirmButton"], + }, // ── Integrations ──────────────────────────────────────────────────────── // Every connection renders the shared integration card, whose three-dot diff --git a/src/__tests__/managed-profile-contract.test.ts b/src/__tests__/managed-profile-contract.test.ts index 5ef5aef9f..41be5b6b2 100644 --- a/src/__tests__/managed-profile-contract.test.ts +++ b/src/__tests__/managed-profile-contract.test.ts @@ -145,15 +145,27 @@ describe("managed-profile contract", () => { path.join(root, "src/lib/sharing/grants.ts"), "utf8", ); - const accountDeletion = await readFile( - path.join(root, "src/app/api/settings/account/route.ts"), + // Both account deletions (the account's own and an admin's) take the one + // request path, and that path decides the Guardian invariant. + const deletionRequest = await readFile( + path.join(root, "src/lib/account-deletion/request.ts"), "utf8", ); + const deletionRoutes = await Promise.all( + [ + "src/app/api/settings/account/route.ts", + "src/app/api/admin/users/[id]/route.ts", + ].map((rel) => readFile(path.join(root, rel), "utf8")), + ); expect(lifecycle).toContain("pg_advisory_xact_lock"); expect(lifecycle).toContain("clearManagedProfileMarker"); expect(lifecycle).toContain("deleteManagedProfile"); expect(grants).toContain("reduceManagedProfileGuardian"); - expect(accountDeletion).toContain("deleteGuardianAccountWithLifecycle"); + expect(deletionRequest).toContain("deleteGuardianAccountWithLifecycle("); + for (const route of deletionRoutes) { + expect(route).toContain("requestAccountDeletion("); + expect(route).not.toContain("lockAccountForDeletion("); + } }); }); diff --git a/src/__tests__/openapi-route-coverage-guard.test.ts b/src/__tests__/openapi-route-coverage-guard.test.ts index de9421d6e..bee50ca55 100644 --- a/src/__tests__/openapi-route-coverage-guard.test.ts +++ b/src/__tests__/openapi-route-coverage-guard.test.ts @@ -295,7 +295,7 @@ const UNPUBLISHED: Readonly> = { "/api/admin/support-report": { kind: "adminConsole", methods: ["GET"] }, "/api/admin/tokens": { kind: "adminConsole", methods: ["GET"] }, "/api/admin/users": { kind: "adminConsole", methods: ["GET"] }, - "/api/admin/users/{id}": { kind: "adminConsole", methods: ["PUT"] }, + "/api/admin/users/{id}": { kind: "adminConsole", methods: ["DELETE", "PUT"] }, "/api/admin/users/{id}/force-logout": { kind: "adminConsole", methods: ["POST"], diff --git a/src/__tests__/recent-proof-surface-guard.test.ts b/src/__tests__/recent-proof-surface-guard.test.ts index f3833b347..4a5ac9287 100644 --- a/src/__tests__/recent-proof-surface-guard.test.ts +++ b/src/__tests__/recent-proof-surface-guard.test.ts @@ -111,6 +111,7 @@ const COOKIE_PROOF_ROUTES = [ "app/api/admin/backups/upload/route.ts", "app/api/admin/data/route.ts", "app/api/admin/users/[id]/reset-password/route.ts", + "app/api/admin/users/[id]/route.ts", "app/api/auth/reproof/route.ts", "app/api/tokens/documents/route.ts", "app/api/tokens/measurements/route.ts", diff --git a/src/__tests__/step-up-web-caller-guard.test.ts b/src/__tests__/step-up-web-caller-guard.test.ts index fbf260a0f..9dd1cd427 100644 --- a/src/__tests__/step-up-web-caller-guard.test.ts +++ b/src/__tests__/step-up-web-caller-guard.test.ts @@ -112,6 +112,7 @@ function discoverGatedRoutes(): string[] { */ const GATED_ROUTES: Record = { "DELETE /api/admin/data": "web", + "DELETE /api/admin/users/[id]": "web", "DELETE /api/auth/me/mfa/webauthn/[id]": "web", "DELETE /api/auth/passkeys/[id]": "web", "DELETE /api/managed-profiles/[id]": "web", diff --git a/src/app/api/admin/users/[id]/route.ts b/src/app/api/admin/users/[id]/route.ts index 687edeaf2..f898e71a6 100644 --- a/src/app/api/admin/users/[id]/route.ts +++ b/src/app/api/admin/users/[id]/route.ts @@ -3,13 +3,21 @@ import { assertAnotherAdminStays, LastAdminError, } from "@/lib/auth/admin-roster"; -import { apiHandler, requireAdmin } from "@/lib/api-handler"; +import { + apiHandler, + assertRecentCookieProof, + requireAdmin, +} from "@/lib/api-handler"; import { auditLog } from "@/lib/auth/audit"; import { apiSuccess, apiError, getClientIp } from "@/lib/api-response"; import { parseJsonBody } from "@/lib/api/body"; import { annotate } from "@/lib/logging/context"; import { NextRequest } from "next/server"; import { z } from "zod/v4"; +import { requestAccountDeletion } from "@/lib/account-deletion/request"; +import { lockPendingAccount } from "@/lib/account-deletion/lifecycle"; +import { LastManagedGuardianError } from "@/lib/managed-profiles/lifecycle"; +import { adminDeleteUserSchema } from "@/lib/validations/admin"; const updateUserSchema = z .object({ @@ -122,3 +130,134 @@ export const PUT = apiHandler( }); }, ); + +/** A refusal decided under the account's row lock, inside the transaction. */ +class DeletionRefusal extends Error { + constructor(readonly kind: "not_found" | "managed_profile" | "pending") { + super(`admin deletion refused: ${kind}`); + this.name = "DeletionRefusal"; + } +} + +function deletionRefusalResponse(kind: DeletionRefusal["kind"]): Response { + if (kind === "not_found") return apiError("User not found", 404); + if (kind === "managed_profile") { + return apiError("A managed profile is deleted by its Guardians", 409, { + errorCode: "admin.users.delete.managed_profile", + }); + } + return apiError("This account is already pending deletion", 409, { + errorCode: "account.deletion.already_pending", + }); +} + +/** + * DELETE /api/admin/users/{id} — an admin deletes another account. + * + * The same request the account's own deletion makes + * (`requestAccountDeletion`): with a grace window the account is locked at + * once (every credential revoked, sign-in refused) and erased by the purge job + * when the window ends, and `POST /api/admin/users/{id}/undo-deletion` gives it + * back until then; with `ACCOUNT_DELETION_GRACE_DAYS=0` it is erased here. + * + * Cookie-only (`requireAdmin`) behind a fresh proof (`assertRecentCookieProof`, + * the gate in front of the other admin acts on someone's data), with the typed + * confirmation the self-deletion asks for. Refused for the caller's own + * account (that is the account settings' deletion, with its own step-up), for + * the last admin who can sign in, for a managed profile (its Guardians delete + * it), for an account already pending deletion, and for the last Guardian of + * a managed profile. Audit-logged for the admin and on the account itself. + */ +export const DELETE = apiHandler( + async ( + request: NextRequest, + { params }: { params: Promise<{ id: string }> }, + ) => { + const { user: admin, session } = await requireAdmin(); + await assertRecentCookieProof(admin, session.id); + + const { id } = await params; + annotate({ + action: { + name: "admin.users.delete", + entity_type: "user", + entity_id: id, + }, + }); + + const parsed = await parseJsonBody(request, adminDeleteUserSchema, { + maxBytes: 4 * 1024, + }); + if (!parsed.ok) return parsed.error; + + if (id === admin.id) { + return apiError( + "Delete your own account from your account settings", + 400, + { errorCode: "admin.users.delete.self" }, + ); + } + + const target = await prisma.user.findUnique({ + where: { id }, + select: { role: true, username: true }, + }); + if (!target) return apiError("User not found", 404); + + let purgeAt: Date | null; + try { + ({ purgeAt } = await requestAccountDeletion({ + userId: id, + targetIsAdmin: target.role === "ADMIN", + requestedById: admin.id, + inTransaction: async (tx, scheduledPurgeAt) => { + const state = await lockPendingAccount(tx, id); + if (!state) throw new DeletionRefusal("not_found"); + if (state.managedProfile) + throw new DeletionRefusal("managed_profile"); + if (state.pending) throw new DeletionRefusal("pending"); + const details = { + targetUserId: id, + username: target.username, + scheduledPurgeAt: scheduledPurgeAt?.toISOString() ?? null, + }; + await auditLog("admin.user.delete", { + userId: admin.id, + ipAddress: getClientIp(request), + details, + client: tx, + }); + // On the account itself too, so its own activity shows who asked. + // An immediate erasure takes this row with the account's history. + await auditLog("user.account.delete", { + userId: id, + actorUserId: admin.id, + details: { ...details, requestedBy: "admin" }, + client: tx, + }); + }, + })); + } catch (error) { + if (error instanceof DeletionRefusal) { + return deletionRefusalResponse(error.kind); + } + if (error instanceof LastAdminError) { + return apiError("The last admin account cannot be deleted", 400); + } + if (error instanceof LastManagedGuardianError) { + return apiError( + "This account is the last Guardian of a managed profile. Add another Guardian or delete the profile first", + 409, + { errorCode: "managed_profile.guardian.required" }, + ); + } + throw error; + } + + annotate({ meta: { deletion_scheduled: purgeAt !== null } }); + return apiSuccess({ + deleted: true, + scheduledPurgeAt: purgeAt?.toISOString() ?? null, + }); + }, +); diff --git a/src/app/api/admin/users/__tests__/route.test.ts b/src/app/api/admin/users/__tests__/route.test.ts index c9ddc4bfd..38d25a864 100644 --- a/src/app/api/admin/users/__tests__/route.test.ts +++ b/src/app/api/admin/users/__tests__/route.test.ts @@ -100,6 +100,7 @@ describe("GET /api/admin/users", () => { // BigInt in Prisma, plain number on the wire. documentQuotaBytes: 2_147_483_648, passkeyCount: 2, + managedProfile: false, // Not pending deletion. deletion: null, }); @@ -128,6 +129,7 @@ describe("GET /api/admin/users", () => { deletionRequestedAt: true, deletionPurgeAt: true, deletionRequestedById: true, + managedProfileAt: true, _count: { select: { passkeys: true } }, }); expect(args.select).not.toHaveProperty("passwordHash"); diff --git a/src/app/api/admin/users/route.ts b/src/app/api/admin/users/route.ts index a46dfd455..1fec01c53 100644 --- a/src/app/api/admin/users/route.ts +++ b/src/app/api/admin/users/route.ts @@ -21,6 +21,7 @@ export const GET = apiHandler(async () => { deletionRequestedAt: true, deletionPurgeAt: true, deletionRequestedById: true, + managedProfileAt: true, _count: { select: { passkeys: true } }, }, orderBy: { createdAt: "asc" }, @@ -42,9 +43,12 @@ export const GET = apiHandler(async () => { documentQuotaBytes: u.documentQuotaBytes === null ? null : Number(u.documentQuotaBytes), passkeyCount: u._count.passkeys, + // A record somebody else runs. Its Guardians delete it, so the list + // offers no admin deletion on it. + managedProfile: u.managedProfileAt != null, // An account pending deletion: when it was asked for, when the - // purge job erases it, and whether the account asked itself or a - // Guardian deleted the managed profile. Null on every other account. + // purge job erases it, and who asked: the account itself, a Guardian + // (a managed profile), or an admin. Null on every other account. // The list offers "Undo deletion" on these rows until `purgeAt`. deletion: u.deletionRequestedAt ? { @@ -53,7 +57,9 @@ export const GET = apiHandler(async () => { requestedBy: u.deletionRequestedById === u.id ? ("self" as const) - : ("guardian" as const), + : u.managedProfileAt != null + ? ("guardian" as const) + : ("admin" as const), } : null, })), diff --git a/src/app/api/settings/account/route.ts b/src/app/api/settings/account/route.ts index 2378f2aee..ead70b83d 100644 --- a/src/app/api/settings/account/route.ts +++ b/src/app/api/settings/account/route.ts @@ -8,24 +8,10 @@ import { MFA_STEP_UP_MAX_AGE_SECONDS, } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; -import { kickOffhostPurge } from "@/lib/jobs/offhost-purge"; -import { - deleteGuardianAccountWithLifecycle, - LastManagedGuardianError, -} from "@/lib/managed-profiles/lifecycle"; -import { - accountDeletionGraceDays, - accountDeletionPurgeAt, -} from "@/lib/account-deletion/grace"; -import { - eraseAccount, - lockAccountForDeletion, -} from "@/lib/account-deletion/lifecycle"; +import { LastManagedGuardianError } from "@/lib/managed-profiles/lifecycle"; +import { requestAccountDeletion } from "@/lib/account-deletion/request"; import { ACTIVE_ACCOUNT } from "@/lib/account-deletion/active-account"; -import { - assertAnotherAdminStays, - LastAdminError, -} from "@/lib/auth/admin-roster"; +import { LastAdminError } from "@/lib/auth/admin-roster"; export const dynamic = "force-dynamic"; @@ -104,53 +90,27 @@ export const DELETE = apiHandler(async (request: NextRequest) => { // the elevation would stay redeemable while the erasure was already running. await auth.commitElevation(); - const requestedAt = new Date(); - const purgeAt = accountDeletionPurgeAt( - requestedAt, - accountDeletionGraceDays(), - ); - + let purgeAt: Date | null; try { - await deleteGuardianAccountWithLifecycle( + ({ purgeAt } = await requestAccountDeletion({ userId, - async (tx) => { - // Checked again under the roster lock: two admins deleting at the - // same moment must not each count the other. - if (user.role === "ADMIN") await assertAnotherAdminStays(tx, userId); - // The invariant has now passed, so this success event cannot survive - // a refusal. On an immediate erasure it is purged with the rest of the - // account's audit history; during a grace window it is the account's - // own record of the request. - await auditLog("user.account.delete", { + targetIsAdmin: user.role === "ADMIN", + requestedById: userId, + // The invariants have passed by the time this runs, so this success + // event cannot survive a refusal. On an immediate erasure it is purged + // with the rest of the account's audit history; during a grace window + // it is the account's own record of the request. + inTransaction: (tx, scheduledPurgeAt) => + auditLog("user.account.delete", { userId, ipAddress: getClientIp(request), details: { username, - scheduledPurgeAt: purgeAt?.toISOString() ?? null, + scheduledPurgeAt: scheduledPurgeAt?.toISOString() ?? null, }, client: tx, - }); - - if (purgeAt) { - // Locked, not erased: credentials revoked in this transaction, - // the record untouched until the purge job reaches `purgeAt`. - await lockAccountForDeletion(tx, { - userId, - requestedById: userId, - requestedAt, - purgeAt, - }); - return; - } - - // No window: erase now, inside the same record lifecycle transaction - // so a refusal leaves every account state intact (sessions, the - // account's audit history — GDPR Art. 17 — the off-host purge - // request, then the row with everything cascading from it). - await eraseAccount(tx, userId, "account_deleted"); - }, - { erasing: purgeAt === null }, - ); + }), + })); } catch (error) { if (error instanceof LastAdminError) { return apiError("The last admin account cannot be deleted", 400); @@ -165,8 +125,6 @@ export const DELETE = apiHandler(async (request: NextRequest) => { throw error; } - if (!purgeAt) await kickOffhostPurge(); - annotate({ action: { name: "settings.account.delete" }, meta: { username, deletion_scheduled: purgeAt !== null }, diff --git a/src/components/admin/__tests__/user-management-responsive.test.tsx b/src/components/admin/__tests__/user-management-responsive.test.tsx index 60c5dedb9..0ee8df3be 100644 --- a/src/components/admin/__tests__/user-management-responsive.test.tsx +++ b/src/components/admin/__tests__/user-management-responsive.test.tsx @@ -30,6 +30,8 @@ const sampleUsers = [ role: "ADMIN" as const, passkeyCount: 2, createdAt: "2026-04-01T08:00:00Z", + managedProfile: false, + deletion: null, }, { id: "u2", @@ -38,6 +40,8 @@ const sampleUsers = [ role: "USER" as const, passkeyCount: 1, createdAt: "2026-04-15T08:00:00Z", + managedProfile: false, + deletion: null, }, ]; @@ -53,6 +57,10 @@ vi.mock("@tanstack/react-query", () => ({ }), })); +vi.mock("@/lib/queries/use-privacy-summary", () => ({ + useAccountDeletionGrace: () => ({ days: 14, pending: false }), +})); + vi.mock("@/hooks/use-auth", () => ({ useAuth: () => ({ user: { @@ -115,6 +123,15 @@ describe("UserManagementSection — mobile responsive", () => { expect(mobileSlice).toContain("Force logout"); }); + it("offers deleting every account but the admin's own", () => { + const html = render(); + // Two rows, each in the desktop table and the mobile list; only the + // other account carries the control. + const deletes = html.match(/data-slot="admin-user-delete"/g) ?? []; + expect(deletes).toHaveLength(2); + expect(html).toContain('aria-label="Delete account"'); + }); + it("uses smaller card padding on mobile (p-4 md:p-6)", () => { const html = render(); expect(html).toMatch(/class="[^"]*\bp-4\b[^"]*md:p-6/); diff --git a/src/components/admin/_shared.tsx b/src/components/admin/_shared.tsx index 2f2223653..6ea08b5a3 100644 --- a/src/components/admin/_shared.tsx +++ b/src/components/admin/_shared.tsx @@ -34,15 +34,17 @@ export interface AdminUser { // the instance-wide default applies. documentQuotaBytes: number | null; passkeyCount: number; + /** A record its Guardians run; they delete it, an admin does not. */ + managedProfile: boolean; /** - * Set while the account is pending deletion: when the purge job - * erases it, and whether the account asked itself or a Guardian deleted - * the managed profile. The row offers "Undo deletion" until then. + * Set while the account is pending deletion: when the purge job erases + * it, and who asked (the account itself, a Guardian of the managed + * profile, or an admin). The row offers "Undo deletion" until then. */ deletion: { requestedAt: string; purgeAt: string | null; - requestedBy: "self" | "guardian"; + requestedBy: "self" | "guardian" | "admin"; } | null; } diff --git a/src/components/admin/delete-user-control.tsx b/src/components/admin/delete-user-control.tsx new file mode 100644 index 000000000..dc25e1591 --- /dev/null +++ b/src/components/admin/delete-user-control.tsx @@ -0,0 +1,136 @@ +"use client"; + +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { Trash2 } from "lucide-react"; + +import { ConfirmButton } from "@/components/ui/confirm-button"; +import { toastWrittenOutcome } from "@/components/outcome/outcome-toast"; +import { showErrorToast } from "@/lib/errors/error-toast"; +import { apiDelete } from "@/lib/api/api-fetch"; +import { formatDate } from "@/lib/format"; +import { useTranslations } from "@/lib/i18n/context"; +import { queryKeys } from "@/lib/query-keys"; +import { useAccountDeletionGrace } from "@/lib/queries/use-privacy-summary"; +import { + recentProofErrorMessage, + useRecentProof, +} from "@/components/settings/security-section/use-recent-proof"; +import type { AdminUser } from "./_shared"; + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** + * Whether the row offers deleting the account: never the admin's own (that is + * the account settings' deletion), never a managed profile (its Guardians + * delete it), and not one already pending deletion (the row offers the undo). + */ +export function canAdminDelete(user: AdminUser, currentUserId: string | null) { + return ( + user.id !== currentUserId && !user.managedProfile && user.deletion === null + ); +} + +/** + * The day the purge job would erase an account deleted now, or null when the + * window is 0 and the erasure happens at once. The server stamps the real + * date; this is the one the dialog names before asking. + */ +export function deletionDateFrom(graceDays: number, now: Date): Date | null { + return graceDays > 0 ? new Date(now.getTime() + graceDays * DAY_MS) : null; +} + +/** + * "Delete account" on a row of the admin users list + * (`DELETE /api/admin/users/{id}`), for the rows `canAdminDelete` allows. + */ +export function DeleteUserControl({ + user, + currentUserId, +}: { + user: AdminUser; + currentUserId: string | null; +}) { + if (!canAdminDelete(user, currentUserId)) return null; + return ; +} + +/** + * The dialog names the date the account is erased and that it can be undone + * here until then, or, with no grace window, that it is erased at once. It + * waits for the window before it offers anything, so the sentence is never a + * guess. The fresh proof the route asks for is collected by the shared + * re-proof dialog. + */ +function DeleteUserButton({ user }: { user: AdminUser }) { + const { t } = useTranslations(); + const queryClient = useQueryClient(); + const grace = useAccountDeletionGrace(); + const recentProof = useRecentProof(); + + const remove = useMutation({ + mutationFn: () => + recentProof.run(() => + apiDelete<{ scheduledPurgeAt: string | null }>( + `/api/admin/users/${user.id}`, + { confirm: "DELETE_ACCOUNT" }, + ), + ), + onSuccess: (result) => { + void queryClient.invalidateQueries({ queryKey: queryKeys.adminUsers() }); + toastWrittenOutcome( + "success", + result.scheduledPurgeAt + ? t("admin.section.users.deleteUserScheduled", { + name: user.username, + date: formatDate(result.scheduledPurgeAt), + }) + : t("admin.section.users.deleteUserDone", { name: user.username }), + ); + }, + onError: (err) => { + const message = recentProofErrorMessage( + err, + err instanceof Error && err.message + ? err.message + : t("admin.section.users.deleteUserFailed"), + ); + if (message) showErrorToast(message, { cause: err }); + }, + }); + + const purgeOn = + grace.days === null ? null : deletionDateFrom(grace.days, new Date()); + + return ( + <> + {recentProof.dialog} +