diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 0fcf0c875..f010bf744 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -15327,6 +15327,400 @@ paths: $ref: "#/components/schemas/CsvImportTooLargeEnvelope" "422": *a4 "429": *a5 + /api/import/labs-csv: + post: + tags: + - Import + summary: Import lab results from CSV + description: "Body is the raw CSV file (`text/csv`, ≤ 16 MB), read as a stream: a UTF-8 byte-order mark is dropped, the + separator is `,`, `;` or a tab, quoted fields may hold separators and line breaks. Every row is checked on its + own and a refused row is reported by line with a reason, never guessed (a number or date that reads two ways is + refused). Re-uploading the same file writes nothing: every row is reported as a duplicate. `dryRun=1` checks the + whole file without writing and returns the first rows as they would be stored; it spends a separate bucket of 30 + checks an hour, the import itself the shared 5-an-hour `import:` bucket. Owner only; refused with + `module.disabled` when the module is switched off. Each row's name resolves to the account's marker the way a + scanned report's rows do (name, alias, catalog name); a new analyte mints one marker from its first row. A + numeric result needs a unit; one in another unit is converted when the conversion is known and keeps the printed + value beside it, otherwise the row is refused with `unit_mismatch`. Any other text in the value column is a + qualitative result. A date without a time is stored as that calendar date. At most 20 000 data rows." + parameters: + - in: query + name: dryRun + schema: + description: "`1` validates every row and reports what would happen, writing nothing." + default: "0" + type: string + enum: + - "0" + - "1" + - "true" + - "false" + description: "`1` validates every row and reports what would happen, writing nothing." + - in: query + name: header + schema: + description: "`1` when the first row holds column names and is not data." + default: "1" + type: string + enum: + - "0" + - "1" + description: "`1` when the first row holds column names and is not data." + - in: query + name: delimiter + schema: + default: auto + description: Column separator; `auto` detects it from the first line. + type: string + enum: + - auto + - comma + - semicolon + - tab + description: Column separator; `auto` detects it from the first line. + - in: query + name: decimal + schema: + default: dot + description: Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is + refused per row as ambiguous. + type: string + enum: + - dot + - comma + description: Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is + refused per row as ambiguous. + - in: query + name: dateOrder + schema: + default: iso + description: "`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit + years are refused." + type: string + enum: + - iso + - dmy + - mdy + description: "`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit + years are refused." + - in: query + name: timezone + schema: + description: IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time + is stored as that calendar date. + type: string + maxLength: 64 + description: IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time + is stored as that calendar date. + - in: query + name: analyte + schema: + type: integer + minimum: 0 + maximum: 99 + description: Column of the marker name (required). + required: true + description: Column of the marker name (required). + - in: query + name: value + schema: + type: integer + minimum: 0 + maximum: 99 + description: Column of the result (required). A number is a numeric result; any other text (`negative`, `<0.5`) is kept + as a qualitative result. + required: true + description: Column of the result (required). A number is a numeric result; any other text (`negative`, `<0.5`) is kept + as a qualitative result. + - in: query + name: unit + schema: + description: Column of the unit. Required for a numeric result. + type: integer + minimum: 0 + maximum: 99 + description: Column of the unit. Required for a numeric result. + - in: query + name: date + schema: + type: integer + minimum: 0 + maximum: 99 + description: Column of the date, or date and time (required). + required: true + description: Column of the date, or date and time (required). + - in: query + name: time + schema: + description: Column of the time, when separate. + type: integer + minimum: 0 + maximum: 99 + description: Column of the time, when separate. + - in: query + name: referenceLow + schema: + type: integer + minimum: 0 + maximum: 99 + description: 0-based index of the file column this field is read from. + - in: query + name: referenceHigh + schema: + type: integer + minimum: 0 + maximum: 99 + description: 0-based index of the file column this field is read from. + - in: query + name: referenceRange + schema: + description: Column of a printed range such as `4.0-5.6` or `< 200`. + type: integer + minimum: 0 + maximum: 99 + description: Column of a printed range such as `4.0-5.6` or `< 200`. + - in: query + name: panel + schema: + type: integer + minimum: 0 + maximum: 99 + description: 0-based index of the file column this field is read from. + - in: query + name: note + schema: + description: Column of a note; stored encrypted. + type: integer + minimum: 0 + maximum: 99 + description: Column of a note; stored encrypted. + requestBody: + required: true + content: + text/csv: + schema: + type: string + example: |- + Analyte;Value;Unit;Date;Range + HbA1c;5,4;%;01.03.2025;4,0-5,6 + Urine protein;negative;;01.03.2025; + responses: + "200": + description: Per-file and per-row outcome. + content: + application/json: + schema: + $ref: "#/components/schemas/LabsCsvImportEnvelope" + "401": *a1 + "403": + description: "`module.disabled`: the module the import writes to is switched off." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "413": + description: "`import.csv.too_large`: the file is larger than 16 MB." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "415": + description: "`import.csv.content_type`: the body is not `text/csv`." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a4 + "429": *a5 + /api/import/medications-csv: + post: + tags: + - Import + summary: Import medications from CSV + description: "Body is the raw CSV file (`text/csv`, ≤ 16 MB), read as a stream: a UTF-8 byte-order mark is dropped, the + separator is `,`, `;` or a tab, quoted fields may hold separators and line breaks. Every row is checked on its + own and a refused row is reported by line with a reason, never guessed (a number or date that reads two ways is + refused). Re-uploading the same file writes nothing: every row is reported as a duplicate. `dryRun=1` checks the + whole file without writing and returns the first rows as they would be stored; it spends a separate bucket of 30 + checks an hour, the import itself the shared 5-an-hour `import:` bucket. Owner only; refused with + `module.disabled` when the module is switched off. Maps name, dose (as written), daily intake times, an + as-needed mark and a start and end date; every other rhythm is set up afterwards in the app. A row needs times + unless it is marked as needed. Each row is checked against the create route's own schema; a dated window opens a + course. Reminders stay off unless `reminders=1`. A medication the record holds under the same name and dose is a + duplicate. At most 500 data rows." + parameters: + - in: query + name: dryRun + schema: + description: "`1` validates every row and reports what would happen, writing nothing." + default: "0" + type: string + enum: + - "0" + - "1" + - "true" + - "false" + description: "`1` validates every row and reports what would happen, writing nothing." + - in: query + name: header + schema: + description: "`1` when the first row holds column names and is not data." + default: "1" + type: string + enum: + - "0" + - "1" + description: "`1` when the first row holds column names and is not data." + - in: query + name: delimiter + schema: + default: auto + description: Column separator; `auto` detects it from the first line. + type: string + enum: + - auto + - comma + - semicolon + - tab + description: Column separator; `auto` detects it from the first line. + - in: query + name: decimal + schema: + default: dot + description: Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is + refused per row as ambiguous. + type: string + enum: + - dot + - comma + description: Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is + refused per row as ambiguous. + - in: query + name: dateOrder + schema: + default: iso + description: "`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit + years are refused." + type: string + enum: + - iso + - dmy + - mdy + description: "`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit + years are refused." + - in: query + name: timezone + schema: + description: IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time + is stored as that calendar date. + type: string + maxLength: 64 + description: IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time + is stored as that calendar date. + - in: query + name: name + schema: + type: integer + minimum: 0 + maximum: 99 + description: Column of the medication name (required). + required: true + description: Column of the medication name (required). + - in: query + name: dose + schema: + type: integer + minimum: 0 + maximum: 99 + description: Column of the dose as written, e.g. `10 mg` (required). + required: true + description: Column of the dose as written, e.g. `10 mg` (required). + - in: query + name: times + schema: + description: "Column of daily intake times (`08:00`, or several: `08:00 20:00`, separated by space, comma, semicolon or + `|`). A medication with times is taken every day at them; any other rhythm is set up afterwards in the + app. A row needs times unless it is marked as needed." + type: integer + minimum: 0 + maximum: 99 + description: "Column of daily intake times (`08:00`, or several: `08:00 20:00`, separated by space, comma, semicolon or + `|`). A medication with times is taken every day at them; any other rhythm is set up afterwards in the app. + A row needs times unless it is marked as needed." + - in: query + name: asNeeded + schema: + description: Column marking an as-needed medication (yes/no). + type: integer + minimum: 0 + maximum: 99 + description: Column marking an as-needed medication (yes/no). + - in: query + name: startsOn + schema: + description: Column of the start date. + type: integer + minimum: 0 + maximum: 99 + description: Column of the start date. + - in: query + name: endsOn + schema: + description: Column of the end date. + type: integer + minimum: 0 + maximum: 99 + description: Column of the end date. + - in: query + name: reminders + schema: + description: "`1` turns reminders on for imported medications that have times. Off by default, so an import never starts + reminding on its own." + default: "0" + type: string + enum: + - "0" + - "1" + description: "`1` turns reminders on for imported medications that have times. Off by default, so an import never starts + reminding on its own." + requestBody: + required: true + content: + text/csv: + schema: + type: string + example: |- + name,dose,times,as needed,start,end + Ramipril,5 mg,08:00,,2025-01-01, + Ibuprofen,400 mg,,yes,, + responses: + "200": + description: Per-file and per-row outcome. + content: + application/json: + schema: + $ref: "#/components/schemas/MedicationsCsvImportEnvelope" + "401": *a1 + "403": + description: "`module.disabled`: the module the import writes to is switched off." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "413": + description: "`import.csv.too_large`: the file is larger than 16 MB." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "415": + description: "`import.csv.content_type`: the body is not `text/csv`." + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + "422": *a4 + "429": *a5 /api/import: post: tags: @@ -36947,7 +37341,8 @@ components: http.unauthenticated, http.unavailable, http.unsupported_media_type; illness — illness.day-log.invalid, illness.disabled, illness.episode.chronic-no-resolve, illness.episode.invalid, illness.episode.invalid-window, illness.episode.parent-not-found, illness.insights.invalid; import — - import.apple_health.busy, import.write_failed; ingest — ingest.rate_limited; insights — + import.apple_health.busy, import.csv.content_type, import.csv.empty, import.csv.too_large, + import.rate_limited, import.write_failed; ingest — ingest.rate_limited; insights — insights.generate.budgetExceeded, insights.generate.outboundScreened, insights.generate.profileScopeChanged, insights.generate.scopeChanged; labs — labs.biomarker.alias.taken, labs.biomarker.create.invalid, labs.biomarker.duplicate, @@ -71828,6 +72223,180 @@ components: - data - error additionalProperties: false + LabsCsvImportEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/RecordImportResult" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + RecordImportResult: + type: object + properties: + kind: + type: string + enum: + - labs + - medications + dryRun: + type: boolean + rows: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Data rows read (the header excluded). + valid: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Rows that passed every check. + inserted: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Rows written; in a dry run, the rows that would be. + duplicates: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Rows skipped because the record already holds them, or an earlier row of the file does. A re-upload of the + same file therefore writes nothing. + failed: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Rows refused; each is listed in `errors`. + errors: + type: array + items: + type: object + properties: + line: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: 1-based line of the file. + reason: + type: string + enum: + - missing_value + - invalid_number + - ambiguous_number + - missing_unit + - invalid_date + - invalid_time + - ambiguous_date + - future_date + - text_too_long + - unit_mismatch + - invalid_reference_range + - invalid_times + - missing_times + - invalid_yes_no + - as_needed_with_times + - end_before_start + - invalid_medication + - oversized_row + - row_limit + - duplicate + field: + description: The mapped field at fault. + type: string + required: + - line + - reason + additionalProperties: false + description: Refused and duplicate rows, in file order, at most 500. + errorsTruncated: + type: boolean + markers: + description: "Lab imports only: how the file's names meet the markers." + type: object + properties: + matched: + type: array + items: + type: object + properties: + name: + type: string + description: The name as the file writes it. + marker: + type: string + description: The marker it resolves to. + how: + type: string + enum: + - name + - alias + - catalog + required: + - name + - marker + - how + additionalProperties: false + description: File names that resolve to an existing marker, at most 200. + created: + type: array + items: + type: string + description: Names that become new markers, at most 200. + required: + - matched + - created + additionalProperties: false + preview: + type: array + items: + type: object + propertyNames: + type: string + additionalProperties: + type: + - string + - number + - boolean + - "null" + description: The first 20 valid rows as they will be stored (dry run only). + required: + - kind + - dryRun + - rows + - valid + - inserted + - duplicates + - failed + - errors + - errorsTruncated + - preview + additionalProperties: false + MedicationsCsvImportEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/RecordImportResult" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false JsonImportStatsEnvelope: type: object properties: diff --git a/src/__tests__/medication-intake-tracking-guard.test.ts b/src/__tests__/medication-intake-tracking-guard.test.ts index 6469bcfa4..5b91d591b 100644 --- a/src/__tests__/medication-intake-tracking-guard.test.ts +++ b/src/__tests__/medication-intake-tracking-guard.test.ts @@ -96,6 +96,9 @@ const MAY_READ_EVERY_ROW: Record = { // as stored, schedule and switch included. "lib/export/full-backup-payload.ts": "backup carries the stored rows", "lib/export/restore-backup.ts": "restore writes the stored rows back", + // v1.44 — the CSV import writes new medications and reads only their names + // and doses to skip duplicates; it derives nothing due. + "lib/import/medications-csv.ts": "imports new rows, derives nothing due", // v1.43 — the units a taken dose consumed, resolved per slot exactly as // the inventory does. It reads only doses already recorded as taken. "lib/medications/ingredients/recompute.ts": diff --git a/src/__tests__/write-rate-limit-guard.test.ts b/src/__tests__/write-rate-limit-guard.test.ts index 3b3b3559b..8498633f5 100644 --- a/src/__tests__/write-rate-limit-guard.test.ts +++ b/src/__tests__/write-rate-limit-guard.test.ts @@ -53,6 +53,9 @@ const LIMITING_HELPERS: Record = { "lib/withings/webhook-handler.ts", "lib/whoop/webhook-handler.ts", ], + // v1.44 — the lab and medication CSV imports: a preview bucket for a dry + // run, the shared `import:` bucket for the import itself. + handleRecordImport: ["lib/import/record-import-route.ts"], }; /** Mutating routes with no limit, each with the reason it needs none. */ diff --git a/src/app/api/import/__tests__/record-csv-routes.test.ts b/src/app/api/import/__tests__/record-csv-routes.test.ts new file mode 100644 index 000000000..2c3ec0294 --- /dev/null +++ b/src/app/api/import/__tests__/record-csv-routes.test.ts @@ -0,0 +1,165 @@ +/** + * The request side of `/api/import/labs-csv` and `/api/import/medications-csv` + * (#1215): module gate first, a separate bucket for dry runs, the content + * type, the byte cap while streaming, the mapping refused before anything is + * read, and an audit row only for a real import. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +const mocks = vi.hoisted(() => ({ + checkRateLimit: vi.fn(), + requireModuleEnabled: vi.fn(), + importLabsCsv: vi.fn(), + importMedicationsCsv: vi.fn(), + auditLog: vi.fn(), +})); + +vi.mock("@/lib/api-handler", () => ({ + apiHandler: (handler: unknown) => handler, + requireAuth: vi.fn(async () => ({ user: { id: "user-1" } })), +})); +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/auth/audit", () => ({ auditLog: mocks.auditLog })); +vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: mocks.checkRateLimit })); +vi.mock("@/lib/modules/gate", () => ({ + requireModuleEnabled: mocks.requireModuleEnabled, +})); +vi.mock("@/lib/tz/resolver", () => ({ + resolveUserTimezone: vi.fn(async () => "Europe/Berlin"), +})); +vi.mock("@/lib/import/labs-csv", () => ({ + importLabsCsv: mocks.importLabsCsv, +})); +vi.mock("@/lib/import/medications-csv", () => ({ + importMedicationsCsv: mocks.importMedicationsCsv, +})); + +import { CsvTooLargeError } from "@/lib/import/csv-stream"; +import { POST as labsPost } from "../labs-csv/route"; +import { POST as medsPost } from "../medications-csv/route"; + +type Handler = (req: NextRequest) => Promise; +const RESULT = { + kind: "labs", + dryRun: false, + rows: 2, + valid: 2, + inserted: 2, + duplicates: 0, + failed: 0, + errors: [], + errorsTruncated: false, + preview: [], +}; + +function post( + handler: unknown, + query: string, + body = "a,b\n1,2\n", + type = "text/csv", +) { + return (handler as Handler)( + new NextRequest(`http://localhost/api/import/x?${query}`, { + method: "POST", + headers: { "content-type": type }, + body, + }), + ); +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.checkRateLimit.mockResolvedValue({ allowed: true }); + mocks.requireModuleEnabled.mockResolvedValue({ enabled: true }); + mocks.importLabsCsv.mockResolvedValue(RESULT); + mocks.importMedicationsCsv.mockResolvedValue({ + ...RESULT, + kind: "medications", + }); +}); + +const LABS = "analyte=0&value=1&unit=2&date=3"; +const MEDS = "name=0&dose=1×=2"; + +describe("record CSV import routes", () => { + it("answers the module gate before reading the file", async () => { + mocks.requireModuleEnabled.mockResolvedValue({ + enabled: false, + response: new Response(null, { status: 403 }), + }); + expect((await post(labsPost, LABS)).status).toBe(403); + expect(mocks.requireModuleEnabled).toHaveBeenCalledWith("user-1", "labs"); + expect(mocks.importLabsCsv).not.toHaveBeenCalled(); + }); + + it("refuses a mapping without the required columns, or with a shared column", async () => { + expect((await post(labsPost, "analyte=0&value=1")).status).toBe(422); + expect((await post(labsPost, "analyte=0&value=0&date=1")).status).toBe(422); + expect((await post(medsPost, "name=0")).status).toBe(422); + expect(mocks.importLabsCsv).not.toHaveBeenCalled(); + }); + + it("spends the preview bucket on a dry run and the import bucket otherwise", async () => { + await post(labsPost, `${LABS}&dryRun=1`); + expect(mocks.checkRateLimit).toHaveBeenLastCalledWith( + "import-preview:user-1", + 30, + 3_600_000, + ); + expect(mocks.auditLog).not.toHaveBeenCalled(); + await post(medsPost, MEDS); + expect(mocks.checkRateLimit).toHaveBeenLastCalledWith( + "import:user-1", + 5, + 3_600_000, + ); + expect(mocks.auditLog).toHaveBeenCalledWith( + "import.medications.csv", + expect.objectContaining({ userId: "user-1" }), + ); + }); + + it("answers 429 when the bucket is spent", async () => { + mocks.checkRateLimit.mockResolvedValue({ allowed: false }); + expect((await post(labsPost, LABS)).status).toBe(429); + }); + + it("refuses a body that is not CSV", async () => { + expect((await post(labsPost, LABS, "{}", "application/json")).status).toBe( + 415, + ); + }); + + it("turns the streaming cap into a 413", async () => { + mocks.importLabsCsv.mockRejectedValue(new CsvTooLargeError(1)); + expect((await post(labsPost, LABS)).status).toBe(413); + }); + + it("calls a file without data rows empty", async () => { + mocks.importLabsCsv.mockResolvedValue({ ...RESULT, rows: 0 }); + expect((await post(labsPost, LABS)).status).toBe(422); + }); + + it("hands the parsed mapping and the account zone to the importer", async () => { + const res = await post( + labsPost, + `${LABS}&decimal=comma&dateOrder=dmy&delimiter=semicolon`, + ); + expect(res.status).toBe(200); + const args = mocks.importLabsCsv.mock.calls[0][0]; + expect(args.tz).toBe("Europe/Berlin"); + expect(args.query).toMatchObject({ + analyte: 0, + value: 1, + unit: 2, + date: 3, + decimal: "comma", + dateOrder: "dmy", + delimiter: "semicolon", + dryRun: false, + header: true, + }); + expect(args.stream).toBeInstanceOf(ReadableStream); + }); +}); diff --git a/src/app/api/import/labs-csv/route.ts b/src/app/api/import/labs-csv/route.ts new file mode 100644 index 000000000..b06ee3e34 --- /dev/null +++ b/src/app/api/import/labs-csv/route.ts @@ -0,0 +1,25 @@ +/** + * `POST /api/import/labs-csv` — lab results from a spreadsheet (#1215). + * + * Body: the CSV file, streamed. Query: the column mapping and the file's + * conventions (`labsCsvQuerySchema`). `dryRun=1` checks every row and reports + * what would happen without writing; the real import writes in chunks, skips + * readings the record already holds and reports every refused row by line. + */ +import { NextRequest } from "next/server"; + +import { apiHandler } from "@/lib/api-handler"; +import { importLabsCsv } from "@/lib/import/labs-csv"; +import { handleRecordImport } from "@/lib/import/record-import-route"; +import { labsCsvQuerySchema } from "@/lib/validations/record-import"; + +export const dynamic = "force-dynamic"; + +export const POST = apiHandler(async (request: NextRequest) => + handleRecordImport(request, { + kind: "labs", + module: "labs", + schema: labsCsvQuerySchema, + run: (args) => importLabsCsv(args), + }), +); diff --git a/src/app/api/import/medications-csv/route.ts b/src/app/api/import/medications-csv/route.ts new file mode 100644 index 000000000..324a3181e --- /dev/null +++ b/src/app/api/import/medications-csv/route.ts @@ -0,0 +1,25 @@ +/** + * `POST /api/import/medications-csv` — medications from a spreadsheet (#1215). + * + * Body: the CSV file, streamed. Query: the column mapping and the file's + * conventions (`medicationsCsvQuerySchema`). Maps only name, dose, daily + * times, as-needed and a start and end date; reminders stay off unless the + * query turns them on. `dryRun=1` checks without writing. + */ +import { NextRequest } from "next/server"; + +import { apiHandler } from "@/lib/api-handler"; +import { importMedicationsCsv } from "@/lib/import/medications-csv"; +import { handleRecordImport } from "@/lib/import/record-import-route"; +import { medicationsCsvQuerySchema } from "@/lib/validations/record-import"; + +export const dynamic = "force-dynamic"; + +export const POST = apiHandler(async (request: NextRequest) => + handleRecordImport(request, { + kind: "medications", + module: "medications", + schema: medicationsCsvQuerySchema, + run: (args) => importMedicationsCsv(args), + }), +); diff --git a/src/lib/http/proxy-bypass-routes.ts b/src/lib/http/proxy-bypass-routes.ts index 39434d3b5..339dc418f 100644 --- a/src/lib/http/proxy-bypass-routes.ts +++ b/src/lib/http/proxy-bypass-routes.ts @@ -42,6 +42,9 @@ export const PROXY_BYPASS_ROUTES: readonly string[] = [ // Bulk JSON / CSV imports, 16 MB. "/api/import", "/api/import/csv", + // Lab and medication CSV imports (v1.44), streamed, 16 MB. + "/api/import/labs-csv", + "/api/import/medications-csv", "/api/medications/intake/dose-history-import", // Lab scan upload, 12 MB. "/api/labs/ocr/extract", diff --git a/src/lib/import/__tests__/csv-stream.test.ts b/src/lib/import/__tests__/csv-stream.test.ts new file mode 100644 index 000000000..cbd4106c8 --- /dev/null +++ b/src/lib/import/__tests__/csv-stream.test.ts @@ -0,0 +1,227 @@ +/** + * The streaming CSV reader: the spreadsheet shapes, every chunk boundary, a + * fuzzed round trip, and a large file read with bounded memory. + */ +import { describe, expect, it } from "vitest"; + +import { + CsvTokenizer, + CsvTooLargeError, + MAX_COLUMNS, + MAX_FIELD_CHARS, + detectDelimiter, + readCsvRecords, + textToStream, + type CsvRecord, +} from "../csv-stream"; + +function all(text: string, delimiter: "auto" | "," | ";" | "\t" = "auto") { + const t = new CsvTokenizer(delimiter); + return [...t.push(text), ...t.end()]; +} + +/** Feed the text split at every position, one boundary at a time. */ +function everySplit(text: string): CsvRecord[][] { + const out: CsvRecord[][] = []; + for (let i = 0; i <= text.length; i++) { + const t = new CsvTokenizer(); + out.push([ + ...t.push(text.slice(0, i)), + ...t.push(text.slice(i)), + ...t.end(), + ]); + } + return out; +} + +const fields = (records: CsvRecord[]) => records.map((r) => r.fields); + +describe("detectDelimiter", () => { + it.each([ + ["a,b,c", ","], + ["a;b;c", ";"], + ["a\tb\tc", "\t"], + ['"x;y",b,c', ","], + ["single", ","], + ] as const)("%j → %j", (line, expected) => { + expect(detectDelimiter(line)).toBe(expected); + }); +}); + +describe("CsvTokenizer", () => { + it("drops a byte-order mark and reads a semicolon file with decimal commas", () => { + expect(fields(all("Analyt;Wert;Einheit\nHbA1c;5,4;%\n"))).toEqual([ + ["Analyt", "Wert", "Einheit"], + ["HbA1c", "5,4", "%"], + ]); + }); + + it("keeps separators, quotes and line breaks inside quoted fields", () => { + const text = 'a,b\n"x, y","say ""hi""\nthere"\n'; + expect(fields(all(text))).toEqual([ + ["a", "b"], + ["x, y", 'say "hi"\nthere'], + ]); + }); + + it("reads LF, CRLF and lone CR line ends and skips blank lines", () => { + expect(fields(all("a,b\r\n1,2\r3,4\n\n\r\n5,6"))).toEqual([ + ["a", "b"], + ["1", "2"], + ["3", "4"], + ["5", "6"], + ]); + }); + + it("reports the physical line each record starts on", () => { + const records = all('h\n"multi\nline"\n\nlast\n'); + expect(records.map((r) => r.line)).toEqual([1, 2, 5]); + }); + + it("keeps empty fields and a trailing empty field", () => { + expect(fields(all("a,,c,\n"))).toEqual([["a", "", "c", ""]]); + }); + + it("gives the same records whatever the chunk boundaries", () => { + const text = + 'name;dose;"note"\r\n"Ram;ipril";"5 mg";"two\r\nlines ""q"""\r\nX;1;\r'; + const expected = fields(all(text)); + for (const records of everySplit(text)) { + expect(fields(records)).toEqual(expected); + } + }); + + it("marks a runaway field instead of growing without end", () => { + const records = all(`a\n"${"x".repeat(MAX_FIELD_CHARS + 50)}\n`); + expect(records[1].oversized).toBe(true); + expect(records[1].fields[0].length).toBe(MAX_FIELD_CHARS); + }); + + it("marks a record wider than the column cap", () => { + const wide = Array.from({ length: MAX_COLUMNS + 5 }, (_, i) => + String(i), + ).join(","); + const [record] = all(`${wide}\n`); + expect(record.oversized).toBe(true); + expect(record.fields).toHaveLength(MAX_COLUMNS); + }); +}); + +/** Deterministic pseudo-random so a failure reproduces. */ +function rng(seed: number) { + let s = seed; + return () => { + s = (s * 1103515245 + 12345) & 0x7fffffff; + return s / 0x7fffffff; + }; +} + +function quoteField(value: string) { + return /[",;\t\r\n]/.test(value) || value === "" + ? `"${value.replace(/"/g, '""')}"` + : value; +} + +describe("fuzzed round trip", () => { + it("reads back exactly what was written, for random tables and chunkings", () => { + const alphabet = [ + "a", + "Z", + "1", + ",", + ";", + '"', + "\n", + "\r\n", + " ", + "ä", + "漢", + "\t", + ".", + ]; + const rand = rng(1215); + for (let round = 0; round < 300; round++) { + const delimiter = ([",", ";", "\t"] as const)[Math.floor(rand() * 3)]; + const cols = 1 + Math.floor(rand() * 5); + const rows = 1 + Math.floor(rand() * 6); + const table: string[][] = []; + for (let r = 0; r < rows; r++) { + const row: string[] = []; + for (let c = 0; c < cols; c++) { + let v = ""; + const len = Math.floor(rand() * 6); + for (let k = 0; k < len; k++) + v += alphabet[Math.floor(rand() * alphabet.length)]; + // A field that is all whitespace-free and quotable stays as written. + row.push(r === 0 ? `h${c}` : v); + } + // A record of only empty fields is skipped by design; keep one cell. + if (!row.some((f) => f.length > 0)) row[0] = "x"; + table.push(row); + } + const text = + table + .map((row) => row.map((f) => quoteField(f)).join(delimiter)) + .join(rand() < 0.5 ? "\n" : "\r\n") + (rand() < 0.5 ? "\n" : ""); + const t = new CsvTokenizer(delimiter); + const out: CsvRecord[] = []; + let i = 0; + while (i < text.length) { + const step = 1 + Math.floor(rand() * 7); + out.push(...t.push(text.slice(i, i + step))); + i += step; + } + out.push(...t.end()); + // Quoted line breaks come back as "\n": the reader normalises them. + const expected = table.map((row) => + row.map((f) => f.replace(/\r\n/g, "\n")), + ); + expect(fields(out), `round ${round}: ${JSON.stringify(text)}`).toEqual( + expected, + ); + } + }); +}); + +describe("readCsvRecords", () => { + it("streams a large file record by record", async () => { + const rows = 200_000; + const lines = ["analyte;value;unit;date"]; + for (let i = 0; i < rows; i++) + lines.push(`Marker ${i % 50};${i % 9},${i % 10};mg/dL;01.02.2024`); + const text = lines.join("\n"); + let count = 0; + let maxFieldLength = 0; + for await (const record of readCsvRecords(textToStream(text, 8192), { + maxBytes: 64 * 1024 * 1024, + })) { + count += 1; + for (const f of record.fields) + maxFieldLength = Math.max(maxFieldLength, f.length); + } + expect(count).toBe(rows + 1); + expect(maxFieldLength).toBeLessThan(20); + }); + + it("stops past the byte cap", async () => { + const text = "a,b\n" + "1,2\n".repeat(1000); + const consume = async () => { + for await (const _ of readCsvRecords(textToStream(text, 100), { + maxBytes: 500, + })) { + void _; + } + }; + await expect(consume()).rejects.toBeInstanceOf(CsvTooLargeError); + }); + + it("decodes a multi-byte character split across chunks", async () => { + const text = "name\nÄrztin 漢字\n"; + const out: string[] = []; + for await (const r of readCsvRecords(textToStream(text, 1), { + maxBytes: 1000, + })) + out.push(r.fields[0]); + expect(out).toEqual(["name", "Ärztin 漢字"]); + }); +}); diff --git a/src/lib/import/__tests__/csv-values.test.ts b/src/lib/import/__tests__/csv-values.test.ts new file mode 100644 index 000000000..d3c2ad27c --- /dev/null +++ b/src/lib/import/__tests__/csv-values.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; + +import { parseCsvDate, parseLocaleNumber, parseYesNo } from "../csv-values"; + +describe("parseLocaleNumber", () => { + it.each([ + ["5.4", "dot", 5.4], + ["5,4", "comma", 5.4], + ["-0.5", "dot", -0.5], + ["1,234.5", "dot", 1234.5], + ["1.234,5", "comma", 1234.5], + ["1,234,567", "dot", 1234567], + ["1.234.567", "comma", 1234567], + ["5,4", "dot", 5.4], + ["5.4", "comma", 5.4], + ["12", "comma", 12], + [" 7,25 ", "comma", 7.25], + ["1 234,5", "comma", 1234.5], + [".5", "dot", 0.5], + ] as const)("%j (%s) → %s", (raw, mark, value) => { + expect(parseLocaleNumber(raw, mark)).toEqual({ ok: true, value }); + }); + + it.each([ + ["1,234", "dot"], + ["1.234", "comma"], + ["1234,567", "dot"], + ] as const)("refuses %j (%s) as ambiguous", (raw, mark) => { + expect(parseLocaleNumber(raw, mark)).toEqual({ + ok: false, + reason: "ambiguous_number", + }); + }); + + it.each(["", "abc", "<0.5", "5..4", "1,2,3", "5.4.3", "1e5", "5,"])( + "refuses %j as not a number", + (raw) => { + expect(parseLocaleNumber(raw, "dot").ok).toBe(false); + }, + ); +}); + +describe("parseCsvDate", () => { + const berlin = { timezone: "Europe/Berlin" }; + + it("stores a date without a time on the date-only anchor", () => { + const r = parseCsvDate("2024-02-01", undefined, { + order: "iso", + ...berlin, + }); + expect(r).toMatchObject({ + ok: true, + dateKey: "2024-02-01", + dateOnly: true, + }); + expect(r.ok && r.instant.toISOString()).toBe("2024-02-01T12:00:00.000Z"); + }); + + it("reads a wall-clock time in the import's zone, across DST", () => { + const winter = parseCsvDate("15.01.2024", "08:30", { + order: "dmy", + ...berlin, + }); + const summer = parseCsvDate("15.07.2024 08:30", undefined, { + order: "dmy", + ...berlin, + }); + expect(winter.ok && winter.instant.toISOString()).toBe( + "2024-01-15T07:30:00.000Z", + ); + expect(summer.ok && summer.instant.toISOString()).toBe( + "2024-07-15T06:30:00.000Z", + ); + }); + + it("honours an explicit offset over the zone", () => { + const r = parseCsvDate("2024-07-15T08:30:00+02:00", undefined, { + order: "iso", + timezone: "America/New_York", + }); + expect(r.ok && r.instant.toISOString()).toBe("2024-07-15T06:30:00.000Z"); + }); + + it("reads US order and refuses a cell that cannot be in it", () => { + expect( + parseCsvDate("02/15/2024", undefined, { order: "mdy", ...berlin }), + ).toMatchObject({ ok: true, dateKey: "2024-02-15" }); + expect( + parseCsvDate("15/02/2024", undefined, { order: "mdy", ...berlin }), + ).toEqual({ ok: false, reason: "ambiguous_date" }); + }); + + it.each([ + ["31.02.2024", "dmy", "invalid_date"], + ["01.02.24", "dmy", "ambiguous_date"], + ["24-02-01", "iso", "ambiguous_date"], + ["Feb 1 2024", "iso", "invalid_date"], + ["", "iso", "invalid_date"], + ] as const)("%j (%s) → %s", (raw, order, reason) => { + expect(parseCsvDate(raw, undefined, { order, ...berlin })).toEqual({ + ok: false, + reason, + }); + }); + + it("refuses an impossible time", () => { + expect( + parseCsvDate("2024-02-01", "25:00", { order: "iso", ...berlin }), + ).toEqual({ ok: false, reason: "invalid_time" }); + }); +}); + +describe("parseYesNo", () => { + it("reads the usual spellings and nothing else", () => { + expect(parseYesNo("Ja")).toBe(true); + expect(parseYesNo("no")).toBe(false); + expect(parseYesNo("")).toBeNull(); + expect(parseYesNo("maybe")).toBeNull(); + }); +}); diff --git a/src/lib/import/__tests__/record-import-rows.test.ts b/src/lib/import/__tests__/record-import-rows.test.ts new file mode 100644 index 000000000..36507af4a --- /dev/null +++ b/src/lib/import/__tests__/record-import-rows.test.ts @@ -0,0 +1,269 @@ +/** + * Row normalisation of the lab and medication CSV imports (#1215): what a + * cell may mean, and the reason a row is refused when it could mean two + * things. Pure; the database half is in + * `tests/integration/record-csv-import.test.ts`. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { createMedicationSchema } from "@/lib/validations/medication"; +import { + labsCsvQuerySchema, + medicationsCsvQuerySchema, +} from "@/lib/validations/record-import"; + +import { normaliseLabRow, readingKeys } from "../labs-csv"; +import { + medicationCreateBody, + normaliseMedicationRow, + parseTimes, +} from "../medications-csv"; + +const NOW = new Date("2026-10-21T10:00:00.000Z"); +const labQuery = (extra: Record = {}) => + labsCsvQuerySchema.parse({ + analyte: "0", + value: "1", + unit: "2", + date: "3", + referenceRange: "4", + ...extra, + }); +const lab = (fields: string[], extra?: Record) => + normaliseLabRow({ line: 7, fields }, labQuery(extra), "Europe/Berlin", NOW); + +describe("lab rows", () => { + it("reads a numeric result with its unit, date and printed range", () => { + const r = lab(["HbA1c", "5.4", "%", "2025-03-01", "4.0-5.6"]); + expect(r).toMatchObject({ + ok: true, + row: { + line: 7, + analyte: "HbA1c", + value: 5.4, + valueText: null, + unit: "%", + sourceReferenceLow: 4, + sourceReferenceHigh: 5.6, + }, + }); + expect(r.ok && r.row.takenAt.toISOString()).toBe( + "2025-03-01T12:00:00.000Z", + ); + }); + + it("reads a German file: decimal comma, day-first dates, comma ranges", () => { + const r = lab(["Kreatinin", "0,92", "mg/dl", "01.03.2025", "0,7 - 1,2"], { + decimal: "comma", + dateOrder: "dmy", + }); + expect(r).toMatchObject({ + ok: true, + row: { value: 0.92, sourceReferenceLow: 0.7, sourceReferenceHigh: 1.2 }, + }); + }); + + it("keeps a non-numeric result as a qualitative reading", () => { + expect( + lab(["Urine protein", "negative", "", "2025-03-01", ""]), + ).toMatchObject({ + ok: true, + row: { value: null, valueText: "negative", unit: null }, + }); + expect(lab(["PSA", "<0.1", "ng/mL", "2025-03-01", ""])).toMatchObject({ + ok: true, + row: { valueText: "<0.1" }, + }); + }); + + it.each([ + [["", "5", "%", "2025-03-01", ""], "missing_value", "analyte"], + [["HbA1c", "", "%", "2025-03-01", ""], "missing_value", "value"], + [["HbA1c", "5.4", "", "2025-03-01", ""], "missing_unit", "unit"], + [ + ["Glucose", "1,234", "mg/dL", "2025-03-01", ""], + "ambiguous_number", + "value", + ], + [ + ["Glucose", "1.2.3", "mg/dL", "2025-03-01", ""], + "invalid_number", + "value", + ], + [["Glucose", "90", "mg/dL", "03/01/25", ""], "invalid_date", "date"], + [["Glucose", "90", "mg/dL", "2027-01-01", ""], "future_date", "date"], + ] as const)("refuses %j with %s on %s", (fields, reason, field) => { + expect(lab([...fields])).toEqual({ ok: false, reason, field }); + }); + + it("refuses a range whose low lies above its high", () => { + expect( + lab(["Glucose", "90", "mg/dL", "2025-03-01", "", "120", "70"], { + referenceRange: "4", + referenceLow: "5", + referenceHigh: "6", + }), + ).toEqual({ + ok: false, + reason: "invalid_reference_range", + field: "referenceRange", + }); + }); + + it("refuses a mapping that feeds two fields from one column", () => { + expect( + labsCsvQuerySchema.safeParse({ analyte: "0", value: "0", date: "1" }) + .success, + ).toBe(false); + }); + + it("keys a reading the way the scan commit's duplicate check does", () => { + const keys = readingKeys( + { + biomarkerId: "bm1", + analyte: "HbA1c", + takenAt: new Date("2025-03-01T12:00:00.000Z"), + value: 39, + valueText: null, + sourceValue: 5.7, + sourceUnit: "%", + }, + "Europe/Berlin", + ); + expect(keys).toEqual([ + "bm:bm1|2025-03-01|n:39", + "bm:bm1|2025-03-01|s:5.7:%", + ]); + }); +}); + +const medQuery = (extra: Record = {}) => + medicationsCsvQuerySchema.parse({ + name: "0", + dose: "1", + times: "2", + asNeeded: "3", + startsOn: "4", + endsOn: "5", + ...extra, + }); +const med = (fields: string[], extra?: Record) => + normaliseMedicationRow({ line: 3, fields }, medQuery(extra), "Europe/Berlin"); + +describe("medication rows", () => { + it.each([ + ["08:00", ["08:00"]], + ["20:00 8:00", ["08:00", "20:00"]], + ["8.00; 20.00|12:30", ["08:00", "12:30", "20:00"]], + ["", []], + ])("times %j → %j", (raw, expected) => { + expect(parseTimes(raw)).toEqual(expected); + }); + + it.each(["8", "25:00", "morning", "08:00 x"])("refuses times %j", (raw) => { + expect(parseTimes(raw)).toBeNull(); + }); + + it("reads a daily medication with a course", () => { + expect( + med(["Ramipril", "5 mg", "08:00", "", "01.02.2025", ""], { + dateOrder: "dmy", + }), + ).toEqual({ + ok: true, + row: { + line: 3, + name: "Ramipril", + dose: "5 mg", + times: ["08:00"], + asNeeded: false, + startsOn: "2025-02-01", + endsOn: null, + }, + }); + }); + + it.each([ + [["", "5 mg", "", "", "", ""], "missing_value", "name"], + [ + ["Ibuprofen", "400 mg", "08:00", "yes", "", ""], + "as_needed_with_times", + "times", + ], + [ + ["Ibuprofen", "400 mg", "", "sometimes", "", ""], + "invalid_yes_no", + "asNeeded", + ], + [ + ["X", "1", "08:00", "", "2025-03-01", "2025-02-01"], + "end_before_start", + "endsOn", + ], + [["Vitamin D", "1000 IE", "", "", "", ""], "missing_times", "times"], + ] as const)("refuses %j with %s", (fields, reason, field) => { + expect(med([...fields])).toEqual({ ok: false, reason, field }); + }); + + it("builds a body the create route's own schema accepts, reminders off by default", () => { + const daily = med(["Ramipril", "5 mg", "08:00 20:00", "", "", ""]); + const asNeeded = med(["Ibuprofen", "400 mg", "", "ja", "", ""]); + for (const r of [daily, asNeeded]) { + expect(r.ok).toBe(true); + if (!r.ok) continue; + const check = createMedicationSchema.safeParse( + medicationCreateBody(r.row, false), + ); + expect(check.success ? [] : check.error.issues).toEqual([]); + } + if (daily.ok) { + expect(medicationCreateBody(daily.row, false)).toMatchObject({ + notificationsEnabled: false, + schedules: [ + { + windowStart: "08:00", + windowEnd: "20:00", + timesOfDay: ["08:00", "20:00"], + }, + ], + }); + expect(medicationCreateBody(daily.row, true).notificationsEnabled).toBe( + true, + ); + } + if (asNeeded.ok) { + expect( + medicationCreateBody(asNeeded.row, true).notificationsEnabled, + ).toBe(false); + } + }); +}); + +describe("structure", () => { + const read = (p: string) => readFileSync(join(process.cwd(), p), "utf8"); + + it("writes field by field and never spreads a parsed row", () => { + for (const file of [ + "src/lib/import/labs-csv.ts", + "src/lib/import/medications-csv.ts", + ]) { + const src = read(file); + expect(src).not.toMatch(/data:\s*\{\s*\.\.\.(row|parsed|data)\b/); + expect(src).not.toMatch(/\.\.\.parsed\.data/); + } + }); + + it("checks every medication row with the create route's schema", () => { + expect(read("src/lib/import/medications-csv.ts")).toContain( + "createMedicationSchema.safeParse(", + ); + }); + + it("streams the body instead of reading it whole", () => { + const src = read("src/lib/import/record-import-route.ts"); + expect(src).toContain("stream: request.body"); + expect(src).not.toMatch(/request\.(text|json|arrayBuffer)\(/); + }); +}); diff --git a/src/lib/import/csv-stream.ts b/src/lib/import/csv-stream.ts new file mode 100644 index 000000000..cd95011d5 --- /dev/null +++ b/src/lib/import/csv-stream.ts @@ -0,0 +1,267 @@ +/** + * A streaming CSV reader for the record imports (#1215). + * + * The measurement importer reads its whole body into one string and splits + * it. That is fine for its 16 MB cap, but a lab or medication history kept + * over years in a spreadsheet arrives as one upload, and the import must hold + * a bounded amount of it at a time. This reader consumes the body chunk by + * chunk and hands out one record at a time; at any moment it holds the bytes + * of the record it is building and nothing before it. + * + * RFC 4180 plus what spreadsheets actually write: + * - a UTF-8 byte-order mark at the start is dropped; + * - the separator is `,`, `;` or a tab, detected from the header line + * outside quotes unless the caller names one (a German export writes `;` + * because `,` is its decimal mark); + * - fields may be quoted with `"`, a doubled `""` inside is one quote, and + * a quoted field may hold the separator and line breaks; + * - lines end in `\n`, `\r\n` or a lone `\r`; + * - a record of only empty fields (a trailing blank line) is skipped. + * + * Bounds: the body is counted while it is read and refused past `maxBytes`; + * a field longer than `MAX_FIELD_CHARS` or a record wider than + * `MAX_COLUMNS` marks that record `oversized` instead of growing without end + * (one runaway quote would otherwise swallow the rest of the file). + */ + +export type CsvDelimiter = "," | ";" | "\t"; + +export const MAX_FIELD_CHARS = 4000; +export const MAX_COLUMNS = 100; + +export interface CsvRecord { + /** 1-based physical line the record starts on. */ + line: number; + fields: string[]; + /** A field or the record ran past the bounds; its fields are cut short. */ + oversized: boolean; +} + +/** Count separators outside quotes in a header line; the most frequent wins. */ +export function detectDelimiter(headerLine: string): CsvDelimiter { + const counts: Record = { ",": 0, ";": 0, "\t": 0 }; + let quoted = false; + for (const ch of headerLine) { + if (ch === '"') quoted = !quoted; + else if (!quoted && (ch === "," || ch === ";" || ch === "\t")) { + counts[ch] += 1; + } + } + // Ties go to the comma, then the semicolon: a one-column file is a comma file. + let best: CsvDelimiter = ","; + for (const d of [";", "\t"] as const) { + if (counts[d] > counts[best]) best = d; + } + return best; +} + +/** + * The incremental tokenizer. `push` takes the next piece of text and returns + * the records it completed; `end` flushes the last one. Pure and synchronous, + * so it is fuzzed without streams. + */ +export class CsvTokenizer { + private delimiter: CsvDelimiter | null; + private field = ""; + private fields: string[] = []; + private quoted = false; + /** The previous character closed a quote, or was a quote inside quotes. */ + private quotePending = false; + private afterCr = false; + private line = 1; + private recordLine = 1; + private oversized = false; + private started = false; + /** Text held back until the header line shows the separator. */ + private pending = ""; + + constructor(delimiter: CsvDelimiter | "auto" = "auto") { + this.delimiter = delimiter === "auto" ? null : delimiter; + } + + get detectedDelimiter(): CsvDelimiter | null { + return this.delimiter; + } + + push(text: string): CsvRecord[] { + if (!this.started && text.length > 0) { + this.started = true; + if (text.charCodeAt(0) === 0xfeff) text = text.slice(1); + } + if (this.delimiter === null) { + this.pending += text; + const end = firstLineEnd(this.pending); + if (end === -1) { + // Bound the hold-back too: a header that never ends is a broken file. + if (this.pending.length <= MAX_FIELD_CHARS * MAX_COLUMNS) return []; + this.delimiter = detectDelimiter(this.pending); + } else { + this.delimiter = detectDelimiter(this.pending.slice(0, end)); + } + text = this.pending; + this.pending = ""; + } + const out: CsvRecord[] = []; + for (let i = 0; i < text.length; i++) { + this.consume(text[i], out); + } + return out; + } + + end(): CsvRecord[] { + const out: CsvRecord[] = []; + if (this.delimiter === null) { + const held = this.pending; + this.pending = ""; + this.delimiter = detectDelimiter(held); + for (const ch of held) this.consume(ch, out); + } + if (this.quotePending) { + // A closing quote right at the end of the input. + this.quoted = false; + this.quotePending = false; + } + if (this.field !== "" || this.fields.length > 0) this.finishRecord(out); + return out; + } + + private append(ch: string) { + if (this.field.length < MAX_FIELD_CHARS) this.field += ch; + else this.oversized = true; + } + + private finishField() { + if (this.fields.length < MAX_COLUMNS) this.fields.push(this.field); + else this.oversized = true; + this.field = ""; + } + + private finishRecord(out: CsvRecord[]) { + this.finishField(); + const blank = this.fields.every((f) => f === ""); + if (!blank) { + out.push({ + line: this.recordLine, + fields: this.fields, + oversized: this.oversized, + }); + } + this.fields = []; + this.oversized = false; + } + + private consume(ch: string, out: CsvRecord[]) { + // A `\n` straight after a `\r` belongs to the same line end. + if (this.afterCr) { + this.afterCr = false; + if (ch === "\n") return; + } + if (this.quoted) { + if (this.quotePending) { + this.quotePending = false; + if (ch === '"') { + this.append('"'); + return; + } + // The quote closed the field; fall through to unquoted handling. + this.quoted = false; + } else if (ch === '"') { + this.quotePending = true; + return; + } else { + if (ch === "\n" || ch === "\r") { + this.line += 1; + if (ch === "\r") this.afterCr = true; + this.append("\n"); + return; + } + this.append(ch); + return; + } + } + if (ch === this.delimiter) { + this.finishField(); + return; + } + if (ch === "\n" || ch === "\r") { + this.finishRecord(out); + this.line += 1; + this.recordLine = this.line; + if (ch === "\r") this.afterCr = true; + return; + } + if (ch === '"' && this.field === "") { + this.quoted = true; + return; + } + this.append(ch); + } +} + +function firstLineEnd(text: string): number { + let quoted = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (ch === '"') quoted = !quoted; + else if (!quoted && (ch === "\n" || ch === "\r")) return i; + } + return -1; +} + +export class CsvTooLargeError extends Error { + constructor(readonly maxBytes: number) { + super(`CSV exceeds ${maxBytes} bytes`); + this.name = "CsvTooLargeError"; + } +} + +/** + * Read a byte stream as CSV records, one at a time. The bytes are counted as + * they arrive and the read stops with `CsvTooLargeError` past `maxBytes`, so + * a body without a Content-Length cannot run past the cap either. + */ +export async function* readCsvRecords( + stream: ReadableStream, + opts: { maxBytes: number; delimiter?: CsvDelimiter | "auto" }, +): AsyncGenerator { + const tokenizer = new CsvTokenizer(opts.delimiter ?? "auto"); + const decoder = new TextDecoder("utf-8"); + const reader = stream.getReader(); + let bytes = 0; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (bytes > opts.maxBytes) throw new CsvTooLargeError(opts.maxBytes); + for (const record of tokenizer.push( + decoder.decode(value, { stream: true }), + )) { + yield record; + } + } + for (const record of tokenizer.push(decoder.decode())) yield record; + for (const record of tokenizer.end()) yield record; + } finally { + reader.releaseLock(); + } +} + +/** A string as a one-chunk-at-a-time stream; for tests and the client preview. */ +export function textToStream( + text: string, + chunkSize = 64 * 1024, +): ReadableStream { + const bytes = new TextEncoder().encode(text); + let offset = 0; + return new ReadableStream({ + pull(controller) { + if (offset >= bytes.length) { + controller.close(); + return; + } + controller.enqueue(bytes.slice(offset, offset + chunkSize)); + offset += chunkSize; + }, + }); +} diff --git a/src/lib/import/csv-values.ts b/src/lib/import/csv-values.ts new file mode 100644 index 000000000..d48ded1ec --- /dev/null +++ b/src/lib/import/csv-values.ts @@ -0,0 +1,230 @@ +/** + * Reading numbers and dates out of spreadsheet cells (#1215). + * + * Conservative by rule: a cell that could mean two things is refused with + * its own reason rather than read one way. The person sees the line and the + * reason and fixes the file or the setting; a wrong number stored quietly is + * the one outcome an import must never have. + */ +import { zonedWallClockToUtc } from "@/lib/tz/wall-clock"; +import { isCalendarDateKey } from "@/lib/tz/date-only"; + +export type DecimalMark = "dot" | "comma"; +export type DateOrder = "iso" | "dmy" | "mdy"; + +export type NumberResult = + | { ok: true; value: number } + | { ok: false; reason: "invalid_number" | "ambiguous_number" }; + +/** + * A number written with the file's decimal mark. Grouping separators are + * accepted only in the unmistakable form (`1.234,5` with a comma mark, + * `1,234.5` with a dot mark). A lone grouping-shaped number in the other + * convention (`1,234` in a dot file, `1.234` in a comma file) is ambiguous: + * it is a thousand and something in one country and one point something in + * the next, and lab values live exactly in that range. + */ +export function parseLocaleNumber( + raw: string, + mark: DecimalMark, +): NumberResult { + let s = raw.trim().replace(/[   ]/g, ""); + let sign = 1; + if (s.startsWith("-") || s.startsWith("−")) { + sign = -1; + s = s.slice(1); + } else if (s.startsWith("+")) { + s = s.slice(1); + } + if (!/^[\d.,]+$/.test(s)) return { ok: false, reason: "invalid_number" }; + const dec = mark === "comma" ? "," : "."; + const grp = mark === "comma" ? "." : ","; + const e = (c: string) => `\\${c}`; + const read = (digits: string) => { + const value = sign * Number(digits); + return Number.isFinite(value) + ? ({ ok: true, value } as const) + : ({ ok: false, reason: "invalid_number" } as const); + }; + + // Only the file's own decimal mark, at most once. + if (new RegExp(`^(\\d+(${e(dec)}\\d+)?|${e(dec)}\\d+)$`).test(s)) { + return read(s.replace(dec, ".")); + } + // Grouped in the file's own convention: 1,234,567 / 1,234.5 (dot file). + if (new RegExp(`^\\d{1,3}(${e(grp)}\\d{3})+(${e(dec)}\\d+)?$`).test(s)) { + const groups = s.split(grp).length - 1; + // One group and no decimal part reads as the other convention's decimal + // too (1,234 = one point two three four in a comma country). + if (groups === 1 && !s.includes(dec)) { + return { ok: false, reason: "ambiguous_number" }; + } + return read(s.split(grp).join("").replace(dec, ".")); + } + // One mark of the other convention with not exactly three digits after it + // can only be a decimal mark (5,4 in a dot file, 5.4 in a comma file). + const other = new RegExp(`^\\d+${e(grp)}(\\d+)$`).exec(s); + if (other) { + if (other[1].length === 3) return { ok: false, reason: "ambiguous_number" }; + return read(s.replace(grp, ".")); + } + return { ok: false, reason: "invalid_number" }; +} + +export type DateResult = + | { + ok: true; + /** The instant to store. */ + instant: Date; + /** The calendar date the cell states. */ + dateKey: string; + /** The cell gave a date without a time of day. */ + dateOnly: boolean; + } + | { + ok: false; + reason: "invalid_date" | "invalid_time" | "ambiguous_date"; + }; + +const pad = (n: number) => String(n).padStart(2, "0"); + +/** + * A date cell (and an optional separate time cell) in the stated order. + * + * - `iso`: `YYYY-MM-DD`, optionally with `T`/space and `HH:mm[:ss]` and an + * offset (`Z`, `+02:00`), the shape every export tool can write; + * - `dmy`: `DD.MM.YYYY`, `DD/MM/YYYY` or `DD-MM-YYYY`; + * - `mdy`: `MM/DD/YYYY`. + * + * A time without an offset is a wall-clock time in `timezone` (the account's + * zone unless the import names another). A date without a time is stored as + * the date-only anchor the scanned and document readings use (noon UTC), so + * it is the same calendar date for everyone who opens the record. + * + * A two-digit year is refused as ambiguous. So is a `dmy`/`mdy` cell whose + * first number cannot be what the order says (13/02/2026 under `mdy`): + * that file is in the other order, and guessing it per row would mix both. + */ +export function parseCsvDate( + rawDate: string, + rawTime: string | undefined, + opts: { order: DateOrder; timezone: string }, +): DateResult { + const cell = rawDate.trim(); + if (cell === "") return { ok: false, reason: "invalid_date" }; + let y: number; + let m: number; + let d: number; + let timePart: string | undefined; + let offset: string | undefined; + + if (opts.order === "iso") { + const match = + /^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?)\s*(Z|[+-]\d{2}:?\d{2})?)?$/i.exec( + cell, + ); + if (!match) { + return /^\d{2}-\d{2}-\d{2}$/.test(cell) + ? { ok: false, reason: "ambiguous_date" } + : { ok: false, reason: "invalid_date" }; + } + y = Number(match[1]); + m = Number(match[2]); + d = Number(match[3]); + timePart = match[4]; + offset = match[5]; + } else { + const match = + /^(\d{1,2})[./-](\d{1,2})[./-](\d{2}|\d{4})(?:[ T](\d{1,2}:\d{2}(?::\d{2})?))?$/.exec( + cell, + ); + if (!match) return { ok: false, reason: "invalid_date" }; + if (match[3].length === 2) return { ok: false, reason: "ambiguous_date" }; + const first = Number(match[1]); + const second = Number(match[2]); + y = Number(match[3]); + [d, m] = opts.order === "dmy" ? [first, second] : [second, first]; + if (m > 12 && d <= 12) return { ok: false, reason: "ambiguous_date" }; + timePart = match[4]; + } + + const dateKey = `${String(y).padStart(4, "0")}-${pad(m)}-${pad(d)}`; + if (!isCalendarDateKey(dateKey)) return { ok: false, reason: "invalid_date" }; + + const time = (rawTime ?? "").trim() || timePart; + if (!time) { + return { + ok: true, + instant: new Date(`${dateKey}T12:00:00.000Z`), + dateKey, + dateOnly: true, + }; + } + const tm = /^(\d{1,2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?$/.exec(time); + if (!tm) return { ok: false, reason: "invalid_time" }; + const hour = Number(tm[1]); + const minute = Number(tm[2]); + const second = Number(tm[3] ?? 0); + if (hour > 23 || minute > 59 || second > 59) { + return { ok: false, reason: "invalid_time" }; + } + if (offset) { + const norm = + offset.toUpperCase() === "Z" + ? "Z" + : offset.replace(/^([+-]\d{2})(\d{2})$/, "$1:$2"); + const instant = new Date( + `${dateKey}T${pad(hour)}:${pad(minute)}:${pad(second)}${norm}`, + ); + if (Number.isNaN(instant.getTime())) + return { ok: false, reason: "invalid_time" }; + return { ok: true, instant, dateKey, dateOnly: false }; + } + const instant = zonedWallClockToUtc( + { year: y, month: m, day: d, hour, minute, second }, + opts.timezone, + ); + return { ok: true, instant, dateKey, dateOnly: false }; +} + +/** A yes/no cell in the languages HealthLog speaks; anything else is null. */ +export function parseYesNo(raw: string): boolean | null { + const v = raw.trim().toLowerCase(); + if (v === "") return null; + if ( + [ + "1", + "yes", + "y", + "true", + "x", + "ja", + "j", + "sí", + "si", + "oui", + "tak", + "예", + "wahr", + ].includes(v) + ) { + return true; + } + if ( + [ + "0", + "no", + "n", + "false", + "nein", + "non", + "nie", + "nee", + "아니오", + "falsch", + ].includes(v) + ) { + return false; + } + return null; +} diff --git a/src/lib/import/labs-csv.ts b/src/lib/import/labs-csv.ts new file mode 100644 index 000000000..86e4b202c --- /dev/null +++ b/src/lib/import/labs-csv.ts @@ -0,0 +1,552 @@ +/** + * CSV import of lab results (#1215). + * + * A row becomes a lab reading the way a scanned report's confirmed row does + * (`/api/labs/ocr/commit`): the name resolves to the account's marker by the + * same index (name, alias, catalog), a marker is minted from the first row + * that names a new analyte, a reading in another unit is converted when the + * conversion is known and kept beside what the file said, and a reading the + * record already holds is skipped. Nothing about a row is guessed: an + * ambiguous number or date, a numeric value without a unit, or a unit that + * does not convert into the marker's is refused with its reason and line. + * + * Memory: the file is read record by record (`readCsvRecords`) and written in + * chunks of `CHUNK_ROWS`; what is held across chunks is the marker index, the + * duplicate keys of rows already accepted and the bounded report. + */ +import { prisma } from "@/lib/db"; +import { + readCsvRecords, + type CsvDelimiter, + type CsvRecord, +} from "@/lib/import/csv-stream"; +import { parseCsvDate, parseLocaleNumber } from "@/lib/import/csv-values"; +import { RecordImportReport } from "@/lib/import/record-import-report"; +import { matchAnalyteKey } from "@/lib/labs/analyte-match"; +import { + addToIndex, + loadMarkerIndex, + resolveInIndex, + unitFitsMarker, + type IndexedMarker, + type MarkerIndex, +} from "@/lib/labs/biomarker-resolve"; +import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store"; +import { parseReferenceRange } from "@/lib/labs/parse-reference-range"; +import { readingInMarkerUnit } from "@/lib/labs/reading-conversion"; +import { + labReadingDay, + labReadingDaySearchRange, +} from "@/lib/labs/reading-day"; +import { encryptNoteToBytes } from "@/lib/labs/store"; +import type { + LabsCsvQuery, + RecordImportResult, + RecordImportRowReason, +} from "@/lib/validations/record-import"; + +export const LABS_CSV_MAX_ROWS = 20_000; +const CHUNK_ROWS = 200; +const MAX_LISTED_MARKERS = 200; +/** A reading dated more than a day ahead is a typing mistake, not a result. */ +const FUTURE_SLACK_MS = 86_400_000; +const EARLIEST = Date.UTC(1900, 0, 1); + +export const LAB_TEXT_LIMITS = { + analyte: 200, + valueText: 200, + unit: 80, + panel: 100, + note: 1000, +} as const; + +export interface LabCsvRow { + line: number; + analyte: string; + value: number | null; + valueText: string | null; + unit: string | null; + takenAt: Date; + sourceReferenceLow: number | null; + sourceReferenceHigh: number | null; + sourceReferenceText: string | null; + panel: string | null; + note: string | null; +} + +export type LabRowResult = + | { ok: true; row: LabCsvRow } + | { ok: false; reason: RecordImportRowReason; field: string }; + +function cell(fields: string[], index: number | undefined): string { + if (index === undefined) return ""; + return (fields[index] ?? "").trim(); +} + +/** One CSV record → a lab row, or the first reason it cannot be one. Pure. */ +export function normaliseLabRow( + record: Pick, + query: LabsCsvQuery, + timezone: string, + now: Date = new Date(), +): LabRowResult { + const f = record.fields; + const fail = ( + reason: RecordImportRowReason, + field: string, + ): LabRowResult => ({ ok: false, reason, field }); + + const analyte = cell(f, query.analyte); + if (!analyte) return fail("missing_value", "analyte"); + if (analyte.length > LAB_TEXT_LIMITS.analyte) + return fail("text_too_long", "analyte"); + + const rawValue = cell(f, query.value); + if (!rawValue) return fail("missing_value", "value"); + let value: number | null = null; + let valueText: string | null = null; + const parsed = parseLocaleNumber(rawValue, query.decimal); + if (parsed.ok) { + value = parsed.value; + } else if (parsed.reason === "ambiguous_number") { + return fail("ambiguous_number", "value"); + } else if (/^[+-]?[\d.,\s]+$/.test(rawValue)) { + // Digits and marks that still do not read as a number: a broken number, + // not a qualitative result. + return fail("invalid_number", "value"); + } else { + if (rawValue.length > LAB_TEXT_LIMITS.valueText) + return fail("text_too_long", "value"); + valueText = rawValue; + } + + const unit = cell(f, query.unit) || null; + if (value !== null && !unit) return fail("missing_unit", "unit"); + if (unit && unit.length > LAB_TEXT_LIMITS.unit) + return fail("text_too_long", "unit"); + + const date = parseCsvDate( + cell(f, query.date), + query.time === undefined ? undefined : cell(f, query.time), + { + order: query.dateOrder, + timezone, + }, + ); + if (!date.ok) return fail(date.reason, "date"); + if (date.instant.getTime() > now.getTime() + FUTURE_SLACK_MS) + return fail("future_date", "date"); + if (date.instant.getTime() < EARLIEST) return fail("invalid_date", "date"); + + let low: number | null = null; + let high: number | null = null; + let rangeText: string | null = null; + for (const [key, index] of [ + ["referenceLow", query.referenceLow], + ["referenceHigh", query.referenceHigh], + ] as const) { + const raw = cell(f, index); + if (!raw) continue; + const n = parseLocaleNumber(raw, query.decimal); + if (!n.ok) + return fail( + n.reason === "ambiguous_number" + ? "ambiguous_number" + : "invalid_reference_range", + key, + ); + if (key === "referenceLow") low = n.value; + else high = n.value; + } + const rawRange = cell(f, query.referenceRange); + if (rawRange) { + // A comma-decimal file writes ranges as "4,0 - 5,6"; the range reader + // speaks the dot, so the marks are swapped first for such a file. + const forReader = + query.decimal === "comma" + ? rawRange.replace(/(\d),(\d)/g, "$1.$2") + : rawRange; + const printed = parseReferenceRange(forReader, unit); + if (printed) { + low ??= printed.low; + high ??= printed.high; + rangeText = rawRange.replace(/\s+/g, " ").slice(0, 120); + } + } + if (low !== null && high !== null && low > high) { + return fail("invalid_reference_range", "referenceRange"); + } + + const panel = cell(f, query.panel) || null; + if (panel && panel.length > LAB_TEXT_LIMITS.panel) + return fail("text_too_long", "panel"); + const note = cell(f, query.note) || null; + if (note && note.length > LAB_TEXT_LIMITS.note) + return fail("text_too_long", "note"); + + return { + ok: true, + row: { + line: record.line, + analyte, + value, + valueText, + unit, + takenAt: date.instant, + sourceReferenceLow: valueText === null ? low : null, + sourceReferenceHigh: valueText === null ? high : null, + sourceReferenceText: valueText === null ? rangeText : null, + panel, + note, + }, + }; +} + +/** + * The keys a reading is a duplicate under, mirroring the scan commit's + * `isDuplicate`: the marker (or the name, for a reading filed without one), + * the reading's calendar day, and the value as stored, as printed, or as text. + */ +export function readingKeys( + reading: { + biomarkerId: string | null; + analyte: string; + takenAt: Date; + value: number | null; + valueText: string | null; + sourceValue?: number | null; + sourceUnit?: string | null; + unit?: string | null; + }, + tz: string, +): string[] { + const who = reading.biomarkerId + ? `bm:${reading.biomarkerId}` + : `an:${reading.analyte.trim().toLowerCase()}`; + const day = labReadingDay(reading.takenAt, tz); + const keys: string[] = []; + if (reading.value !== null) keys.push(`${who}|${day}|n:${reading.value}`); + if ( + reading.sourceValue !== null && + reading.sourceValue !== undefined && + reading.sourceUnit + ) { + keys.push( + `${who}|${day}|s:${reading.sourceValue}:${reading.sourceUnit.trim().toLowerCase()}`, + ); + } + if (reading.valueText !== null) { + keys.push(`${who}|${day}|t:${reading.valueText.trim().toLowerCase()}`); + } + return keys; +} + +interface Planned { + row: LabCsvRow; + marker: IndexedMarker | null; +} + +function displayRow(row: LabCsvRow, marker: string) { + return { + line: row.line, + marker, + value: row.value, + valueText: row.valueText, + unit: row.unit, + takenAt: row.takenAt.toISOString(), + referenceLow: row.sourceReferenceLow, + referenceHigh: row.sourceReferenceHigh, + panel: row.panel, + note: row.note !== null, + }; +} + +/** + * Read, check and (unless a dry run) write a lab CSV. Throws + * `CsvTooLargeError` past the byte cap; everything else is reported per row. + */ +export async function importLabsCsv(args: { + userId: string; + tz: string; + stream: ReadableStream; + maxBytes: number; + query: LabsCsvQuery; + now?: Date; +}): Promise { + const { userId, tz, query } = args; + const dryRun = query.dryRun; + const timezone = query.timezone ?? tz; + const report = new RecordImportReport("labs", dryRun); + const index: MarkerIndex = await loadMarkerIndex(prisma, userId); + // A dry run works on a copy, so the markers it pretends to mint never leak. + const working: MarkerIndex = dryRun + ? { markers: [...index.markers], aliases: [...index.aliases] } + : index; + const matched = new Map< + string, + { name: string; marker: string; how: "name" | "alias" | "catalog" } + >(); + const created = new Set(); + const seen = new Set(); + let chunk: LabCsvRow[] = []; + let pendingId = 0; + + const delimiter: CsvDelimiter | "auto" = + query.delimiter === "comma" + ? "," + : query.delimiter === "semicolon" + ? ";" + : query.delimiter === "tab" + ? "\t" + : "auto"; + + const flush = async () => { + if (chunk.length === 0) return; + const rows = chunk; + chunk = []; + + // 1. Resolve every row's marker; a dry run pretends to mint. + const planned: Planned[] = []; + for (const row of rows) { + const hit = resolveInIndex( + working, + row.analyte, + row.valueText === null ? row.unit : null, + ); + let marker: IndexedMarker | null = hit?.marker ?? null; + if (hit) { + // Listed once per spelling, so the person sees which of the file's + // names landed on which of their markers. + const key = row.analyte.toLowerCase(); + if (!matched.has(key) && matched.size < MAX_LISTED_MARKERS) { + matched.set(key, { + name: row.analyte, + marker: hit.marker.name, + how: hit.matchKind, + }); + } + } else if (dryRun) { + const unit = row.unit ?? ""; + marker = { + id: `pending:${pendingId++}`, + name: row.analyte, + unit, + lowerBound: row.sourceReferenceLow, + upperBound: row.sourceReferenceHigh, + panel: row.panel, + analyteKey: matchAnalyteKey({ name: row.analyte, unit }), + hidden: false, + createdAt: new Date(), + }; + addToIndex(working, marker); + if (created.size < MAX_LISTED_MARKERS) created.add(row.analyte); + } + // The unit must fit the marker, or convert into it. + if ( + marker && + row.value !== null && + // The key as the write derives it (`ensureAnalyteKey`), so the check + // and the conversion the import then runs cannot disagree. + !unitFitsMarker( + { + unit: marker.unit, + analyteKey: + matchAnalyteKey({ name: marker.name, unit: marker.unit }) ?? + marker.analyteKey, + }, + row.unit, + null, + ) + ) { + report.fail(row.line, "unit_mismatch", "unit"); + continue; + } + planned.push({ row, marker }); + } + + // 2. Duplicates against the record, one query for the chunk. + const days = planned.map((p) => labReadingDay(p.row.takenAt, tz)).sort(); + const existingKeys = new Set(); + if (planned.length > 0) { + const from = labReadingDaySearchRange(days[0]).gte; + const to = labReadingDaySearchRange(days[days.length - 1]).lt; + const ids = [ + ...new Set( + planned + .map((p) => p.marker?.id) + .filter((id): id is string => !!id && !id.startsWith("pending:")), + ), + ]; + const names = [...new Set(planned.map((p) => p.row.analyte))]; + const existing = await prisma.labResult.findMany({ + where: { + userId, + deletedAt: null, + takenAt: { gte: from, lt: to }, + OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }], + }, + select: { + biomarkerId: true, + analyte: true, + takenAt: true, + value: true, + valueText: true, + sourceValue: true, + sourceUnit: true, + }, + }); + for (const e of existing) { + for (const k of readingKeys(e, tz)) existingKeys.add(k); + // A reading filed under a marker also answers to its printed name. + if (e.biomarkerId) { + for (const k of readingKeys({ ...e, biomarkerId: null }, tz)) + existingKeys.add(k); + } + } + } + + // 3. Write (or count) what is new. + for (const { row, marker } of planned) { + const probe = { + biomarkerId: + marker && !marker.id.startsWith("pending:") ? marker.id : null, + analyte: row.analyte, + takenAt: row.takenAt, + value: row.value, + valueText: row.valueText, + sourceValue: row.value, + sourceUnit: row.unit, + }; + const keys = [ + ...readingKeys(probe, tz), + ...readingKeys({ ...probe, biomarkerId: null }, tz), + ...(marker + ? readingKeys({ ...probe, biomarkerId: marker.id }, tz) + : []), + ]; + if (keys.some((k) => existingKeys.has(k) || seen.has(k))) { + report.duplicate(row.line); + continue; + } + + if (dryRun) { + for (const k of keys) seen.add(k); + report.valid += 1; + report.inserted += 1; + report.addPreview(displayRow(row, marker?.name ?? row.analyte)); + continue; + } + + const biomarker = + marker ?? + (await resolveOrMintBiomarker( + userId, + { + analyte: row.analyte, + unit: row.unit ?? "", + referenceLow: + row.valueText === null ? row.sourceReferenceLow : null, + referenceHigh: + row.valueText === null ? row.sourceReferenceHigh : null, + panel: row.panel, + }, + { index: working }, + )); + if (!marker) created.add(row.analyte); + + let storedValue = row.value; + let storedLow = row.sourceReferenceLow; + let storedHigh = row.sourceReferenceHigh; + let convertedFrom: { sourceValue: number; sourceUnit: string } | null = + null; + if (row.value !== null) { + const reading = await readingInMarkerUnit(prisma, { + marker: biomarker, + value: row.value, + unit: row.unit ?? undefined, + sourceReferenceLow: storedLow, + sourceReferenceHigh: storedHigh, + }); + if (!reading.ok) { + report.fail(row.line, "unit_mismatch", "unit"); + continue; + } + storedValue = reading.value; + storedLow = reading.sourceReferenceLow; + storedHigh = reading.sourceReferenceHigh; + if (reading.converted) { + convertedFrom = { + sourceValue: reading.sourceValue, + sourceUnit: reading.sourceUnit, + }; + } + } + + // Field by field, never a spread of the parsed row. + await prisma.labResult.create({ + data: { + userId, + biomarkerId: biomarker.id, + panel: row.panel ?? biomarker.panel, + analyte: biomarker.name, + value: storedValue, + valueText: row.valueText, + unit: biomarker.unit, + sourceValue: convertedFrom?.sourceValue ?? null, + sourceUnit: convertedFrom?.sourceUnit ?? null, + referenceLow: biomarker.lowerBound, + referenceHigh: biomarker.upperBound, + sourceReferenceLow: storedLow, + sourceReferenceHigh: storedHigh, + sourceReferenceText: row.sourceReferenceText, + takenAt: row.takenAt, + source: "IMPORT", + noteEncrypted: row.note ? encryptNoteToBytes(row.note) : null, + }, + }); + for (const k of [ + ...keys, + ...readingKeys({ ...probe, biomarkerId: biomarker.id }, tz), + ]) + seen.add(k); + report.valid += 1; + report.inserted += 1; + } + }; + + let first = true; + for await (const record of readCsvRecords(args.stream, { + maxBytes: args.maxBytes, + delimiter, + })) { + if (first && query.header) { + first = false; + continue; + } + first = false; + report.rows += 1; + if (report.rows > LABS_CSV_MAX_ROWS) { + report.fail(record.line, "row_limit"); + continue; + } + if (record.oversized) { + report.fail(record.line, "oversized_row"); + continue; + } + const result = normaliseLabRow(record, query, timezone, args.now); + if (!result.ok) { + report.fail(record.line, result.reason, result.field); + continue; + } + chunk.push(result.row); + if (chunk.length >= CHUNK_ROWS) await flush(); + } + await flush(); + + return report.toResult({ + markers: { + matched: [...matched.values()], + created: [...created].slice(0, MAX_LISTED_MARKERS), + }, + }); +} diff --git a/src/lib/import/medications-csv.ts b/src/lib/import/medications-csv.ts new file mode 100644 index 000000000..d616ce675 --- /dev/null +++ b/src/lib/import/medications-csv.ts @@ -0,0 +1,331 @@ +/** + * CSV import of medications (#1215). + * + * Conservative by design: a row maps only what is unambiguous in a + * spreadsheet and in the medication model alike. Name and dose (as written), + * daily intake times, an as-needed flag, and a start and end date. Every other + * rhythm (weekdays, every other week, cycles, injections) is set up in the + * app afterwards, where the person sees what it means. A row needs either + * times or the as-needed mark: the medication model holds no scheduled + * medication without a time, and an import does not invent one. + * + * Every row is checked against `createMedicationSchema`, the schema the + * create route parses with, so an import can never store a medication the + * form could not. Reminders stay off unless the import asks for them, so an + * upload never starts notifying on its own. A medication the record already + * holds under the same name and dose, or an earlier row of the file, is + * skipped. + */ +import { prisma } from "@/lib/db"; +import { invalidateUserMedications } from "@/lib/cache/invalidate"; +import { + readCsvRecords, + type CsvDelimiter, + type CsvRecord, +} from "@/lib/import/csv-stream"; +import { parseCsvDate, parseYesNo } from "@/lib/import/csv-values"; +import { RecordImportReport } from "@/lib/import/record-import-report"; +import { foldForMatch } from "@/lib/i18n/fold-for-match"; +import { setMedicationCategory } from "@/lib/medication-category"; +import { serializeScheduleRecurrence } from "@/lib/medication-schedule"; +import { + dateOfDayKey, + dayKeyOfDate, + validateCourses, +} from "@/lib/medications/course-window"; +import { userDayKey } from "@/lib/tz/format"; +import { createMedicationSchema } from "@/lib/validations/medication"; +import type { + MedicationsCsvQuery, + RecordImportResult, + RecordImportRowReason, +} from "@/lib/validations/record-import"; + +export const MEDICATIONS_CSV_MAX_ROWS = 500; + +export interface MedicationCsvRow { + line: number; + name: string; + dose: string; + times: string[]; + asNeeded: boolean; + startsOn: string | null; + endsOn: string | null; +} + +export type MedicationRowResult = + | { ok: true; row: MedicationCsvRow } + | { ok: false; reason: RecordImportRowReason; field: string }; + +function cell(fields: string[], index: number | undefined): string { + if (index === undefined) return ""; + return (fields[index] ?? "").trim(); +} + +/** `8:00`, `08:00 20:00`, `08:00|20:00`, `8.00; 20.00` → sorted `HH:mm`s. */ +export function parseTimes(raw: string): string[] | null { + if (raw === "") return []; + const parts = raw.split(/[\s,;|/]+/).filter(Boolean); + const out = new Set(); + for (const part of parts) { + const m = /^(\d{1,2})[:.](\d{2})$/.exec(part); + if (!m) return null; + const h = Number(m[1]); + const min = Number(m[2]); + if (h > 23 || min > 59) return null; + out.add(`${String(h).padStart(2, "0")}:${String(min).padStart(2, "0")}`); + } + const sorted = [...out].sort(); + return sorted.length > 8 ? null : sorted; +} + +/** One record → a medication row, or the first reason it cannot be one. Pure. */ +export function normaliseMedicationRow( + record: Pick, + query: MedicationsCsvQuery, + timezone: string, +): MedicationRowResult { + const f = record.fields; + const fail = ( + reason: RecordImportRowReason, + field: string, + ): MedicationRowResult => ({ ok: false, reason, field }); + + const name = cell(f, query.name); + if (!name) return fail("missing_value", "name"); + if (name.length > 100) return fail("text_too_long", "name"); + const dose = cell(f, query.dose); + if (!dose) return fail("missing_value", "dose"); + if (dose.length > 50) return fail("text_too_long", "dose"); + + const times = parseTimes(cell(f, query.times)); + if (times === null) return fail("invalid_times", "times"); + + let asNeeded = false; + const rawAsNeeded = cell(f, query.asNeeded); + if (rawAsNeeded) { + const yes = parseYesNo(rawAsNeeded); + if (yes === null) return fail("invalid_yes_no", "asNeeded"); + asNeeded = yes; + } + if (asNeeded && times.length > 0) + return fail("as_needed_with_times", "times"); + if (!asNeeded && times.length === 0) return fail("missing_times", "times"); + + const day = (field: "startsOn" | "endsOn") => { + const raw = cell(f, query[field]); + if (!raw) return { ok: true as const, key: null }; + const parsed = parseCsvDate(raw, undefined, { + order: query.dateOrder, + timezone, + }); + if (!parsed.ok) return { ok: false as const, reason: parsed.reason }; + return { ok: true as const, key: parsed.dateKey }; + }; + const start = day("startsOn"); + if (!start.ok) return fail(start.reason, "startsOn"); + const end = day("endsOn"); + if (!end.ok) return fail(end.reason, "endsOn"); + if (start.key && end.key && end.key < start.key) { + return fail("end_before_start", "endsOn"); + } + + return { + ok: true, + row: { + line: record.line, + name, + dose, + times, + asNeeded, + startsOn: start.key, + endsOn: end.key, + }, + }; +} + +/** The create body this row stands for, exactly as the form would send it. */ +export function medicationCreateBody( + row: MedicationCsvRow, + reminders: boolean, +) { + const windowEnd = (() => { + if (row.times.length > 1) return row.times[row.times.length - 1]; + const [h, m] = (row.times[0] ?? "08:00").split(":").map(Number); + return `${String((h + 1) % 24).padStart(2, "0")}:${String(m).padStart(2, "0")}`; + })(); + return { + name: row.name, + dose: row.dose, + asNeeded: row.asNeeded, + notificationsEnabled: reminders && row.times.length > 0, + ...(row.startsOn ? { startsOn: row.startsOn } : {}), + ...(row.endsOn ? { endsOn: row.endsOn } : {}), + ...(row.times.length > 0 + ? { + schedules: [ + { + windowStart: row.times[0], + windowEnd, + timesOfDay: row.times, + }, + ], + } + : {}), + }; +} + +const identity = (name: string, dose: string) => + `${foldForMatch(name)}|${foldForMatch(dose).replace(/\s+/g, "")}`; + +export async function importMedicationsCsv(args: { + userId: string; + tz: string; + stream: ReadableStream; + maxBytes: number; + query: MedicationsCsvQuery; + now?: Date; +}): Promise { + const { userId, tz, query } = args; + const dryRun = query.dryRun; + const timezone = query.timezone ?? tz; + const report = new RecordImportReport("medications", dryRun); + const todayKey = userDayKey(args.now ?? new Date(), tz); + + const existing = await prisma.medication.findMany({ + where: { userId }, + select: { name: true, dose: true }, + }); + const seen = new Set(existing.map((m) => identity(m.name, m.dose))); + + const delimiter: CsvDelimiter | "auto" = + query.delimiter === "comma" + ? "," + : query.delimiter === "semicolon" + ? ";" + : query.delimiter === "tab" + ? "\t" + : "auto"; + + let first = true; + for await (const record of readCsvRecords(args.stream, { + maxBytes: args.maxBytes, + delimiter, + })) { + if (first && query.header) { + first = false; + continue; + } + first = false; + report.rows += 1; + if (report.rows > MEDICATIONS_CSV_MAX_ROWS) { + report.fail(record.line, "row_limit"); + continue; + } + if (record.oversized) { + report.fail(record.line, "oversized_row"); + continue; + } + const result = normaliseMedicationRow(record, query, timezone); + if (!result.ok) { + report.fail(record.line, result.reason, result.field); + continue; + } + const row = result.row; + const parsed = createMedicationSchema.safeParse( + medicationCreateBody(row, query.reminders), + ); + if (!parsed.success) { + report.fail(row.line, "invalid_medication"); + continue; + } + // The course the create route would open for this window. + let course: { startsOn: Date; endsOn: Date | null } | null = null; + if (parsed.data.startsOn || parsed.data.endsOn) { + const endKey = parsed.data.endsOn + ? dayKeyOfDate(parsed.data.endsOn) + : null; + course = { + startsOn: + parsed.data.startsOn ?? + dateOfDayKey(endKey && endKey < todayKey ? endKey : todayKey), + endsOn: parsed.data.endsOn ?? null, + }; + if (validateCourses([course], todayKey, false)) { + report.fail(row.line, "end_before_start", "endsOn"); + continue; + } + } + const key = identity(row.name, row.dose); + if (seen.has(key)) { + report.duplicate(row.line); + continue; + } + seen.add(key); + report.valid += 1; + report.inserted += 1; + if (dryRun) { + report.addPreview({ + line: row.line, + name: row.name, + dose: row.dose, + times: row.times.join(" "), + asNeeded: row.asNeeded, + startsOn: row.startsOn, + endsOn: row.endsOn, + }); + continue; + } + + const data = parsed.data; + // Field by field, with the create route's invariants for the shapes an + // import produces: a daily schedule (FREQ=DAILY, times dual-written), an + // opening course for a dated window, no schedule for as-needed. + const medication = await prisma.medication.create({ + data: { + userId, + name: data.name, + dose: data.dose, + asNeeded: data.asNeeded ?? false, + notificationsEnabled: data.notificationsEnabled ?? false, + ...(course + ? { + startsOn: course.startsOn, + endsOn: course.endsOn, + courses: { + create: { + userId, + startsOn: course.startsOn, + endsOn: course.endsOn, + }, + }, + } + : {}), + schedules: { + create: (data.schedules ?? []).map((s) => ({ + windowStart: s.windowStart, + windowEnd: s.windowEnd, + label: null, + dose: null, + daysOfWeek: serializeScheduleRecurrence({ + daysOfWeek: [], + intervalWeeks: 1, + }), + timesOfDay: + s.timesOfDay && s.timesOfDay.length > 0 + ? s.timesOfDay + : [s.windowStart], + rrule: "FREQ=DAILY", + })), + }, + }, + select: { id: true }, + }); + await setMedicationCategory(medication.id, undefined); + } + + if (!dryRun && report.inserted > 0) { + invalidateUserMedications(userId, { evict: true }); + } + return report.toResult(); +} diff --git a/src/lib/import/record-import-report.ts b/src/lib/import/record-import-report.ts new file mode 100644 index 000000000..af797c690 --- /dev/null +++ b/src/lib/import/record-import-report.ts @@ -0,0 +1,75 @@ +/** + * The per-import report both record importers fill (#1215): counts, the row + * errors in file order and a short preview, each bounded so a file of twenty + * thousand broken rows answers with a report of a fixed size. + */ +import type { + RecordImportResult, + RecordImportRowReason, +} from "@/lib/validations/record-import"; + +export const MAX_REPORTED_ERRORS = 500; +export const MAX_PREVIEW_ROWS = 20; + +export type PreviewRow = Record; + +export class RecordImportReport { + rows = 0; + valid = 0; + inserted = 0; + duplicates = 0; + failed = 0; + private errors: RecordImportResult["errors"] = []; + private truncated = false; + private preview: PreviewRow[] = []; + + constructor( + readonly kind: RecordImportResult["kind"], + readonly dryRun: boolean, + ) {} + + private note(line: number, reason: RecordImportRowReason, field?: string) { + if (this.errors.length < MAX_REPORTED_ERRORS) { + this.errors.push(field ? { line, reason, field } : { line, reason }); + } else { + this.truncated = true; + } + } + + fail(line: number, reason: RecordImportRowReason, field?: string) { + this.failed += 1; + this.note(line, reason, field); + } + + duplicate(line: number) { + this.duplicates += 1; + this.note(line, "duplicate"); + } + + addPreview(row: PreviewRow) { + if (this.dryRun && this.preview.length < MAX_PREVIEW_ROWS) { + this.preview.push(row); + } + } + + toResult( + extra: Pick = {}, + ): RecordImportResult { + // Errors arrive per chunk; a chunk's duplicates are found after its + // refusals, so put the list back in file order. + const errors = [...this.errors].sort((a, b) => a.line - b.line); + return { + kind: this.kind, + dryRun: this.dryRun, + rows: this.rows, + valid: this.valid, + inserted: this.inserted, + duplicates: this.duplicates, + failed: this.failed, + errors, + errorsTruncated: this.truncated, + ...extra, + preview: this.preview, + }; + } +} diff --git a/src/lib/import/record-import-route.ts b/src/lib/import/record-import-route.ts new file mode 100644 index 000000000..ded29bc6e --- /dev/null +++ b/src/lib/import/record-import-route.ts @@ -0,0 +1,150 @@ +/** + * The request side the two CSV record imports share (#1215): owner-only auth, + * the module gate, the rate buckets, the query parse, the content type and + * the streamed body with its byte cap. Each route supplies its schema and its + * importer; what a row becomes lives in `labs-csv.ts` / `medications-csv.ts`. + * + * Buckets: a dry run spends `import-preview:` (30 an hour), because the + * mapping screen re-checks the file whenever a column choice changes; the + * real import spends the shared `import:` bucket (5 an hour) every other + * importer uses. + */ +import type { NextRequest } from "next/server"; +import type { z } from "zod/v4"; + +import { requireAuth } from "@/lib/api-handler"; +import { + apiError, + apiSuccess, + getClientIp, + returnAllZodIssues, +} from "@/lib/api-response"; +import { auditLog } from "@/lib/auth/audit"; +import { CsvTooLargeError } from "@/lib/import/csv-stream"; +import { annotate } from "@/lib/logging/context"; +import { requireModuleEnabled } from "@/lib/modules/gate"; +import type { ModuleKey } from "@/lib/modules/registry"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { resolveUserTimezone } from "@/lib/tz/resolver"; +import type { RecordImportResult } from "@/lib/validations/record-import"; + +export const RECORD_IMPORT_MAX_BYTES = 16 * 1024 * 1024; +const PREVIEW_PER_HOUR = 30; +const IMPORT_PER_HOUR = 5; +const HOUR_MS = 60 * 60 * 1000; + +export async function handleRecordImport< + S extends z.ZodType<{ dryRun: boolean }>, +>( + request: NextRequest, + opts: { + kind: RecordImportResult["kind"]; + module: ModuleKey; + schema: S; + run: (args: { + userId: string; + tz: string; + stream: ReadableStream; + maxBytes: number; + query: z.infer; + }) => Promise; + }, +): Promise { + const { user } = await requireAuth(); + const action = `import.${opts.kind}.csv`; + annotate({ action: { name: action } }); + + const gate = await requireModuleEnabled(user.id, opts.module); + if (!gate.enabled) return gate.response; + + const parsed = opts.schema.safeParse( + Object.fromEntries(new URL(request.url).searchParams), + ); + if (!parsed.success) return returnAllZodIssues(parsed.error, 422); + const query = parsed.data; + + const rl = query.dryRun + ? await checkRateLimit( + `import-preview:${user.id}`, + PREVIEW_PER_HOUR, + HOUR_MS, + ) + : await checkRateLimit(`import:${user.id}`, IMPORT_PER_HOUR, HOUR_MS); + if (!rl.allowed) { + return apiError( + query.dryRun + ? `Maximum ${PREVIEW_PER_HOUR} checks per hour` + : `Maximum ${IMPORT_PER_HOUR} imports per hour`, + 429, + { errorCode: "import.rate_limited" }, + ); + } + + const contentType = (request.headers.get("content-type") ?? "").toLowerCase(); + if ( + !contentType.includes("text/csv") && + !contentType.includes("text/plain") + ) { + return apiError("Content-Type must be text/csv", 415, { + errorCode: "import.csv.content_type", + }); + } + const declared = Number(request.headers.get("content-length") ?? "0"); + if (Number.isFinite(declared) && declared > RECORD_IMPORT_MAX_BYTES) { + return apiError("CSV exceeds the 16 MB limit", 413, { + errorCode: "import.csv.too_large", + }); + } + if (!request.body) { + return apiError("The CSV file is empty", 422, { + errorCode: "import.csv.empty", + }); + } + + const tz = await resolveUserTimezone(user.id); + let result: RecordImportResult; + try { + result = await opts.run({ + userId: user.id, + tz, + stream: request.body, + maxBytes: RECORD_IMPORT_MAX_BYTES, + query, + }); + } catch (err) { + if (err instanceof CsvTooLargeError) { + return apiError("CSV exceeds the 16 MB limit", 413, { + errorCode: "import.csv.too_large", + }); + } + throw err; + } + if (result.rows === 0) { + return apiError("The CSV file has no data rows", 422, { + errorCode: "import.csv.empty", + }); + } + + if (!query.dryRun) { + await auditLog(action, { + userId: user.id, + ipAddress: getClientIp(request), + details: { + rows: result.rows, + inserted: result.inserted, + duplicates: result.duplicates, + failed: result.failed, + }, + }); + } + annotate({ + meta: { + import_dry_run: query.dryRun, + import_rows: result.rows, + import_inserted: result.inserted, + import_duplicates: result.duplicates, + import_failed: result.failed, + }, + }); + return apiSuccess(result); +} diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 0aac71fa4..b91330fb2 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -226,7 +226,14 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "illness.episode.parent-not-found", "illness.insights.invalid", ], - import: ["import.apple_health.busy", "import.write_failed"], + import: [ + "import.apple_health.busy", + "import.csv.content_type", + "import.csv.empty", + "import.csv.too_large", + "import.rate_limited", + "import.write_failed", + ], ingest: ["ingest.rate_limited"], insights: [ "insights.generate.budgetExceeded", diff --git a/src/lib/openapi/routes/import.ts b/src/lib/openapi/routes/import.ts index 2611958c4..6fee3fa7f 100644 --- a/src/lib/openapi/routes/import.ts +++ b/src/lib/openapi/routes/import.ts @@ -23,6 +23,41 @@ import { stdResponses, } from "./shared"; import { isoDateTime } from "@/lib/validations/datetime"; +import { + labsCsvQuerySchema, + medicationsCsvQuerySchema, + recordImportResultSchema, +} from "@/lib/validations/record-import"; + +const RECORD_IMPORT_COMMON = + "Body is the raw CSV file (`text/csv`, ≤ 16 MB), read as a stream: a UTF-8 byte-order mark is dropped, the separator is `,`, `;` or a tab, quoted fields may hold separators and line breaks. Every row is checked on its own and a refused row is reported by line with a reason, never guessed (a number or date that reads two ways is refused). Re-uploading the same file writes nothing: every row is reported as a duplicate. `dryRun=1` checks the whole file without writing and returns the first rows as they would be stored; it spends a separate bucket of 30 checks an hour, the import itself the shared 5-an-hour `import:` bucket. Owner only; refused with `module.disabled` when the module is switched off."; + +function recordImportResponses(id: string) { + return { + "200": { + description: "Per-file and per-row outcome.", + content: { + "application/json": { + schema: dataEnvelope(recordImportResultSchema, id), + }, + }, + }, + ...stdResponses, + "403": { + description: + "`module.disabled`: the module the import writes to is switched off.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "413": { + description: "`import.csv.too_large`: the file is larger than 16 MB.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "415": { + description: "`import.csv.content_type`: the body is not `text/csv`.", + content: { "application/json": { schema: errorEnvelope } }, + }, + }; +} const csvImportRowResult = z .object({ @@ -351,6 +386,48 @@ export const importPaths: NonNullable = { }, }, }, + "/api/import/labs-csv": { + post: { + tags: ["Import"], + summary: "Import lab results from CSV", + description: `${RECORD_IMPORT_COMMON} Each row's name resolves to the account's marker the way a scanned report's rows do (name, alias, catalog name); a new analyte mints one marker from its first row. A numeric result needs a unit; one in another unit is converted when the conversion is known and keeps the printed value beside it, otherwise the row is refused with \`unit_mismatch\`. Any other text in the value column is a qualitative result. A date without a time is stored as that calendar date. At most 20 000 data rows.`, + requestParams: { query: labsCsvQuerySchema }, + requestBody: { + required: true, + content: { + "text/csv": { + schema: { + type: "string", + example: + "Analyte;Value;Unit;Date;Range\nHbA1c;5,4;%;01.03.2025;4,0-5,6\nUrine protein;negative;;01.03.2025;", + }, + }, + }, + }, + responses: recordImportResponses("LabsCsvImportEnvelope"), + }, + }, + "/api/import/medications-csv": { + post: { + tags: ["Import"], + summary: "Import medications from CSV", + description: `${RECORD_IMPORT_COMMON} Maps name, dose (as written), daily intake times, an as-needed mark and a start and end date; every other rhythm is set up afterwards in the app. A row needs times unless it is marked as needed. Each row is checked against the create route's own schema; a dated window opens a course. Reminders stay off unless \`reminders=1\`. A medication the record holds under the same name and dose is a duplicate. At most 500 data rows.`, + requestParams: { query: medicationsCsvQuerySchema }, + requestBody: { + required: true, + content: { + "text/csv": { + schema: { + type: "string", + example: + "name,dose,times,as needed,start,end\nRamipril,5 mg,08:00,,2025-01-01,\nIbuprofen,400 mg,,yes,,", + }, + }, + }, + }, + responses: recordImportResponses("MedicationsCsvImportEnvelope"), + }, + }, "/api/import": { post: { tags: ["Import"], diff --git a/src/lib/validations/record-import.ts b/src/lib/validations/record-import.ts new file mode 100644 index 000000000..c6c572d1f --- /dev/null +++ b/src/lib/validations/record-import.ts @@ -0,0 +1,241 @@ +/** + * Query and response schemas of the CSV record imports for lab results and + * medications (#1215). + * + * The body of both routes is the raw CSV file (`text/csv`), streamed. How to + * read it travels in the query: the column each field comes from (0-based + * index into the file's columns), the separator, the decimal mark, the date + * order and the zone a time without an offset is read in. The client works + * these out with the person on the mapping screen; the server never guesses + * a column. + */ +import { z } from "zod/v4"; + +import { MAX_COLUMNS } from "@/lib/import/csv-stream"; +import { isValidTimezone } from "@/lib/tz/format"; + +const column = z.coerce + .number() + .int() + .min(0) + .max(MAX_COLUMNS - 1) + .describe("0-based index of the file column this field is read from."); + +const formatFields = { + dryRun: z + .enum(["1", "0", "true", "false"]) + .default("0") + .transform((v) => v === "1" || v === "true") + .describe( + "`1` validates every row and reports what would happen, writing nothing.", + ), + header: z + .enum(["1", "0"]) + .default("1") + .transform((v) => v === "1") + .describe("`1` when the first row holds column names and is not data."), + delimiter: z + .enum(["auto", "comma", "semicolon", "tab"]) + .default("auto") + .describe("Column separator; `auto` detects it from the first line."), + decimal: z + .enum(["dot", "comma"]) + .default("dot") + .describe( + "Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is refused per row as ambiguous.", + ), + dateOrder: z + .enum(["iso", "dmy", "mdy"]) + .default("iso") + .describe( + "`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit years are refused.", + ), + timezone: z + .string() + .max(64) + .refine(isValidTimezone, { message: "Unknown time zone" }) + .optional() + .describe( + "IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time is stored as that calendar date.", + ), +}; + +function distinctColumns( + value: Record, + keys: readonly string[], +): boolean { + const used = keys + .map((k) => value[k]) + .filter((v): v is number => typeof v === "number"); + return new Set(used).size === used.length; +} + +const LAB_COLUMN_KEYS = [ + "analyte", + "value", + "unit", + "date", + "time", + "referenceLow", + "referenceHigh", + "referenceRange", + "panel", + "note", +] as const; + +export const labsCsvQuerySchema = z + .object({ + ...formatFields, + analyte: column.describe("Column of the marker name (required)."), + value: column.describe( + "Column of the result (required). A number is a numeric result; any other text (`negative`, `<0.5`) is kept as a qualitative result.", + ), + unit: column + .optional() + .describe("Column of the unit. Required for a numeric result."), + date: column.describe("Column of the date, or date and time (required)."), + time: column.optional().describe("Column of the time, when separate."), + referenceLow: column.optional(), + referenceHigh: column.optional(), + referenceRange: column + .optional() + .describe("Column of a printed range such as `4.0-5.6` or `< 200`."), + panel: column.optional(), + note: column.optional().describe("Column of a note; stored encrypted."), + }) + .refine((v) => distinctColumns(v, LAB_COLUMN_KEYS), { + message: "Each column can feed one field", + path: ["analyte"], + }); + +export type LabsCsvQuery = z.infer; + +const MEDICATION_COLUMN_KEYS = [ + "name", + "dose", + "times", + "asNeeded", + "startsOn", + "endsOn", +] as const; + +export const medicationsCsvQuerySchema = z + .object({ + ...formatFields, + name: column.describe("Column of the medication name (required)."), + dose: column.describe( + "Column of the dose as written, e.g. `10 mg` (required).", + ), + times: column + .optional() + .describe( + "Column of daily intake times (`08:00`, or several: `08:00 20:00`, separated by space, comma, semicolon or `|`). A medication with times is taken every day at them; any other rhythm is set up afterwards in the app. A row needs times unless it is marked as needed.", + ), + asNeeded: column + .optional() + .describe("Column marking an as-needed medication (yes/no)."), + startsOn: column.optional().describe("Column of the start date."), + endsOn: column.optional().describe("Column of the end date."), + reminders: z + .enum(["1", "0"]) + .default("0") + .transform((v) => v === "1") + .describe( + "`1` turns reminders on for imported medications that have times. Off by default, so an import never starts reminding on its own.", + ), + }) + .refine((v) => distinctColumns(v, MEDICATION_COLUMN_KEYS), { + message: "Each column can feed one field", + path: ["name"], + }); + +export type MedicationsCsvQuery = z.infer; + +export const RECORD_IMPORT_ROW_REASONS = [ + "missing_value", + "invalid_number", + "ambiguous_number", + "missing_unit", + "invalid_date", + "invalid_time", + "ambiguous_date", + "future_date", + "text_too_long", + "unit_mismatch", + "invalid_reference_range", + "invalid_times", + "missing_times", + "invalid_yes_no", + "as_needed_with_times", + "end_before_start", + "invalid_medication", + "oversized_row", + "row_limit", + "duplicate", +] as const; + +export type RecordImportRowReason = (typeof RECORD_IMPORT_ROW_REASONS)[number]; + +export const recordImportResultSchema = z + .object({ + kind: z.enum(["labs", "medications"]), + dryRun: z.boolean(), + rows: z.number().int().describe("Data rows read (the header excluded)."), + valid: z.number().int().describe("Rows that passed every check."), + inserted: z + .number() + .int() + .describe("Rows written; in a dry run, the rows that would be."), + duplicates: z + .number() + .int() + .describe( + "Rows skipped because the record already holds them, or an earlier row of the file does. A re-upload of the same file therefore writes nothing.", + ), + failed: z + .number() + .int() + .describe("Rows refused; each is listed in `errors`."), + errors: z + .array( + z.object({ + line: z.number().int().describe("1-based line of the file."), + reason: z.enum(RECORD_IMPORT_ROW_REASONS), + field: z.string().optional().describe("The mapped field at fault."), + }), + ) + .describe("Refused and duplicate rows, in file order, at most 500."), + errorsTruncated: z.boolean(), + markers: z + .object({ + matched: z + .array( + z.object({ + name: z.string().describe("The name as the file writes it."), + marker: z.string().describe("The marker it resolves to."), + how: z.enum(["name", "alias", "catalog"]), + }), + ) + .describe( + "File names that resolve to an existing marker, at most 200.", + ), + created: z + .array(z.string()) + .describe("Names that become new markers, at most 200."), + }) + .optional() + .describe("Lab imports only: how the file's names meet the markers."), + preview: z + .array( + z.record( + z.string(), + z.union([z.string(), z.number(), z.boolean(), z.null()]), + ), + ) + .describe( + "The first 20 valid rows as they will be stored (dry run only).", + ), + }) + .meta({ id: "RecordImportResult" }); + +export type RecordImportResult = z.infer; diff --git a/src/proxy.ts b/src/proxy.ts index d2f8e2e81..132e9c31a 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -865,6 +865,6 @@ export const config = { // the matcher and `apiHandler` performs the proxy's duties for it. The // alternation must stay equal to that list; `proxy-body-limit.test.ts` // compiles this exactly as Next does and checks. - "/((?!_next/static|_next/image|favicon.ico|sw\\.js|manifest\\.json|robots\\.txt|sitemap\\.xml|api/(?:import(?:/csv|/apple-health-export|/health-connect-export)?|admin/import-apple-health-export|admin/backups/upload|documents/inbound|medications/intake/(?:dose-history-import|bulk)|labs/ocr/extract|medications/[^/]+/ingredients/extract|user/avatar|measurements/batch|workouts/batch|insights/ecg|mood-entries/bulk|cycle/day-logs/bulk)/?$|.*\\.(?:svg|png|jpg|jpeg|gif|webp|txt|xml)$).*)", + "/((?!_next/static|_next/image|favicon.ico|sw\\.js|manifest\\.json|robots\\.txt|sitemap\\.xml|api/(?:import(?:/csv|/labs-csv|/medications-csv|/apple-health-export|/health-connect-export)?|admin/import-apple-health-export|admin/backups/upload|documents/inbound|medications/intake/(?:dose-history-import|bulk)|labs/ocr/extract|medications/[^/]+/ingredients/extract|user/avatar|measurements/batch|workouts/batch|insights/ecg|mood-entries/bulk|cycle/day-logs/bulk)/?$|.*\\.(?:svg|png|jpg|jpeg|gif|webp|txt|xml)$).*)", ], }; diff --git a/tests/integration/record-csv-import.test.ts b/tests/integration/record-csv-import.test.ts new file mode 100644 index 000000000..665fc6da6 --- /dev/null +++ b/tests/integration/record-csv-import.test.ts @@ -0,0 +1,251 @@ +/** + * The lab and medication CSV imports against a real Postgres (#1215). + * + * - a dry run writes nothing (no reading, no marker, no medication) and + * reports what the real run then does; + * - names resolve to existing markers, new analytes mint one marker each, + * a reading in another unit is converted and keeps what the file said; + * - re-uploading the same file writes nothing: every row is a duplicate; + * - a refused row is reported by line and does not stop the rest; + * - medications take a daily schedule, reminders off, as-needed without + * schedule, and a dated window opens a course. + */ +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { textToStream } from "@/lib/import/csv-stream"; +import { importLabsCsv } from "@/lib/import/labs-csv"; +import { importMedicationsCsv } from "@/lib/import/medications-csv"; +import { + labsCsvQuerySchema, + medicationsCsvQuerySchema, +} from "@/lib/validations/record-import"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: vi.fn(() => null), +})); + +const prisma = getPrismaClient(); +const USER = "csv-import-user"; +const TZ = "Europe/Berlin"; + +const LABS = [ + "Analyt;Wert;Einheit;Datum;Referenz", + "Glucose;5,5;mmol/L;01.03.2025;3,9 - 5,5", + "Glukose;99;mg/dL;02.03.2025;", + "HbA1c;5,4;%;01.03.2025;4,0-5,6", + "Urinstatus Eiweiß;negativ;;01.03.2025;", + "Ferritin;1.234;ng/mL;01.03.2025;", + "Ferritin;85;;01.03.2025;", + "Neuer Marker;12;U/l;03.03.2025;", +].join("\n"); + +function labsQuery(dryRun: boolean) { + return labsCsvQuerySchema.parse({ + analyte: "0", + value: "1", + unit: "2", + date: "3", + referenceRange: "4", + decimal: "comma", + dateOrder: "dmy", + dryRun: dryRun ? "1" : "0", + }); +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: TZ }, + }); + await prisma.biomarker.create({ + // No analyte key stored: the import derives it the way the write does. + data: { userId: USER, name: "Glucose", unit: "mg/dL" }, + }); + await prisma.biomarkerAlias.create({ + data: { + userId: USER, + biomarkerId: ( + await prisma.biomarker.findFirstOrThrow({ where: { userId: USER } }) + ).id, + name: "Glukose", + nameFolded: "glukose", + source: "MANUAL", + }, + }); +}); + +describe("lab CSV import", () => { + it("a dry run writes nothing and reports the plan", async () => { + const result = await importLabsCsv({ + userId: USER, + tz: TZ, + stream: textToStream(LABS, 16), + maxBytes: 1_000_000, + query: labsQuery(true), + }); + expect(result).toMatchObject({ + rows: 7, + valid: 5, + inserted: 5, + failed: 2, + duplicates: 0, + }); + expect(result.errors).toEqual([ + { line: 6, reason: "ambiguous_number", field: "value" }, + { line: 7, reason: "missing_unit", field: "unit" }, + ]); + expect(result.markers?.created.sort()).toEqual([ + "HbA1c", + "Neuer Marker", + "Urinstatus Eiweiß", + ]); + expect(result.markers?.matched.map((m) => m.how).sort()).toEqual([ + "alias", + "name", + ]); + expect(await prisma.labResult.count({ where: { userId: USER } })).toBe(0); + expect(await prisma.biomarker.count({ where: { userId: USER } })).toBe(1); + }); + + it("the import converts, mints and keeps what the file said", async () => { + const result = await importLabsCsv({ + userId: USER, + tz: TZ, + stream: textToStream(LABS, 16), + maxBytes: 1_000_000, + query: labsQuery(false), + }); + expect(result).toMatchObject({ inserted: 5, failed: 2 }); + const glucose = await prisma.labResult.findMany({ + where: { userId: USER, analyte: "Glucose" }, + orderBy: { takenAt: "asc" }, + }); + expect(glucose).toHaveLength(2); + expect(glucose[0].unit).toBe("mg/dL"); + expect(glucose[0].value).toBeCloseTo(99.1, 0); + expect(glucose[0].sourceValue).toBe(5.5); + expect(glucose[0].sourceUnit).toBe("mmol/L"); + expect(glucose[0].source).toBe("IMPORT"); + expect(glucose[1].value).toBe(99); + const text = await prisma.labResult.findFirstOrThrow({ + where: { userId: USER, valueText: "negativ" }, + }); + expect(text.value).toBeNull(); + expect(await prisma.biomarker.count({ where: { userId: USER } })).toBe(4); + }); + + it("re-uploading the same file writes nothing", async () => { + const before = await prisma.labResult.count({ where: { userId: USER } }); + const result = await importLabsCsv({ + userId: USER, + tz: TZ, + stream: textToStream(LABS, 7), + maxBytes: 1_000_000, + query: labsQuery(false), + }); + expect(result).toMatchObject({ inserted: 0, duplicates: 5, failed: 2 }); + expect(await prisma.labResult.count({ where: { userId: USER } })).toBe( + before, + ); + }); + + it("refuses a unit that does not convert into the marker's", async () => { + const result = await importLabsCsv({ + userId: USER, + tz: TZ, + stream: textToStream( + "Analyt;Wert;Einheit;Datum\nGlucose;7;mmHg;05.03.2025\n", + ), + maxBytes: 1_000_000, + query: labsCsvQuerySchema.parse({ + analyte: "0", + value: "1", + unit: "2", + date: "3", + decimal: "comma", + dateOrder: "dmy", + }), + }); + expect(result.errors).toEqual([ + { line: 2, reason: "unit_mismatch", field: "unit" }, + ]); + }); +}); + +describe("medication CSV import", () => { + const MEDS = [ + "name,dose,times,as needed,start,end", + "Ramipril,5 mg,08:00,,2025-01-01,", + "Metformin,500 mg,08:00 20:00,,,", + "Ibuprofen,400 mg,,yes,,", + "Vitamin D,1000 IE,,,,", + "ramipril,5 MG,08:00,,,", + ].join("\n"); + const query = (dryRun: boolean) => + medicationsCsvQuerySchema.parse({ + name: "0", + dose: "1", + times: "2", + asNeeded: "3", + startsOn: "4", + endsOn: "5", + dryRun: dryRun ? "1" : "0", + }); + + it("checks, then imports, then skips everything on a second upload", async () => { + const dry = await importMedicationsCsv({ + userId: USER, + tz: TZ, + stream: textToStream(MEDS), + maxBytes: 100_000, + query: query(true), + }); + expect(dry).toMatchObject({ + rows: 5, + inserted: 3, + duplicates: 1, + failed: 1, + }); + expect(dry.errors).toEqual([ + { line: 5, reason: "missing_times", field: "times" }, + { line: 6, reason: "duplicate" }, + ]); + expect(await prisma.medication.count({ where: { userId: USER } })).toBe(0); + + const real = await importMedicationsCsv({ + userId: USER, + tz: TZ, + stream: textToStream(MEDS), + maxBytes: 100_000, + query: query(false), + }); + expect(real).toMatchObject({ inserted: 3, duplicates: 1, failed: 1 }); + const meds = await prisma.medication.findMany({ + where: { userId: USER }, + include: { schedules: true, courses: true }, + orderBy: { name: "asc" }, + }); + const byName = new Map(meds.map((m) => [m.name, m])); + expect(byName.get("Metformin")?.schedules[0]).toMatchObject({ + rrule: "FREQ=DAILY", + timesOfDay: ["08:00", "20:00"], + }); + expect(byName.get("Metformin")?.notificationsEnabled).toBe(false); + expect(byName.get("Ibuprofen")).toMatchObject({ asNeeded: true }); + expect(byName.get("Ibuprofen")?.schedules).toHaveLength(0); + expect(byName.get("Ramipril")?.courses).toHaveLength(1); + + const again = await importMedicationsCsv({ + userId: USER, + tz: TZ, + stream: textToStream(MEDS), + maxBytes: 100_000, + query: query(false), + }); + expect(again).toMatchObject({ inserted: 0, duplicates: 4 }); + }); +});