tom/HealthLog
1
0 Fork 0
mirror of https://github.com/MBombeck/HealthLog.git synced 2026-11-09 13:01:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(import): import lab results and medications from CSV

Browse Source 
Two streamed endpoints read a spreadsheet record by record with a
column mapping from the query: separator, decimal mark, date order and
zone are stated, never guessed per row. Lab rows resolve to markers the
way scanned reports do, convert units where the conversion is known and
keep what the file said; medication rows map name, dose, daily times,
as-needed and a dated window, checked against the create schema, with
reminders off unless asked for. A dry run writes nothing, a re-upload
only finds duplicates, and every refused row comes back with its line
and reason.
This commit is contained in:
Marc-André Bombeck committed 2026-10-21 22:17:59 +02:00
1 parent c94766efc3
commit a84fb4b467
21 files changed
+3593 -3
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/api/openapi.yaml
+570 -1
View File
@@ -15327,6 +15327,400 @@ paths:
$ref: "#/components/schemas/CsvImportTooLargeEnvelope"
"422": *a4
"429": *a5
/api/import/labs-csv:
post:
tags:
- Import
summary: Import lab results from CSV
description: "Body is the raw CSV file (`text/csv`, ≤ 16 MB), read as a stream: a UTF-8 byte-order mark is dropped, the
separator is `,`, `;` or a tab, quoted fields may hold separators and line breaks. Every row is checked on its
own and a refused row is reported by line with a reason, never guessed (a number or date that reads two ways is
refused). Re-uploading the same file writes nothing: every row is reported as a duplicate. `dryRun=1` checks the
whole file without writing and returns the first rows as they would be stored; it spends a separate bucket of 30
checks an hour, the import itself the shared 5-an-hour `import:` bucket. Owner only; refused with
`module.disabled` when the module is switched off. Each row's name resolves to the account's marker the way a
scanned report's rows do (name, alias, catalog name); a new analyte mints one marker from its first row. A
numeric result needs a unit; one in another unit is converted when the conversion is known and keeps the printed
value beside it, otherwise the row is refused with `unit_mismatch`. Any other text in the value column is a
qualitative result. A date without a time is stored as that calendar date. At most 20 000 data rows."
parameters:
- in: query
name: dryRun
schema:
description: "`1` validates every row and reports what would happen, writing nothing."
default: "0"
type: string
enum:
- "0"
- "1"
- "true"
- "false"
description: "`1` validates every row and reports what would happen, writing nothing."
- in: query
name: header
schema:
description: "`1` when the first row holds column names and is not data."
default: "1"
type: string
enum:
- "0"
- "1"
description: "`1` when the first row holds column names and is not data."
- in: query
name: delimiter
schema:
default: auto
description: Column separator; `auto` detects it from the first line.
type: string
enum:
- auto
- comma
- semicolon
- tab
description: Column separator; `auto` detects it from the first line.
- in: query
name: decimal
schema:
default: dot
description: Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is
refused per row as ambiguous.
type: string
enum:
- dot
- comma
description: Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is
refused per row as ambiguous.
- in: query
name: dateOrder
schema:
default: iso
description: "`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit
years are refused."
type: string
enum:
- iso
- dmy
- mdy
description: "`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit
years are refused."
- in: query
name: timezone
schema:
description: IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time
is stored as that calendar date.
type: string
maxLength: 64
description: IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time
is stored as that calendar date.
- in: query
name: analyte
schema:
type: integer
minimum: 0
maximum: 99
description: Column of the marker name (required).
required: true
description: Column of the marker name (required).
- in: query
name: value
schema:
type: integer
minimum: 0
maximum: 99
description: Column of the result (required). A number is a numeric result; any other text (`negative`, `<0.5`) is kept
as a qualitative result.
required: true
description: Column of the result (required). A number is a numeric result; any other text (`negative`, `<0.5`) is kept
as a qualitative result.
- in: query
name: unit
schema:
description: Column of the unit. Required for a numeric result.
type: integer
minimum: 0
maximum: 99
description: Column of the unit. Required for a numeric result.
- in: query
name: date
schema:
type: integer
minimum: 0
maximum: 99
description: Column of the date, or date and time (required).
required: true
description: Column of the date, or date and time (required).
- in: query
name: time
schema:
description: Column of the time, when separate.
type: integer
minimum: 0
maximum: 99
description: Column of the time, when separate.
- in: query
name: referenceLow
schema:
type: integer
minimum: 0
maximum: 99
description: 0-based index of the file column this field is read from.
- in: query
name: referenceHigh
schema:
type: integer
minimum: 0
maximum: 99
description: 0-based index of the file column this field is read from.
- in: query
name: referenceRange
schema:
description: Column of a printed range such as `4.0-5.6` or `< 200`.
type: integer
minimum: 0
maximum: 99
description: Column of a printed range such as `4.0-5.6` or `< 200`.
- in: query
name: panel
schema:
type: integer
minimum: 0
maximum: 99
description: 0-based index of the file column this field is read from.
- in: query
name: note
schema:
description: Column of a note; stored encrypted.
type: integer
minimum: 0
maximum: 99
description: Column of a note; stored encrypted.
requestBody:
required: true
content:
text/csv:
schema:
type: string
example: |-
Analyte;Value;Unit;Date;Range
HbA1c;5,4;%;01.03.2025;4,0-5,6
Urine protein;negative;;01.03.2025;
responses:
"200":
description: Per-file and per-row outcome.
content:
application/json:
schema:
$ref: "#/components/schemas/LabsCsvImportEnvelope"
"401": *a1
"403":
description: "`module.disabled`: the module the import writes to is switched off."
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorEnvelope"
"413":
description: "`import.csv.too_large`: the file is larger than 16 MB."
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorEnvelope"
"415":
description: "`import.csv.content_type`: the body is not `text/csv`."
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorEnvelope"
"422": *a4
"429": *a5
/api/import/medications-csv:
post:
tags:
- Import
summary: Import medications from CSV
description: "Body is the raw CSV file (`text/csv`, ≤ 16 MB), read as a stream: a UTF-8 byte-order mark is dropped, the
separator is `,`, `;` or a tab, quoted fields may hold separators and line breaks. Every row is checked on its
own and a refused row is reported by line with a reason, never guessed (a number or date that reads two ways is
refused). Re-uploading the same file writes nothing: every row is reported as a duplicate. `dryRun=1` checks the
whole file without writing and returns the first rows as they would be stored; it spends a separate bucket of 30
checks an hour, the import itself the shared 5-an-hour `import:` bucket. Owner only; refused with
`module.disabled` when the module is switched off. Maps name, dose (as written), daily intake times, an
as-needed mark and a start and end date; every other rhythm is set up afterwards in the app. A row needs times
unless it is marked as needed. Each row is checked against the create route's own schema; a dated window opens a
course. Reminders stay off unless `reminders=1`. A medication the record holds under the same name and dose is a
duplicate. At most 500 data rows."
parameters:
- in: query
name: dryRun
schema:
description: "`1` validates every row and reports what would happen, writing nothing."
default: "0"
type: string
enum:
- "0"
- "1"
- "true"
- "false"
description: "`1` validates every row and reports what would happen, writing nothing."
- in: query
name: header
schema:
description: "`1` when the first row holds column names and is not data."
default: "1"
type: string
enum:
- "0"
- "1"
description: "`1` when the first row holds column names and is not data."
- in: query
name: delimiter
schema:
default: auto
description: Column separator; `auto` detects it from the first line.
type: string
enum:
- auto
- comma
- semicolon
- tab
description: Column separator; `auto` detects it from the first line.
- in: query
name: decimal
schema:
default: dot
description: Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is
refused per row as ambiguous.
type: string
enum:
- dot
- comma
description: Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is
refused per row as ambiguous.
- in: query
name: dateOrder
schema:
default: iso
description: "`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit
years are refused."
type: string
enum:
- iso
- dmy
- mdy
description: "`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit
years are refused."
- in: query
name: timezone
schema:
description: IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time
is stored as that calendar date.
type: string
maxLength: 64
description: IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time
is stored as that calendar date.
- in: query
name: name
schema:
type: integer
minimum: 0
maximum: 99
description: Column of the medication name (required).
required: true
description: Column of the medication name (required).
- in: query
name: dose
schema:
type: integer
minimum: 0
maximum: 99
description: Column of the dose as written, e.g. `10 mg` (required).
required: true
description: Column of the dose as written, e.g. `10 mg` (required).
- in: query
name: times
schema:
description: "Column of daily intake times (`08:00`, or several: `08:00 20:00`, separated by space, comma, semicolon or
`|`). A medication with times is taken every day at them; any other rhythm is set up afterwards in the
app. A row needs times unless it is marked as needed."
type: integer
minimum: 0
maximum: 99
description: "Column of daily intake times (`08:00`, or several: `08:00 20:00`, separated by space, comma, semicolon or
`|`). A medication with times is taken every day at them; any other rhythm is set up afterwards in the app.
A row needs times unless it is marked as needed."
- in: query
name: asNeeded
schema:
description: Column marking an as-needed medication (yes/no).
type: integer
minimum: 0
maximum: 99
description: Column marking an as-needed medication (yes/no).
- in: query
name: startsOn
schema:
description: Column of the start date.
type: integer
minimum: 0
maximum: 99
description: Column of the start date.
- in: query
name: endsOn
schema:
description: Column of the end date.
type: integer
minimum: 0
maximum: 99
description: Column of the end date.
- in: query
name: reminders
schema:
description: "`1` turns reminders on for imported medications that have times. Off by default, so an import never starts
reminding on its own."
default: "0"
type: string
enum:
- "0"
- "1"
description: "`1` turns reminders on for imported medications that have times. Off by default, so an import never starts
reminding on its own."
requestBody:
required: true
content:
text/csv:
schema:
type: string
example: |-
name,dose,times,as needed,start,end
Ramipril,5 mg,08:00,,2025-01-01,
Ibuprofen,400 mg,,yes,,
responses:
"200":
description: Per-file and per-row outcome.
content:
application/json:
schema:
$ref: "#/components/schemas/MedicationsCsvImportEnvelope"
"401": *a1
"403":
description: "`module.disabled`: the module the import writes to is switched off."
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorEnvelope"
"413":
description: "`import.csv.too_large`: the file is larger than 16 MB."
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorEnvelope"
"415":
description: "`import.csv.content_type`: the body is not `text/csv`."
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorEnvelope"
"422": *a4
"429": *a5
/api/import:
post:
tags:
@@ -36947,7 +37341,8 @@ components:
http.unauthenticated, http.unavailable, http.unsupported_media_type; illness — illness.day-log.invalid,
illness.disabled, illness.episode.chronic-no-resolve, illness.episode.invalid,
illness.episode.invalid-window, illness.episode.parent-not-found, illness.insights.invalid; import —
import.apple_health.busy, import.write_failed; ingest — ingest.rate_limited; insights —
import.apple_health.busy, import.csv.content_type, import.csv.empty, import.csv.too_large,
import.rate_limited, import.write_failed; ingest — ingest.rate_limited; insights —
insights.generate.budgetExceeded, insights.generate.outboundScreened,
insights.generate.profileScopeChanged, insights.generate.scopeChanged; labs —
labs.biomarker.alias.taken, labs.biomarker.create.invalid, labs.biomarker.duplicate,
@@ -71828,6 +72223,180 @@ components:
- data
- error
additionalProperties: false
LabsCsvImportEnvelope:
type: object
properties:
data:
$ref: "#/components/schemas/RecordImportResult"
error:
type: "null"
meta:
type: object
properties:
requestId:
type: string
additionalProperties: false
required:
- data
- error
additionalProperties: false
RecordImportResult:
type: object
properties:
kind:
type: string
enum:
- labs
- medications
dryRun:
type: boolean
rows:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
description: Data rows read (the header excluded).
valid:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
description: Rows that passed every check.
inserted:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
description: Rows written; in a dry run, the rows that would be.
duplicates:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
description: Rows skipped because the record already holds them, or an earlier row of the file does. A re-upload of the
same file therefore writes nothing.
failed:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
description: Rows refused; each is listed in `errors`.
errors:
type: array
items:
type: object
properties:
line:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
description: 1-based line of the file.
reason:
type: string
enum:
- missing_value
- invalid_number
- ambiguous_number
- missing_unit
- invalid_date
- invalid_time
- ambiguous_date
- future_date
- text_too_long
- unit_mismatch
- invalid_reference_range
- invalid_times
- missing_times
- invalid_yes_no
- as_needed_with_times
- end_before_start
- invalid_medication
- oversized_row
- row_limit
- duplicate
field:
description: The mapped field at fault.
type: string
required:
- line
- reason
additionalProperties: false
description: Refused and duplicate rows, in file order, at most 500.
errorsTruncated:
type: boolean
markers:
description: "Lab imports only: how the file's names meet the markers."
type: object
properties:
matched:
type: array
items:
type: object
properties:
name:
type: string
description: The name as the file writes it.
marker:
type: string
description: The marker it resolves to.
how:
type: string
enum:
- name
- alias
- catalog
required:
- name
- marker
- how
additionalProperties: false
description: File names that resolve to an existing marker, at most 200.
created:
type: array
items:
type: string
description: Names that become new markers, at most 200.
required:
- matched
- created
additionalProperties: false
preview:
type: array
items:
type: object
propertyNames:
type: string
additionalProperties:
type:
- string
- number
- boolean
- "null"
description: The first 20 valid rows as they will be stored (dry run only).
required:
- kind
- dryRun
- rows
- valid
- inserted
- duplicates
- failed
- errors
- errorsTruncated
- preview
additionalProperties: false
MedicationsCsvImportEnvelope:
type: object
properties:
data:
$ref: "#/components/schemas/RecordImportResult"
error:
type: "null"
meta:
type: object
properties:
requestId:
type: string
additionalProperties: false
required:
- data
- error
additionalProperties: false
JsonImportStatsEnvelope:
type: object
properties:
src/__tests__/medication-intake-tracking-guard.test.ts
+3
View File
@@ -96,6 +96,9 @@ const MAY_READ_EVERY_ROW: Record<string, string> = {
// as stored, schedule and switch included.
"lib/export/full-backup-payload.ts": "backup carries the stored rows",
"lib/export/restore-backup.ts": "restore writes the stored rows back",
// v1.44 — the CSV import writes new medications and reads only their names
// and doses to skip duplicates; it derives nothing due.
"lib/import/medications-csv.ts": "imports new rows, derives nothing due",
// v1.43 — the units a taken dose consumed, resolved per slot exactly as
// the inventory does. It reads only doses already recorded as taken.
"lib/medications/ingredients/recompute.ts":
src/__tests__/write-rate-limit-guard.test.ts
+3
View File
@@ -53,6 +53,9 @@ const LIMITING_HELPERS: Record<string, string[]> = {
"lib/withings/webhook-handler.ts",
"lib/whoop/webhook-handler.ts",
],
// v1.44 — the lab and medication CSV imports: a preview bucket for a dry
// run, the shared `import:` bucket for the import itself.
handleRecordImport: ["lib/import/record-import-route.ts"],
};
/** Mutating routes with no limit, each with the reason it needs none. */
src/app/api/import/__tests__/record-csv-routes.test.ts
+165
View File
@@ -0,0 +1,165 @@
/**
* The request side of `/api/import/labs-csv` and `/api/import/medications-csv`
* (#1215): module gate first, a separate bucket for dry runs, the content
* type, the byte cap while streaming, the mapping refused before anything is
* read, and an audit row only for a real import.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
import { NextRequest } from "next/server";
const mocks = vi.hoisted(() => ({
checkRateLimit: vi.fn(),
requireModuleEnabled: vi.fn(),
importLabsCsv: vi.fn(),
importMedicationsCsv: vi.fn(),
auditLog: vi.fn(),
}));
vi.mock("@/lib/api-handler", () => ({
apiHandler: (handler: unknown) => handler,
requireAuth: vi.fn(async () => ({ user: { id: "user-1" } })),
}));
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
vi.mock("@/lib/auth/audit", () => ({ auditLog: mocks.auditLog }));
vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: mocks.checkRateLimit }));
vi.mock("@/lib/modules/gate", () => ({
requireModuleEnabled: mocks.requireModuleEnabled,
}));
vi.mock("@/lib/tz/resolver", () => ({
resolveUserTimezone: vi.fn(async () => "Europe/Berlin"),
}));
vi.mock("@/lib/import/labs-csv", () => ({
importLabsCsv: mocks.importLabsCsv,
}));
vi.mock("@/lib/import/medications-csv", () => ({
importMedicationsCsv: mocks.importMedicationsCsv,
}));
import { CsvTooLargeError } from "@/lib/import/csv-stream";
import { POST as labsPost } from "../labs-csv/route";
import { POST as medsPost } from "../medications-csv/route";
type Handler = (req: NextRequest) => Promise<Response>;
const RESULT = {
kind: "labs",
dryRun: false,
rows: 2,
valid: 2,
inserted: 2,
duplicates: 0,
failed: 0,
errors: [],
errorsTruncated: false,
preview: [],
};
function post(
handler: unknown,
query: string,
body = "a,b\n1,2\n",
type = "text/csv",
) {
return (handler as Handler)(
new NextRequest(`http://localhost/api/import/x?${query}`, {
method: "POST",
headers: { "content-type": type },
body,
}),
);
}
beforeEach(() => {
vi.clearAllMocks();
mocks.checkRateLimit.mockResolvedValue({ allowed: true });
mocks.requireModuleEnabled.mockResolvedValue({ enabled: true });
mocks.importLabsCsv.mockResolvedValue(RESULT);
mocks.importMedicationsCsv.mockResolvedValue({
...RESULT,
kind: "medications",
});
});
const LABS = "analyte=0&value=1&unit=2&date=3";
const MEDS = "name=0&dose=1&times=2";
describe("record CSV import routes", () => {
it("answers the module gate before reading the file", async () => {
mocks.requireModuleEnabled.mockResolvedValue({
enabled: false,
response: new Response(null, { status: 403 }),
});
expect((await post(labsPost, LABS)).status).toBe(403);
expect(mocks.requireModuleEnabled).toHaveBeenCalledWith("user-1", "labs");
expect(mocks.importLabsCsv).not.toHaveBeenCalled();
});
it("refuses a mapping without the required columns, or with a shared column", async () => {
expect((await post(labsPost, "analyte=0&value=1")).status).toBe(422);
expect((await post(labsPost, "analyte=0&value=0&date=1")).status).toBe(422);
expect((await post(medsPost, "name=0")).status).toBe(422);
expect(mocks.importLabsCsv).not.toHaveBeenCalled();
});
it("spends the preview bucket on a dry run and the import bucket otherwise", async () => {
await post(labsPost, `${LABS}&dryRun=1`);
expect(mocks.checkRateLimit).toHaveBeenLastCalledWith(
"import-preview:user-1",
30,
3_600_000,
);
expect(mocks.auditLog).not.toHaveBeenCalled();
await post(medsPost, MEDS);
expect(mocks.checkRateLimit).toHaveBeenLastCalledWith(
"import:user-1",
5,
3_600_000,
);
expect(mocks.auditLog).toHaveBeenCalledWith(
"import.medications.csv",
expect.objectContaining({ userId: "user-1" }),
);
});
it("answers 429 when the bucket is spent", async () => {
mocks.checkRateLimit.mockResolvedValue({ allowed: false });
expect((await post(labsPost, LABS)).status).toBe(429);
});
it("refuses a body that is not CSV", async () => {
expect((await post(labsPost, LABS, "{}", "application/json")).status).toBe(
415,
);
});
it("turns the streaming cap into a 413", async () => {
mocks.importLabsCsv.mockRejectedValue(new CsvTooLargeError(1));
expect((await post(labsPost, LABS)).status).toBe(413);
});
it("calls a file without data rows empty", async () => {
mocks.importLabsCsv.mockResolvedValue({ ...RESULT, rows: 0 });
expect((await post(labsPost, LABS)).status).toBe(422);
});
it("hands the parsed mapping and the account zone to the importer", async () => {
const res = await post(
labsPost,
`${LABS}&decimal=comma&dateOrder=dmy&delimiter=semicolon`,
);
expect(res.status).toBe(200);
const args = mocks.importLabsCsv.mock.calls[0][0];
expect(args.tz).toBe("Europe/Berlin");
expect(args.query).toMatchObject({
analyte: 0,
value: 1,
unit: 2,
date: 3,
decimal: "comma",
dateOrder: "dmy",
delimiter: "semicolon",
dryRun: false,
header: true,
});
expect(args.stream).toBeInstanceOf(ReadableStream);
});
});
src/app/api/import/labs-csv/route.ts
+25
View File
@@ -0,0 +1,25 @@
/**
* `POST /api/import/labs-csv` — lab results from a spreadsheet (#1215).
*
* Body: the CSV file, streamed. Query: the column mapping and the file's
* conventions (`labsCsvQuerySchema`). `dryRun=1` checks every row and reports
* what would happen without writing; the real import writes in chunks, skips
* readings the record already holds and reports every refused row by line.
*/
import { NextRequest } from "next/server";
import { apiHandler } from "@/lib/api-handler";
import { importLabsCsv } from "@/lib/import/labs-csv";
import { handleRecordImport } from "@/lib/import/record-import-route";
import { labsCsvQuerySchema } from "@/lib/validations/record-import";
export const dynamic = "force-dynamic";
export const POST = apiHandler(async (request: NextRequest) =>
handleRecordImport(request, {
kind: "labs",
module: "labs",
schema: labsCsvQuerySchema,
run: (args) => importLabsCsv(args),
}),
);
src/app/api/import/medications-csv/route.ts
+25
View File
@@ -0,0 +1,25 @@
/**
* `POST /api/import/medications-csv` — medications from a spreadsheet (#1215).
*
* Body: the CSV file, streamed. Query: the column mapping and the file's
* conventions (`medicationsCsvQuerySchema`). Maps only name, dose, daily
* times, as-needed and a start and end date; reminders stay off unless the
* query turns them on. `dryRun=1` checks without writing.
*/
import { NextRequest } from "next/server";
import { apiHandler } from "@/lib/api-handler";
import { importMedicationsCsv } from "@/lib/import/medications-csv";
import { handleRecordImport } from "@/lib/import/record-import-route";
import { medicationsCsvQuerySchema } from "@/lib/validations/record-import";
export const dynamic = "force-dynamic";
export const POST = apiHandler(async (request: NextRequest) =>
handleRecordImport(request, {
kind: "medications",
module: "medications",
schema: medicationsCsvQuerySchema,
run: (args) => importMedicationsCsv(args),
}),
);
src/lib/http/proxy-bypass-routes.ts
+3
View File
@@ -42,6 +42,9 @@ export const PROXY_BYPASS_ROUTES: readonly string[] = [
// Bulk JSON / CSV imports, 16 MB.
"/api/import",
"/api/import/csv",
// Lab and medication CSV imports (v1.44), streamed, 16 MB.
"/api/import/labs-csv",
"/api/import/medications-csv",
"/api/medications/intake/dose-history-import",
// Lab scan upload, 12 MB.
"/api/labs/ocr/extract",
src/lib/import/__tests__/csv-stream.test.ts
+227
View File
@@ -0,0 +1,227 @@
/**
* The streaming CSV reader: the spreadsheet shapes, every chunk boundary, a
* fuzzed round trip, and a large file read with bounded memory.
*/
import { describe, expect, it } from "vitest";
import {
CsvTokenizer,
CsvTooLargeError,
MAX_COLUMNS,
MAX_FIELD_CHARS,
detectDelimiter,
readCsvRecords,
textToStream,
type CsvRecord,
} from "../csv-stream";
function all(text: string, delimiter: "auto" | "," | ";" | "\t" = "auto") {
const t = new CsvTokenizer(delimiter);
return [...t.push(text), ...t.end()];
}
/** Feed the text split at every position, one boundary at a time. */
function everySplit(text: string): CsvRecord[][] {
const out: CsvRecord[][] = [];
for (let i = 0; i <= text.length; i++) {
const t = new CsvTokenizer();
out.push([
...t.push(text.slice(0, i)),
...t.push(text.slice(i)),
...t.end(),
]);
}
return out;
}
const fields = (records: CsvRecord[]) => records.map((r) => r.fields);
describe("detectDelimiter", () => {
it.each([
["a,b,c", ","],
["a;b;c", ";"],
["a\tb\tc", "\t"],
['"x;y",b,c', ","],
["single", ","],
] as const)("%j → %j", (line, expected) => {
expect(detectDelimiter(line)).toBe(expected);
});
});
describe("CsvTokenizer", () => {
it("drops a byte-order mark and reads a semicolon file with decimal commas", () => {
expect(fields(all("Analyt;Wert;Einheit\nHbA1c;5,4;%\n"))).toEqual([
["Analyt", "Wert", "Einheit"],
["HbA1c", "5,4", "%"],
]);
});
it("keeps separators, quotes and line breaks inside quoted fields", () => {
const text = 'a,b\n"x, y","say ""hi""\nthere"\n';
expect(fields(all(text))).toEqual([
["a", "b"],
["x, y", 'say "hi"\nthere'],
]);
});
it("reads LF, CRLF and lone CR line ends and skips blank lines", () => {
expect(fields(all("a,b\r\n1,2\r3,4\n\n\r\n5,6"))).toEqual([
["a", "b"],
["1", "2"],
["3", "4"],
["5", "6"],
]);
});
it("reports the physical line each record starts on", () => {
const records = all('h\n"multi\nline"\n\nlast\n');
expect(records.map((r) => r.line)).toEqual([1, 2, 5]);
});
it("keeps empty fields and a trailing empty field", () => {
expect(fields(all("a,,c,\n"))).toEqual([["a", "", "c", ""]]);
});
it("gives the same records whatever the chunk boundaries", () => {
const text =
'name;dose;"note"\r\n"Ram;ipril";"5 mg";"two\r\nlines ""q"""\r\nX;1;\r';
const expected = fields(all(text));
for (const records of everySplit(text)) {
expect(fields(records)).toEqual(expected);
}
});
it("marks a runaway field instead of growing without end", () => {
const records = all(`a\n"${"x".repeat(MAX_FIELD_CHARS + 50)}\n`);
expect(records[1].oversized).toBe(true);
expect(records[1].fields[0].length).toBe(MAX_FIELD_CHARS);
});
it("marks a record wider than the column cap", () => {
const wide = Array.from({ length: MAX_COLUMNS + 5 }, (_, i) =>
String(i),
).join(",");
const [record] = all(`${wide}\n`);
expect(record.oversized).toBe(true);
expect(record.fields).toHaveLength(MAX_COLUMNS);
});
});
/** Deterministic pseudo-random so a failure reproduces. */
function rng(seed: number) {
let s = seed;
return () => {
s = (s * 1103515245 + 12345) & 0x7fffffff;
return s / 0x7fffffff;
};
}
function quoteField(value: string) {
return /[",;\t\r\n]/.test(value) || value === ""
? `"${value.replace(/"/g, '""')}"`
: value;
}
describe("fuzzed round trip", () => {
it("reads back exactly what was written, for random tables and chunkings", () => {
const alphabet = [
"a",
"Z",
"1",
",",
";",
'"',
"\n",
"\r\n",
" ",
"ä",
"漢",
"\t",
".",
];
const rand = rng(1215);
for (let round = 0; round < 300; round++) {
const delimiter = ([",", ";", "\t"] as const)[Math.floor(rand() * 3)];
const cols = 1 + Math.floor(rand() * 5);
const rows = 1 + Math.floor(rand() * 6);
const table: string[][] = [];
for (let r = 0; r < rows; r++) {
const row: string[] = [];
for (let c = 0; c < cols; c++) {
let v = "";
const len = Math.floor(rand() * 6);
for (let k = 0; k < len; k++)
v += alphabet[Math.floor(rand() * alphabet.length)];
// A field that is all whitespace-free and quotable stays as written.
row.push(r === 0 ? `h${c}` : v);
}
// A record of only empty fields is skipped by design; keep one cell.
if (!row.some((f) => f.length > 0)) row[0] = "x";
table.push(row);
}
const text =
table
.map((row) => row.map((f) => quoteField(f)).join(delimiter))
.join(rand() < 0.5 ? "\n" : "\r\n") + (rand() < 0.5 ? "\n" : "");
const t = new CsvTokenizer(delimiter);
const out: CsvRecord[] = [];
let i = 0;
while (i < text.length) {
const step = 1 + Math.floor(rand() * 7);
out.push(...t.push(text.slice(i, i + step)));
i += step;
}
out.push(...t.end());
// Quoted line breaks come back as "\n": the reader normalises them.
const expected = table.map((row) =>
row.map((f) => f.replace(/\r\n/g, "\n")),
);
expect(fields(out), `round ${round}: ${JSON.stringify(text)}`).toEqual(
expected,
);
}
});
});
describe("readCsvRecords", () => {
it("streams a large file record by record", async () => {
const rows = 200_000;
const lines = ["analyte;value;unit;date"];
for (let i = 0; i < rows; i++)
lines.push(`Marker ${i % 50};${i % 9},${i % 10};mg/dL;01.02.2024`);
const text = lines.join("\n");
let count = 0;
let maxFieldLength = 0;
for await (const record of readCsvRecords(textToStream(text, 8192), {
maxBytes: 64 * 1024 * 1024,
})) {
count += 1;
for (const f of record.fields)
maxFieldLength = Math.max(maxFieldLength, f.length);
}
expect(count).toBe(rows + 1);
expect(maxFieldLength).toBeLessThan(20);
});
it("stops past the byte cap", async () => {
const text = "a,b\n" + "1,2\n".repeat(1000);
const consume = async () => {
for await (const _ of readCsvRecords(textToStream(text, 100), {
maxBytes: 500,
})) {
void _;
}
};
await expect(consume()).rejects.toBeInstanceOf(CsvTooLargeError);
});
it("decodes a multi-byte character split across chunks", async () => {
const text = "name\nÄrztin 漢字\n";
const out: string[] = [];
for await (const r of readCsvRecords(textToStream(text, 1), {
maxBytes: 1000,
}))
out.push(r.fields[0]);
expect(out).toEqual(["name", "Ärztin 漢字"]);
});
});
src/lib/import/__tests__/csv-values.test.ts
+120
View File
@@ -0,0 +1,120 @@
import { describe, expect, it } from "vitest";
import { parseCsvDate, parseLocaleNumber, parseYesNo } from "../csv-values";
describe("parseLocaleNumber", () => {
it.each([
["5.4", "dot", 5.4],
["5,4", "comma", 5.4],
["-0.5", "dot", -0.5],
["1,234.5", "dot", 1234.5],
["1.234,5", "comma", 1234.5],
["1,234,567", "dot", 1234567],
["1.234.567", "comma", 1234567],
["5,4", "dot", 5.4],
["5.4", "comma", 5.4],
["12", "comma", 12],
[" 7,25 ", "comma", 7.25],
["1 234,5", "comma", 1234.5],
[".5", "dot", 0.5],
] as const)("%j (%s) → %s", (raw, mark, value) => {
expect(parseLocaleNumber(raw, mark)).toEqual({ ok: true, value });
});
it.each([
["1,234", "dot"],
["1.234", "comma"],
["1234,567", "dot"],
] as const)("refuses %j (%s) as ambiguous", (raw, mark) => {
expect(parseLocaleNumber(raw, mark)).toEqual({
ok: false,
reason: "ambiguous_number",
});
});
it.each(["", "abc", "<0.5", "5..4", "1,2,3", "5.4.3", "1e5", "5,"])(
"refuses %j as not a number",
(raw) => {
expect(parseLocaleNumber(raw, "dot").ok).toBe(false);
},
);
});
describe("parseCsvDate", () => {
const berlin = { timezone: "Europe/Berlin" };
it("stores a date without a time on the date-only anchor", () => {
const r = parseCsvDate("2024-02-01", undefined, {
order: "iso",
...berlin,
});
expect(r).toMatchObject({
ok: true,
dateKey: "2024-02-01",
dateOnly: true,
});
expect(r.ok && r.instant.toISOString()).toBe("2024-02-01T12:00:00.000Z");
});
it("reads a wall-clock time in the import's zone, across DST", () => {
const winter = parseCsvDate("15.01.2024", "08:30", {
order: "dmy",
...berlin,
});
const summer = parseCsvDate("15.07.2024 08:30", undefined, {
order: "dmy",
...berlin,
});
expect(winter.ok && winter.instant.toISOString()).toBe(
"2024-01-15T07:30:00.000Z",
);
expect(summer.ok && summer.instant.toISOString()).toBe(
"2024-07-15T06:30:00.000Z",
);
});
it("honours an explicit offset over the zone", () => {
const r = parseCsvDate("2024-07-15T08:30:00+02:00", undefined, {
order: "iso",
timezone: "America/New_York",
});
expect(r.ok && r.instant.toISOString()).toBe("2024-07-15T06:30:00.000Z");
});
it("reads US order and refuses a cell that cannot be in it", () => {
expect(
parseCsvDate("02/15/2024", undefined, { order: "mdy", ...berlin }),
).toMatchObject({ ok: true, dateKey: "2024-02-15" });
expect(
parseCsvDate("15/02/2024", undefined, { order: "mdy", ...berlin }),
).toEqual({ ok: false, reason: "ambiguous_date" });
});
it.each([
["31.02.2024", "dmy", "invalid_date"],
["01.02.24", "dmy", "ambiguous_date"],
["24-02-01", "iso", "ambiguous_date"],
["Feb 1 2024", "iso", "invalid_date"],
["", "iso", "invalid_date"],
] as const)("%j (%s) → %s", (raw, order, reason) => {
expect(parseCsvDate(raw, undefined, { order, ...berlin })).toEqual({
ok: false,
reason,
});
});
it("refuses an impossible time", () => {
expect(
parseCsvDate("2024-02-01", "25:00", { order: "iso", ...berlin }),
).toEqual({ ok: false, reason: "invalid_time" });
});
});
describe("parseYesNo", () => {
it("reads the usual spellings and nothing else", () => {
expect(parseYesNo("Ja")).toBe(true);
expect(parseYesNo("no")).toBe(false);
expect(parseYesNo("")).toBeNull();
expect(parseYesNo("maybe")).toBeNull();
});
});
src/lib/import/__tests__/record-import-rows.test.ts
+269
View File
@@ -0,0 +1,269 @@
/**
* Row normalisation of the lab and medication CSV imports (#1215): what a
* cell may mean, and the reason a row is refused when it could mean two
* things. Pure; the database half is in
* `tests/integration/record-csv-import.test.ts`.
*/
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { createMedicationSchema } from "@/lib/validations/medication";
import {
labsCsvQuerySchema,
medicationsCsvQuerySchema,
} from "@/lib/validations/record-import";
import { normaliseLabRow, readingKeys } from "../labs-csv";
import {
medicationCreateBody,
normaliseMedicationRow,
parseTimes,
} from "../medications-csv";
const NOW = new Date("2026-10-21T10:00:00.000Z");
const labQuery = (extra: Record<string, string> = {}) =>
labsCsvQuerySchema.parse({
analyte: "0",
value: "1",
unit: "2",
date: "3",
referenceRange: "4",
...extra,
});
const lab = (fields: string[], extra?: Record<string, string>) =>
normaliseLabRow({ line: 7, fields }, labQuery(extra), "Europe/Berlin", NOW);
describe("lab rows", () => {
it("reads a numeric result with its unit, date and printed range", () => {
const r = lab(["HbA1c", "5.4", "%", "2025-03-01", "4.0-5.6"]);
expect(r).toMatchObject({
ok: true,
row: {
line: 7,
analyte: "HbA1c",
value: 5.4,
valueText: null,
unit: "%",
sourceReferenceLow: 4,
sourceReferenceHigh: 5.6,
},
});
expect(r.ok && r.row.takenAt.toISOString()).toBe(
"2025-03-01T12:00:00.000Z",
);
});
it("reads a German file: decimal comma, day-first dates, comma ranges", () => {
const r = lab(["Kreatinin", "0,92", "mg/dl", "01.03.2025", "0,7 - 1,2"], {
decimal: "comma",
dateOrder: "dmy",
});
expect(r).toMatchObject({
ok: true,
row: { value: 0.92, sourceReferenceLow: 0.7, sourceReferenceHigh: 1.2 },
});
});
it("keeps a non-numeric result as a qualitative reading", () => {
expect(
lab(["Urine protein", "negative", "", "2025-03-01", ""]),
).toMatchObject({
ok: true,
row: { value: null, valueText: "negative", unit: null },
});
expect(lab(["PSA", "<0.1", "ng/mL", "2025-03-01", ""])).toMatchObject({
ok: true,
row: { valueText: "<0.1" },
});
});
it.each([
[["", "5", "%", "2025-03-01", ""], "missing_value", "analyte"],
[["HbA1c", "", "%", "2025-03-01", ""], "missing_value", "value"],
[["HbA1c", "5.4", "", "2025-03-01", ""], "missing_unit", "unit"],
[
["Glucose", "1,234", "mg/dL", "2025-03-01", ""],
"ambiguous_number",
"value",
],
[
["Glucose", "1.2.3", "mg/dL", "2025-03-01", ""],
"invalid_number",
"value",
],
[["Glucose", "90", "mg/dL", "03/01/25", ""], "invalid_date", "date"],
[["Glucose", "90", "mg/dL", "2027-01-01", ""], "future_date", "date"],
] as const)("refuses %j with %s on %s", (fields, reason, field) => {
expect(lab([...fields])).toEqual({ ok: false, reason, field });
});
it("refuses a range whose low lies above its high", () => {
expect(
lab(["Glucose", "90", "mg/dL", "2025-03-01", "", "120", "70"], {
referenceRange: "4",
referenceLow: "5",
referenceHigh: "6",
}),
).toEqual({
ok: false,
reason: "invalid_reference_range",
field: "referenceRange",
});
});
it("refuses a mapping that feeds two fields from one column", () => {
expect(
labsCsvQuerySchema.safeParse({ analyte: "0", value: "0", date: "1" })
.success,
).toBe(false);
});
it("keys a reading the way the scan commit's duplicate check does", () => {
const keys = readingKeys(
{
biomarkerId: "bm1",
analyte: "HbA1c",
takenAt: new Date("2025-03-01T12:00:00.000Z"),
value: 39,
valueText: null,
sourceValue: 5.7,
sourceUnit: "%",
},
"Europe/Berlin",
);
expect(keys).toEqual([
"bm:bm1|2025-03-01|n:39",
"bm:bm1|2025-03-01|s:5.7:%",
]);
});
});
const medQuery = (extra: Record<string, string> = {}) =>
medicationsCsvQuerySchema.parse({
name: "0",
dose: "1",
times: "2",
asNeeded: "3",
startsOn: "4",
endsOn: "5",
...extra,
});
const med = (fields: string[], extra?: Record<string, string>) =>
normaliseMedicationRow({ line: 3, fields }, medQuery(extra), "Europe/Berlin");
describe("medication rows", () => {
it.each([
["08:00", ["08:00"]],
["20:00 8:00", ["08:00", "20:00"]],
["8.00; 20.00|12:30", ["08:00", "12:30", "20:00"]],
["", []],
])("times %j → %j", (raw, expected) => {
expect(parseTimes(raw)).toEqual(expected);
});
it.each(["8", "25:00", "morning", "08:00 x"])("refuses times %j", (raw) => {
expect(parseTimes(raw)).toBeNull();
});
it("reads a daily medication with a course", () => {
expect(
med(["Ramipril", "5 mg", "08:00", "", "01.02.2025", ""], {
dateOrder: "dmy",
}),
).toEqual({
ok: true,
row: {
line: 3,
name: "Ramipril",
dose: "5 mg",
times: ["08:00"],
asNeeded: false,
startsOn: "2025-02-01",
endsOn: null,
},
});
});
it.each([
[["", "5 mg", "", "", "", ""], "missing_value", "name"],
[
["Ibuprofen", "400 mg", "08:00", "yes", "", ""],
"as_needed_with_times",
"times",
],
[
["Ibuprofen", "400 mg", "", "sometimes", "", ""],
"invalid_yes_no",
"asNeeded",
],
[
["X", "1", "08:00", "", "2025-03-01", "2025-02-01"],
"end_before_start",
"endsOn",
],
[["Vitamin D", "1000 IE", "", "", "", ""], "missing_times", "times"],
] as const)("refuses %j with %s", (fields, reason, field) => {
expect(med([...fields])).toEqual({ ok: false, reason, field });
});
it("builds a body the create route's own schema accepts, reminders off by default", () => {
const daily = med(["Ramipril", "5 mg", "08:00 20:00", "", "", ""]);
const asNeeded = med(["Ibuprofen", "400 mg", "", "ja", "", ""]);
for (const r of [daily, asNeeded]) {
expect(r.ok).toBe(true);
if (!r.ok) continue;
const check = createMedicationSchema.safeParse(
medicationCreateBody(r.row, false),
);
expect(check.success ? [] : check.error.issues).toEqual([]);
}
if (daily.ok) {
expect(medicationCreateBody(daily.row, false)).toMatchObject({
notificationsEnabled: false,
schedules: [
{
windowStart: "08:00",
windowEnd: "20:00",
timesOfDay: ["08:00", "20:00"],
},
],
});
expect(medicationCreateBody(daily.row, true).notificationsEnabled).toBe(
true,
);
}
if (asNeeded.ok) {
expect(
medicationCreateBody(asNeeded.row, true).notificationsEnabled,
).toBe(false);
}
});
});
describe("structure", () => {
const read = (p: string) => readFileSync(join(process.cwd(), p), "utf8");
it("writes field by field and never spreads a parsed row", () => {
for (const file of [
"src/lib/import/labs-csv.ts",
"src/lib/import/medications-csv.ts",
]) {
const src = read(file);
expect(src).not.toMatch(/data:\s*\{\s*\.\.\.(row|parsed|data)\b/);
expect(src).not.toMatch(/\.\.\.parsed\.data/);
}
});
it("checks every medication row with the create route's schema", () => {
expect(read("src/lib/import/medications-csv.ts")).toContain(
"createMedicationSchema.safeParse(",
);
});
it("streams the body instead of reading it whole", () => {
const src = read("src/lib/import/record-import-route.ts");
expect(src).toContain("stream: request.body");
expect(src).not.toMatch(/request\.(text|json|arrayBuffer)\(/);
});
});
src/lib/import/csv-stream.ts
+267
View File
@@ -0,0 +1,267 @@
/**
* A streaming CSV reader for the record imports (#1215).
*
* The measurement importer reads its whole body into one string and splits
* it. That is fine for its 16 MB cap, but a lab or medication history kept
* over years in a spreadsheet arrives as one upload, and the import must hold
* a bounded amount of it at a time. This reader consumes the body chunk by
* chunk and hands out one record at a time; at any moment it holds the bytes
* of the record it is building and nothing before it.
*
* RFC 4180 plus what spreadsheets actually write:
* - a UTF-8 byte-order mark at the start is dropped;
* - the separator is `,`, `;` or a tab, detected from the header line
* outside quotes unless the caller names one (a German export writes `;`
* because `,` is its decimal mark);
* - fields may be quoted with `"`, a doubled `""` inside is one quote, and
* a quoted field may hold the separator and line breaks;
* - lines end in `\n`, `\r\n` or a lone `\r`;
* - a record of only empty fields (a trailing blank line) is skipped.
*
* Bounds: the body is counted while it is read and refused past `maxBytes`;
* a field longer than `MAX_FIELD_CHARS` or a record wider than
* `MAX_COLUMNS` marks that record `oversized` instead of growing without end
* (one runaway quote would otherwise swallow the rest of the file).
*/
export type CsvDelimiter = "," | ";" | "\t";
export const MAX_FIELD_CHARS = 4000;
export const MAX_COLUMNS = 100;
export interface CsvRecord {
/** 1-based physical line the record starts on. */
line: number;
fields: string[];
/** A field or the record ran past the bounds; its fields are cut short. */
oversized: boolean;
}
/** Count separators outside quotes in a header line; the most frequent wins. */
export function detectDelimiter(headerLine: string): CsvDelimiter {
const counts: Record<CsvDelimiter, number> = { ",": 0, ";": 0, "\t": 0 };
let quoted = false;
for (const ch of headerLine) {
if (ch === '"') quoted = !quoted;
else if (!quoted && (ch === "," || ch === ";" || ch === "\t")) {
counts[ch] += 1;
}
}
// Ties go to the comma, then the semicolon: a one-column file is a comma file.
let best: CsvDelimiter = ",";
for (const d of [";", "\t"] as const) {
if (counts[d] > counts[best]) best = d;
}
return best;
}
/**
* The incremental tokenizer. `push` takes the next piece of text and returns
* the records it completed; `end` flushes the last one. Pure and synchronous,
* so it is fuzzed without streams.
*/
export class CsvTokenizer {
private delimiter: CsvDelimiter | null;
private field = "";
private fields: string[] = [];
private quoted = false;
/** The previous character closed a quote, or was a quote inside quotes. */
private quotePending = false;
private afterCr = false;
private line = 1;
private recordLine = 1;
private oversized = false;
private started = false;
/** Text held back until the header line shows the separator. */
private pending = "";
constructor(delimiter: CsvDelimiter | "auto" = "auto") {
this.delimiter = delimiter === "auto" ? null : delimiter;
}
get detectedDelimiter(): CsvDelimiter | null {
return this.delimiter;
}
push(text: string): CsvRecord[] {
if (!this.started && text.length > 0) {
this.started = true;
if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
}
if (this.delimiter === null) {
this.pending += text;
const end = firstLineEnd(this.pending);
if (end === -1) {
// Bound the hold-back too: a header that never ends is a broken file.
if (this.pending.length <= MAX_FIELD_CHARS * MAX_COLUMNS) return [];
this.delimiter = detectDelimiter(this.pending);
} else {
this.delimiter = detectDelimiter(this.pending.slice(0, end));
}
text = this.pending;
this.pending = "";
}
const out: CsvRecord[] = [];
for (let i = 0; i < text.length; i++) {
this.consume(text[i], out);
}
return out;
}
end(): CsvRecord[] {
const out: CsvRecord[] = [];
if (this.delimiter === null) {
const held = this.pending;
this.pending = "";
this.delimiter = detectDelimiter(held);
for (const ch of held) this.consume(ch, out);
}
if (this.quotePending) {
// A closing quote right at the end of the input.
this.quoted = false;
this.quotePending = false;
}
if (this.field !== "" || this.fields.length > 0) this.finishRecord(out);
return out;
}
private append(ch: string) {
if (this.field.length < MAX_FIELD_CHARS) this.field += ch;
else this.oversized = true;
}
private finishField() {
if (this.fields.length < MAX_COLUMNS) this.fields.push(this.field);
else this.oversized = true;
this.field = "";
}
private finishRecord(out: CsvRecord[]) {
this.finishField();
const blank = this.fields.every((f) => f === "");
if (!blank) {
out.push({
line: this.recordLine,
fields: this.fields,
oversized: this.oversized,
});
}
this.fields = [];
this.oversized = false;
}
private consume(ch: string, out: CsvRecord[]) {
// A `\n` straight after a `\r` belongs to the same line end.
if (this.afterCr) {
this.afterCr = false;
if (ch === "\n") return;
}
if (this.quoted) {
if (this.quotePending) {
this.quotePending = false;
if (ch === '"') {
this.append('"');
return;
}
// The quote closed the field; fall through to unquoted handling.
this.quoted = false;
} else if (ch === '"') {
this.quotePending = true;
return;
} else {
if (ch === "\n" || ch === "\r") {
this.line += 1;
if (ch === "\r") this.afterCr = true;
this.append("\n");
return;
}
this.append(ch);
return;
}
}
if (ch === this.delimiter) {
this.finishField();
return;
}
if (ch === "\n" || ch === "\r") {
this.finishRecord(out);
this.line += 1;
this.recordLine = this.line;
if (ch === "\r") this.afterCr = true;
return;
}
if (ch === '"' && this.field === "") {
this.quoted = true;
return;
}
this.append(ch);
}
}
function firstLineEnd(text: string): number {
let quoted = false;
for (let i = 0; i < text.length; i++) {
const ch = text[i];
if (ch === '"') quoted = !quoted;
else if (!quoted && (ch === "\n" || ch === "\r")) return i;
}
return -1;
}
export class CsvTooLargeError extends Error {
constructor(readonly maxBytes: number) {
super(`CSV exceeds ${maxBytes} bytes`);
this.name = "CsvTooLargeError";
}
}
/**
* Read a byte stream as CSV records, one at a time. The bytes are counted as
* they arrive and the read stops with `CsvTooLargeError` past `maxBytes`, so
* a body without a Content-Length cannot run past the cap either.
*/
export async function* readCsvRecords(
stream: ReadableStream<Uint8Array>,
opts: { maxBytes: number; delimiter?: CsvDelimiter | "auto" },
): AsyncGenerator<CsvRecord, void, undefined> {
const tokenizer = new CsvTokenizer(opts.delimiter ?? "auto");
const decoder = new TextDecoder("utf-8");
const reader = stream.getReader();
let bytes = 0;
try {
for (;;) {
const { value, done } = await reader.read();
if (done) break;
bytes += value.byteLength;
if (bytes > opts.maxBytes) throw new CsvTooLargeError(opts.maxBytes);
for (const record of tokenizer.push(
decoder.decode(value, { stream: true }),
)) {
yield record;
}
}
for (const record of tokenizer.push(decoder.decode())) yield record;
for (const record of tokenizer.end()) yield record;
} finally {
reader.releaseLock();
}
}
/** A string as a one-chunk-at-a-time stream; for tests and the client preview. */
export function textToStream(
text: string,
chunkSize = 64 * 1024,
): ReadableStream<Uint8Array> {
const bytes = new TextEncoder().encode(text);
let offset = 0;
return new ReadableStream<Uint8Array>({
pull(controller) {
if (offset >= bytes.length) {
controller.close();
return;
}
controller.enqueue(bytes.slice(offset, offset + chunkSize));
offset += chunkSize;
},
});
}
src/lib/import/csv-values.ts
+230
View File
@@ -0,0 +1,230 @@
/**
* Reading numbers and dates out of spreadsheet cells (#1215).
*
* Conservative by rule: a cell that could mean two things is refused with
* its own reason rather than read one way. The person sees the line and the
* reason and fixes the file or the setting; a wrong number stored quietly is
* the one outcome an import must never have.
*/
import { zonedWallClockToUtc } from "@/lib/tz/wall-clock";
import { isCalendarDateKey } from "@/lib/tz/date-only";
export type DecimalMark = "dot" | "comma";
export type DateOrder = "iso" | "dmy" | "mdy";
export type NumberResult =
| { ok: true; value: number }
| { ok: false; reason: "invalid_number" | "ambiguous_number" };
/**
* A number written with the file's decimal mark. Grouping separators are
* accepted only in the unmistakable form (`1.234,5` with a comma mark,
* `1,234.5` with a dot mark). A lone grouping-shaped number in the other
* convention (`1,234` in a dot file, `1.234` in a comma file) is ambiguous:
* it is a thousand and something in one country and one point something in
* the next, and lab values live exactly in that range.
*/
export function parseLocaleNumber(
raw: string,
mark: DecimalMark,
): NumberResult {
let s = raw.trim().replace(/[   ]/g, "");
let sign = 1;
if (s.startsWith("-") || s.startsWith("−")) {
sign = -1;
s = s.slice(1);
} else if (s.startsWith("+")) {
s = s.slice(1);
}
if (!/^[\d.,]+$/.test(s)) return { ok: false, reason: "invalid_number" };
const dec = mark === "comma" ? "," : ".";
const grp = mark === "comma" ? "." : ",";
const e = (c: string) => `\\${c}`;
const read = (digits: string) => {
const value = sign * Number(digits);
return Number.isFinite(value)
? ({ ok: true, value } as const)
: ({ ok: false, reason: "invalid_number" } as const);
};
// Only the file's own decimal mark, at most once.
if (new RegExp(`^(\\d+(${e(dec)}\\d+)?|${e(dec)}\\d+)$`).test(s)) {
return read(s.replace(dec, "."));
}
// Grouped in the file's own convention: 1,234,567 / 1,234.5 (dot file).
if (new RegExp(`^\\d{1,3}(${e(grp)}\\d{3})+(${e(dec)}\\d+)?$`).test(s)) {
const groups = s.split(grp).length - 1;
// One group and no decimal part reads as the other convention's decimal
// too (1,234 = one point two three four in a comma country).
if (groups === 1 && !s.includes(dec)) {
return { ok: false, reason: "ambiguous_number" };
}
return read(s.split(grp).join("").replace(dec, "."));
}
// One mark of the other convention with not exactly three digits after it
// can only be a decimal mark (5,4 in a dot file, 5.4 in a comma file).
const other = new RegExp(`^\\d+${e(grp)}(\\d+)$`).exec(s);
if (other) {
if (other[1].length === 3) return { ok: false, reason: "ambiguous_number" };
return read(s.replace(grp, "."));
}
return { ok: false, reason: "invalid_number" };
}
export type DateResult =
| {
ok: true;
/** The instant to store. */
instant: Date;
/** The calendar date the cell states. */
dateKey: string;
/** The cell gave a date without a time of day. */
dateOnly: boolean;
}
| {
ok: false;
reason: "invalid_date" | "invalid_time" | "ambiguous_date";
};
const pad = (n: number) => String(n).padStart(2, "0");
/**
* A date cell (and an optional separate time cell) in the stated order.
*
* - `iso`: `YYYY-MM-DD`, optionally with `T`/space and `HH:mm[:ss]` and an
* offset (`Z`, `+02:00`), the shape every export tool can write;
* - `dmy`: `DD.MM.YYYY`, `DD/MM/YYYY` or `DD-MM-YYYY`;
* - `mdy`: `MM/DD/YYYY`.
*
* A time without an offset is a wall-clock time in `timezone` (the account's
* zone unless the import names another). A date without a time is stored as
* the date-only anchor the scanned and document readings use (noon UTC), so
* it is the same calendar date for everyone who opens the record.
*
* A two-digit year is refused as ambiguous. So is a `dmy`/`mdy` cell whose
* first number cannot be what the order says (13/02/2026 under `mdy`):
* that file is in the other order, and guessing it per row would mix both.
*/
export function parseCsvDate(
rawDate: string,
rawTime: string | undefined,
opts: { order: DateOrder; timezone: string },
): DateResult {
const cell = rawDate.trim();
if (cell === "") return { ok: false, reason: "invalid_date" };
let y: number;
let m: number;
let d: number;
let timePart: string | undefined;
let offset: string | undefined;
if (opts.order === "iso") {
const match =
/^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?)\s*(Z|[+-]\d{2}:?\d{2})?)?$/i.exec(
cell,
);
if (!match) {
return /^\d{2}-\d{2}-\d{2}$/.test(cell)
? { ok: false, reason: "ambiguous_date" }
: { ok: false, reason: "invalid_date" };
}
y = Number(match[1]);
m = Number(match[2]);
d = Number(match[3]);
timePart = match[4];
offset = match[5];
} else {
const match =
/^(\d{1,2})[./-](\d{1,2})[./-](\d{2}|\d{4})(?:[ T](\d{1,2}:\d{2}(?::\d{2})?))?$/.exec(
cell,
);
if (!match) return { ok: false, reason: "invalid_date" };
if (match[3].length === 2) return { ok: false, reason: "ambiguous_date" };
const first = Number(match[1]);
const second = Number(match[2]);
y = Number(match[3]);
[d, m] = opts.order === "dmy" ? [first, second] : [second, first];
if (m > 12 && d <= 12) return { ok: false, reason: "ambiguous_date" };
timePart = match[4];
}
const dateKey = `${String(y).padStart(4, "0")}-${pad(m)}-${pad(d)}`;
if (!isCalendarDateKey(dateKey)) return { ok: false, reason: "invalid_date" };
const time = (rawTime ?? "").trim() || timePart;
if (!time) {
return {
ok: true,
instant: new Date(`${dateKey}T12:00:00.000Z`),
dateKey,
dateOnly: true,
};
}
const tm = /^(\d{1,2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?$/.exec(time);
if (!tm) return { ok: false, reason: "invalid_time" };
const hour = Number(tm[1]);
const minute = Number(tm[2]);
const second = Number(tm[3] ?? 0);
if (hour > 23 || minute > 59 || second > 59) {
return { ok: false, reason: "invalid_time" };
}
if (offset) {
const norm =
offset.toUpperCase() === "Z"
? "Z"
: offset.replace(/^([+-]\d{2})(\d{2})$/, "$1:$2");
const instant = new Date(
`${dateKey}T${pad(hour)}:${pad(minute)}:${pad(second)}${norm}`,
);
if (Number.isNaN(instant.getTime()))
return { ok: false, reason: "invalid_time" };
return { ok: true, instant, dateKey, dateOnly: false };
}
const instant = zonedWallClockToUtc(
{ year: y, month: m, day: d, hour, minute, second },
opts.timezone,
);
return { ok: true, instant, dateKey, dateOnly: false };
}
/** A yes/no cell in the languages HealthLog speaks; anything else is null. */
export function parseYesNo(raw: string): boolean | null {
const v = raw.trim().toLowerCase();
if (v === "") return null;
if (
[
"1",
"yes",
"y",
"true",
"x",
"ja",
"j",
"sí",
"si",
"oui",
"tak",
"예",
"wahr",
].includes(v)
) {
return true;
}
if (
[
"0",
"no",
"n",
"false",
"nein",
"non",
"nie",
"nee",
"아니오",
"falsch",
].includes(v)
) {
return false;
}
return null;
}
src/lib/import/labs-csv.ts
+552
View File
@@ -0,0 +1,552 @@
/**
* CSV import of lab results (#1215).
*
* A row becomes a lab reading the way a scanned report's confirmed row does
* (`/api/labs/ocr/commit`): the name resolves to the account's marker by the
* same index (name, alias, catalog), a marker is minted from the first row
* that names a new analyte, a reading in another unit is converted when the
* conversion is known and kept beside what the file said, and a reading the
* record already holds is skipped. Nothing about a row is guessed: an
* ambiguous number or date, a numeric value without a unit, or a unit that
* does not convert into the marker's is refused with its reason and line.
*
* Memory: the file is read record by record (`readCsvRecords`) and written in
* chunks of `CHUNK_ROWS`; what is held across chunks is the marker index, the
* duplicate keys of rows already accepted and the bounded report.
*/
import { prisma } from "@/lib/db";
import {
readCsvRecords,
type CsvDelimiter,
type CsvRecord,
} from "@/lib/import/csv-stream";
import { parseCsvDate, parseLocaleNumber } from "@/lib/import/csv-values";
import { RecordImportReport } from "@/lib/import/record-import-report";
import { matchAnalyteKey } from "@/lib/labs/analyte-match";
import {
addToIndex,
loadMarkerIndex,
resolveInIndex,
unitFitsMarker,
type IndexedMarker,
type MarkerIndex,
} from "@/lib/labs/biomarker-resolve";
import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store";
import { parseReferenceRange } from "@/lib/labs/parse-reference-range";
import { readingInMarkerUnit } from "@/lib/labs/reading-conversion";
import {
labReadingDay,
labReadingDaySearchRange,
} from "@/lib/labs/reading-day";
import { encryptNoteToBytes } from "@/lib/labs/store";
import type {
LabsCsvQuery,
RecordImportResult,
RecordImportRowReason,
} from "@/lib/validations/record-import";
export const LABS_CSV_MAX_ROWS = 20_000;
const CHUNK_ROWS = 200;
const MAX_LISTED_MARKERS = 200;
/** A reading dated more than a day ahead is a typing mistake, not a result. */
const FUTURE_SLACK_MS = 86_400_000;
const EARLIEST = Date.UTC(1900, 0, 1);
export const LAB_TEXT_LIMITS = {
analyte: 200,
valueText: 200,
unit: 80,
panel: 100,
note: 1000,
} as const;
export interface LabCsvRow {
line: number;
analyte: string;
value: number | null;
valueText: string | null;
unit: string | null;
takenAt: Date;
sourceReferenceLow: number | null;
sourceReferenceHigh: number | null;
sourceReferenceText: string | null;
panel: string | null;
note: string | null;
}
export type LabRowResult =
| { ok: true; row: LabCsvRow }
| { ok: false; reason: RecordImportRowReason; field: string };
function cell(fields: string[], index: number | undefined): string {
if (index === undefined) return "";
return (fields[index] ?? "").trim();
}
/** One CSV record → a lab row, or the first reason it cannot be one. Pure. */
export function normaliseLabRow(
record: Pick<CsvRecord, "line" | "fields">,
query: LabsCsvQuery,
timezone: string,
now: Date = new Date(),
): LabRowResult {
const f = record.fields;
const fail = (
reason: RecordImportRowReason,
field: string,
): LabRowResult => ({ ok: false, reason, field });
const analyte = cell(f, query.analyte);
if (!analyte) return fail("missing_value", "analyte");
if (analyte.length > LAB_TEXT_LIMITS.analyte)
return fail("text_too_long", "analyte");
const rawValue = cell(f, query.value);
if (!rawValue) return fail("missing_value", "value");
let value: number | null = null;
let valueText: string | null = null;
const parsed = parseLocaleNumber(rawValue, query.decimal);
if (parsed.ok) {
value = parsed.value;
} else if (parsed.reason === "ambiguous_number") {
return fail("ambiguous_number", "value");
} else if (/^[+-]?[\d.,\s]+$/.test(rawValue)) {
// Digits and marks that still do not read as a number: a broken number,
// not a qualitative result.
return fail("invalid_number", "value");
} else {
if (rawValue.length > LAB_TEXT_LIMITS.valueText)
return fail("text_too_long", "value");
valueText = rawValue;
}
const unit = cell(f, query.unit) || null;
if (value !== null && !unit) return fail("missing_unit", "unit");
if (unit && unit.length > LAB_TEXT_LIMITS.unit)
return fail("text_too_long", "unit");
const date = parseCsvDate(
cell(f, query.date),
query.time === undefined ? undefined : cell(f, query.time),
{
order: query.dateOrder,
timezone,
},
);
if (!date.ok) return fail(date.reason, "date");
if (date.instant.getTime() > now.getTime() + FUTURE_SLACK_MS)
return fail("future_date", "date");
if (date.instant.getTime() < EARLIEST) return fail("invalid_date", "date");
let low: number | null = null;
let high: number | null = null;
let rangeText: string | null = null;
for (const [key, index] of [
["referenceLow", query.referenceLow],
["referenceHigh", query.referenceHigh],
] as const) {
const raw = cell(f, index);
if (!raw) continue;
const n = parseLocaleNumber(raw, query.decimal);
if (!n.ok)
return fail(
n.reason === "ambiguous_number"
? "ambiguous_number"
: "invalid_reference_range",
key,
);
if (key === "referenceLow") low = n.value;
else high = n.value;
}
const rawRange = cell(f, query.referenceRange);
if (rawRange) {
// A comma-decimal file writes ranges as "4,0 - 5,6"; the range reader
// speaks the dot, so the marks are swapped first for such a file.
const forReader =
query.decimal === "comma"
? rawRange.replace(/(\d),(\d)/g, "$1.$2")
: rawRange;
const printed = parseReferenceRange(forReader, unit);
if (printed) {
low ??= printed.low;
high ??= printed.high;
rangeText = rawRange.replace(/\s+/g, " ").slice(0, 120);
}
}
if (low !== null && high !== null && low > high) {
return fail("invalid_reference_range", "referenceRange");
}
const panel = cell(f, query.panel) || null;
if (panel && panel.length > LAB_TEXT_LIMITS.panel)
return fail("text_too_long", "panel");
const note = cell(f, query.note) || null;
if (note && note.length > LAB_TEXT_LIMITS.note)
return fail("text_too_long", "note");
return {
ok: true,
row: {
line: record.line,
analyte,
value,
valueText,
unit,
takenAt: date.instant,
sourceReferenceLow: valueText === null ? low : null,
sourceReferenceHigh: valueText === null ? high : null,
sourceReferenceText: valueText === null ? rangeText : null,
panel,
note,
},
};
}
/**
* The keys a reading is a duplicate under, mirroring the scan commit's
* `isDuplicate`: the marker (or the name, for a reading filed without one),
* the reading's calendar day, and the value as stored, as printed, or as text.
*/
export function readingKeys(
reading: {
biomarkerId: string | null;
analyte: string;
takenAt: Date;
value: number | null;
valueText: string | null;
sourceValue?: number | null;
sourceUnit?: string | null;
unit?: string | null;
},
tz: string,
): string[] {
const who = reading.biomarkerId
? `bm:${reading.biomarkerId}`
: `an:${reading.analyte.trim().toLowerCase()}`;
const day = labReadingDay(reading.takenAt, tz);
const keys: string[] = [];
if (reading.value !== null) keys.push(`${who}|${day}|n:${reading.value}`);
if (
reading.sourceValue !== null &&
reading.sourceValue !== undefined &&
reading.sourceUnit
) {
keys.push(
`${who}|${day}|s:${reading.sourceValue}:${reading.sourceUnit.trim().toLowerCase()}`,
);
}
if (reading.valueText !== null) {
keys.push(`${who}|${day}|t:${reading.valueText.trim().toLowerCase()}`);
}
return keys;
}
interface Planned {
row: LabCsvRow;
marker: IndexedMarker | null;
}
function displayRow(row: LabCsvRow, marker: string) {
return {
line: row.line,
marker,
value: row.value,
valueText: row.valueText,
unit: row.unit,
takenAt: row.takenAt.toISOString(),
referenceLow: row.sourceReferenceLow,
referenceHigh: row.sourceReferenceHigh,
panel: row.panel,
note: row.note !== null,
};
}
/**
* Read, check and (unless a dry run) write a lab CSV. Throws
* `CsvTooLargeError` past the byte cap; everything else is reported per row.
*/
export async function importLabsCsv(args: {
userId: string;
tz: string;
stream: ReadableStream<Uint8Array>;
maxBytes: number;
query: LabsCsvQuery;
now?: Date;
}): Promise<RecordImportResult> {
const { userId, tz, query } = args;
const dryRun = query.dryRun;
const timezone = query.timezone ?? tz;
const report = new RecordImportReport("labs", dryRun);
const index: MarkerIndex = await loadMarkerIndex(prisma, userId);
// A dry run works on a copy, so the markers it pretends to mint never leak.
const working: MarkerIndex = dryRun
? { markers: [...index.markers], aliases: [...index.aliases] }
: index;
const matched = new Map<
string,
{ name: string; marker: string; how: "name" | "alias" | "catalog" }
>();
const created = new Set<string>();
const seen = new Set<string>();
let chunk: LabCsvRow[] = [];
let pendingId = 0;
const delimiter: CsvDelimiter | "auto" =
query.delimiter === "comma"
? ","
: query.delimiter === "semicolon"
? ";"
: query.delimiter === "tab"
? "\t"
: "auto";
const flush = async () => {
if (chunk.length === 0) return;
const rows = chunk;
chunk = [];
// 1. Resolve every row's marker; a dry run pretends to mint.
const planned: Planned[] = [];
for (const row of rows) {
const hit = resolveInIndex(
working,
row.analyte,
row.valueText === null ? row.unit : null,
);
let marker: IndexedMarker | null = hit?.marker ?? null;
if (hit) {
// Listed once per spelling, so the person sees which of the file's
// names landed on which of their markers.
const key = row.analyte.toLowerCase();
if (!matched.has(key) && matched.size < MAX_LISTED_MARKERS) {
matched.set(key, {
name: row.analyte,
marker: hit.marker.name,
how: hit.matchKind,
});
}
} else if (dryRun) {
const unit = row.unit ?? "";
marker = {
id: `pending:${pendingId++}`,
name: row.analyte,
unit,
lowerBound: row.sourceReferenceLow,
upperBound: row.sourceReferenceHigh,
panel: row.panel,
analyteKey: matchAnalyteKey({ name: row.analyte, unit }),
hidden: false,
createdAt: new Date(),
};
addToIndex(working, marker);
if (created.size < MAX_LISTED_MARKERS) created.add(row.analyte);
}
// The unit must fit the marker, or convert into it.
if (
marker &&
row.value !== null &&
// The key as the write derives it (`ensureAnalyteKey`), so the check
// and the conversion the import then runs cannot disagree.
!unitFitsMarker(
{
unit: marker.unit,
analyteKey:
matchAnalyteKey({ name: marker.name, unit: marker.unit }) ??
marker.analyteKey,
},
row.unit,
null,
)
) {
report.fail(row.line, "unit_mismatch", "unit");
continue;
}
planned.push({ row, marker });
}
// 2. Duplicates against the record, one query for the chunk.
const days = planned.map((p) => labReadingDay(p.row.takenAt, tz)).sort();
const existingKeys = new Set<string>();
if (planned.length > 0) {
const from = labReadingDaySearchRange(days[0]).gte;
const to = labReadingDaySearchRange(days[days.length - 1]).lt;
const ids = [
...new Set(
planned
.map((p) => p.marker?.id)
.filter((id): id is string => !!id && !id.startsWith("pending:")),
),
];
const names = [...new Set(planned.map((p) => p.row.analyte))];
const existing = await prisma.labResult.findMany({
where: {
userId,
deletedAt: null,
takenAt: { gte: from, lt: to },
OR: [{ biomarkerId: { in: ids } }, { analyte: { in: names } }],
},
select: {
biomarkerId: true,
analyte: true,
takenAt: true,
value: true,
valueText: true,
sourceValue: true,
sourceUnit: true,
},
});
for (const e of existing) {
for (const k of readingKeys(e, tz)) existingKeys.add(k);
// A reading filed under a marker also answers to its printed name.
if (e.biomarkerId) {
for (const k of readingKeys({ ...e, biomarkerId: null }, tz))
existingKeys.add(k);
}
}
}
// 3. Write (or count) what is new.
for (const { row, marker } of planned) {
const probe = {
biomarkerId:
marker && !marker.id.startsWith("pending:") ? marker.id : null,
analyte: row.analyte,
takenAt: row.takenAt,
value: row.value,
valueText: row.valueText,
sourceValue: row.value,
sourceUnit: row.unit,
};
const keys = [
...readingKeys(probe, tz),
...readingKeys({ ...probe, biomarkerId: null }, tz),
...(marker
? readingKeys({ ...probe, biomarkerId: marker.id }, tz)
: []),
];
if (keys.some((k) => existingKeys.has(k) || seen.has(k))) {
report.duplicate(row.line);
continue;
}
if (dryRun) {
for (const k of keys) seen.add(k);
report.valid += 1;
report.inserted += 1;
report.addPreview(displayRow(row, marker?.name ?? row.analyte));
continue;
}
const biomarker =
marker ??
(await resolveOrMintBiomarker(
userId,
{
analyte: row.analyte,
unit: row.unit ?? "",
referenceLow:
row.valueText === null ? row.sourceReferenceLow : null,
referenceHigh:
row.valueText === null ? row.sourceReferenceHigh : null,
panel: row.panel,
},
{ index: working },
));
if (!marker) created.add(row.analyte);
let storedValue = row.value;
let storedLow = row.sourceReferenceLow;
let storedHigh = row.sourceReferenceHigh;
let convertedFrom: { sourceValue: number; sourceUnit: string } | null =
null;
if (row.value !== null) {
const reading = await readingInMarkerUnit(prisma, {
marker: biomarker,
value: row.value,
unit: row.unit ?? undefined,
sourceReferenceLow: storedLow,
sourceReferenceHigh: storedHigh,
});
if (!reading.ok) {
report.fail(row.line, "unit_mismatch", "unit");
continue;
}
storedValue = reading.value;
storedLow = reading.sourceReferenceLow;
storedHigh = reading.sourceReferenceHigh;
if (reading.converted) {
convertedFrom = {
sourceValue: reading.sourceValue,
sourceUnit: reading.sourceUnit,
};
}
}
// Field by field, never a spread of the parsed row.
await prisma.labResult.create({
data: {
userId,
biomarkerId: biomarker.id,
panel: row.panel ?? biomarker.panel,
analyte: biomarker.name,
value: storedValue,
valueText: row.valueText,
unit: biomarker.unit,
sourceValue: convertedFrom?.sourceValue ?? null,
sourceUnit: convertedFrom?.sourceUnit ?? null,
referenceLow: biomarker.lowerBound,
referenceHigh: biomarker.upperBound,
sourceReferenceLow: storedLow,
sourceReferenceHigh: storedHigh,
sourceReferenceText: row.sourceReferenceText,
takenAt: row.takenAt,
source: "IMPORT",
noteEncrypted: row.note ? encryptNoteToBytes(row.note) : null,
},
});
for (const k of [
...keys,
...readingKeys({ ...probe, biomarkerId: biomarker.id }, tz),
])
seen.add(k);
report.valid += 1;
report.inserted += 1;
}
};
let first = true;
for await (const record of readCsvRecords(args.stream, {
maxBytes: args.maxBytes,
delimiter,
})) {
if (first && query.header) {
first = false;
continue;
}
first = false;
report.rows += 1;
if (report.rows > LABS_CSV_MAX_ROWS) {
report.fail(record.line, "row_limit");
continue;
}
if (record.oversized) {
report.fail(record.line, "oversized_row");
continue;
}
const result = normaliseLabRow(record, query, timezone, args.now);
if (!result.ok) {
report.fail(record.line, result.reason, result.field);
continue;
}
chunk.push(result.row);
if (chunk.length >= CHUNK_ROWS) await flush();
}
await flush();
return report.toResult({
markers: {
matched: [...matched.values()],
created: [...created].slice(0, MAX_LISTED_MARKERS),
},
});
}
src/lib/import/medications-csv.ts
+331
View File
@@ -0,0 +1,331 @@
/**
* CSV import of medications (#1215).
*
* Conservative by design: a row maps only what is unambiguous in a
* spreadsheet and in the medication model alike. Name and dose (as written),
* daily intake times, an as-needed flag, and a start and end date. Every other
* rhythm (weekdays, every other week, cycles, injections) is set up in the
* app afterwards, where the person sees what it means. A row needs either
* times or the as-needed mark: the medication model holds no scheduled
* medication without a time, and an import does not invent one.
*
* Every row is checked against `createMedicationSchema`, the schema the
* create route parses with, so an import can never store a medication the
* form could not. Reminders stay off unless the import asks for them, so an
* upload never starts notifying on its own. A medication the record already
* holds under the same name and dose, or an earlier row of the file, is
* skipped.
*/
import { prisma } from "@/lib/db";
import { invalidateUserMedications } from "@/lib/cache/invalidate";
import {
readCsvRecords,
type CsvDelimiter,
type CsvRecord,
} from "@/lib/import/csv-stream";
import { parseCsvDate, parseYesNo } from "@/lib/import/csv-values";
import { RecordImportReport } from "@/lib/import/record-import-report";
import { foldForMatch } from "@/lib/i18n/fold-for-match";
import { setMedicationCategory } from "@/lib/medication-category";
import { serializeScheduleRecurrence } from "@/lib/medication-schedule";
import {
dateOfDayKey,
dayKeyOfDate,
validateCourses,
} from "@/lib/medications/course-window";
import { userDayKey } from "@/lib/tz/format";
import { createMedicationSchema } from "@/lib/validations/medication";
import type {
MedicationsCsvQuery,
RecordImportResult,
RecordImportRowReason,
} from "@/lib/validations/record-import";
export const MEDICATIONS_CSV_MAX_ROWS = 500;
export interface MedicationCsvRow {
line: number;
name: string;
dose: string;
times: string[];
asNeeded: boolean;
startsOn: string | null;
endsOn: string | null;
}
export type MedicationRowResult =
| { ok: true; row: MedicationCsvRow }
| { ok: false; reason: RecordImportRowReason; field: string };
function cell(fields: string[], index: number | undefined): string {
if (index === undefined) return "";
return (fields[index] ?? "").trim();
}
/** `8:00`, `08:00 20:00`, `08:00|20:00`, `8.00; 20.00` → sorted `HH:mm`s. */
export function parseTimes(raw: string): string[] | null {
if (raw === "") return [];
const parts = raw.split(/[\s,;|/]+/).filter(Boolean);
const out = new Set<string>();
for (const part of parts) {
const m = /^(\d{1,2})[:.](\d{2})$/.exec(part);
if (!m) return null;
const h = Number(m[1]);
const min = Number(m[2]);
if (h > 23 || min > 59) return null;
out.add(`${String(h).padStart(2, "0")}:${String(min).padStart(2, "0")}`);
}
const sorted = [...out].sort();
return sorted.length > 8 ? null : sorted;
}
/** One record → a medication row, or the first reason it cannot be one. Pure. */
export function normaliseMedicationRow(
record: Pick<CsvRecord, "line" | "fields">,
query: MedicationsCsvQuery,
timezone: string,
): MedicationRowResult {
const f = record.fields;
const fail = (
reason: RecordImportRowReason,
field: string,
): MedicationRowResult => ({ ok: false, reason, field });
const name = cell(f, query.name);
if (!name) return fail("missing_value", "name");
if (name.length > 100) return fail("text_too_long", "name");
const dose = cell(f, query.dose);
if (!dose) return fail("missing_value", "dose");
if (dose.length > 50) return fail("text_too_long", "dose");
const times = parseTimes(cell(f, query.times));
if (times === null) return fail("invalid_times", "times");
let asNeeded = false;
const rawAsNeeded = cell(f, query.asNeeded);
if (rawAsNeeded) {
const yes = parseYesNo(rawAsNeeded);
if (yes === null) return fail("invalid_yes_no", "asNeeded");
asNeeded = yes;
}
if (asNeeded && times.length > 0)
return fail("as_needed_with_times", "times");
if (!asNeeded && times.length === 0) return fail("missing_times", "times");
const day = (field: "startsOn" | "endsOn") => {
const raw = cell(f, query[field]);
if (!raw) return { ok: true as const, key: null };
const parsed = parseCsvDate(raw, undefined, {
order: query.dateOrder,
timezone,
});
if (!parsed.ok) return { ok: false as const, reason: parsed.reason };
return { ok: true as const, key: parsed.dateKey };
};
const start = day("startsOn");
if (!start.ok) return fail(start.reason, "startsOn");
const end = day("endsOn");
if (!end.ok) return fail(end.reason, "endsOn");
if (start.key && end.key && end.key < start.key) {
return fail("end_before_start", "endsOn");
}
return {
ok: true,
row: {
line: record.line,
name,
dose,
times,
asNeeded,
startsOn: start.key,
endsOn: end.key,
},
};
}
/** The create body this row stands for, exactly as the form would send it. */
export function medicationCreateBody(
row: MedicationCsvRow,
reminders: boolean,
) {
const windowEnd = (() => {
if (row.times.length > 1) return row.times[row.times.length - 1];
const [h, m] = (row.times[0] ?? "08:00").split(":").map(Number);
return `${String((h + 1) % 24).padStart(2, "0")}:${String(m).padStart(2, "0")}`;
})();
return {
name: row.name,
dose: row.dose,
asNeeded: row.asNeeded,
notificationsEnabled: reminders && row.times.length > 0,
...(row.startsOn ? { startsOn: row.startsOn } : {}),
...(row.endsOn ? { endsOn: row.endsOn } : {}),
...(row.times.length > 0
? {
schedules: [
{
windowStart: row.times[0],
windowEnd,
timesOfDay: row.times,
},
],
}
: {}),
};
}
const identity = (name: string, dose: string) =>
`${foldForMatch(name)}|${foldForMatch(dose).replace(/\s+/g, "")}`;
export async function importMedicationsCsv(args: {
userId: string;
tz: string;
stream: ReadableStream<Uint8Array>;
maxBytes: number;
query: MedicationsCsvQuery;
now?: Date;
}): Promise<RecordImportResult> {
const { userId, tz, query } = args;
const dryRun = query.dryRun;
const timezone = query.timezone ?? tz;
const report = new RecordImportReport("medications", dryRun);
const todayKey = userDayKey(args.now ?? new Date(), tz);
const existing = await prisma.medication.findMany({
where: { userId },
select: { name: true, dose: true },
});
const seen = new Set(existing.map((m) => identity(m.name, m.dose)));
const delimiter: CsvDelimiter | "auto" =
query.delimiter === "comma"
? ","
: query.delimiter === "semicolon"
? ";"
: query.delimiter === "tab"
? "\t"
: "auto";
let first = true;
for await (const record of readCsvRecords(args.stream, {
maxBytes: args.maxBytes,
delimiter,
})) {
if (first && query.header) {
first = false;
continue;
}
first = false;
report.rows += 1;
if (report.rows > MEDICATIONS_CSV_MAX_ROWS) {
report.fail(record.line, "row_limit");
continue;
}
if (record.oversized) {
report.fail(record.line, "oversized_row");
continue;
}
const result = normaliseMedicationRow(record, query, timezone);
if (!result.ok) {
report.fail(record.line, result.reason, result.field);
continue;
}
const row = result.row;
const parsed = createMedicationSchema.safeParse(
medicationCreateBody(row, query.reminders),
);
if (!parsed.success) {
report.fail(row.line, "invalid_medication");
continue;
}
// The course the create route would open for this window.
let course: { startsOn: Date; endsOn: Date | null } | null = null;
if (parsed.data.startsOn || parsed.data.endsOn) {
const endKey = parsed.data.endsOn
? dayKeyOfDate(parsed.data.endsOn)
: null;
course = {
startsOn:
parsed.data.startsOn ??
dateOfDayKey(endKey && endKey < todayKey ? endKey : todayKey),
endsOn: parsed.data.endsOn ?? null,
};
if (validateCourses([course], todayKey, false)) {
report.fail(row.line, "end_before_start", "endsOn");
continue;
}
}
const key = identity(row.name, row.dose);
if (seen.has(key)) {
report.duplicate(row.line);
continue;
}
seen.add(key);
report.valid += 1;
report.inserted += 1;
if (dryRun) {
report.addPreview({
line: row.line,
name: row.name,
dose: row.dose,
times: row.times.join(" "),
asNeeded: row.asNeeded,
startsOn: row.startsOn,
endsOn: row.endsOn,
});
continue;
}
const data = parsed.data;
// Field by field, with the create route's invariants for the shapes an
// import produces: a daily schedule (FREQ=DAILY, times dual-written), an
// opening course for a dated window, no schedule for as-needed.
const medication = await prisma.medication.create({
data: {
userId,
name: data.name,
dose: data.dose,
asNeeded: data.asNeeded ?? false,
notificationsEnabled: data.notificationsEnabled ?? false,
...(course
? {
startsOn: course.startsOn,
endsOn: course.endsOn,
courses: {
create: {
userId,
startsOn: course.startsOn,
endsOn: course.endsOn,
},
},
}
: {}),
schedules: {
create: (data.schedules ?? []).map((s) => ({
windowStart: s.windowStart,
windowEnd: s.windowEnd,
label: null,
dose: null,
daysOfWeek: serializeScheduleRecurrence({
daysOfWeek: [],
intervalWeeks: 1,
}),
timesOfDay:
s.timesOfDay && s.timesOfDay.length > 0
? s.timesOfDay
: [s.windowStart],
rrule: "FREQ=DAILY",
})),
},
},
select: { id: true },
});
await setMedicationCategory(medication.id, undefined);
}
if (!dryRun && report.inserted > 0) {
invalidateUserMedications(userId, { evict: true });
}
return report.toResult();
}
src/lib/import/record-import-report.ts
+75
View File
@@ -0,0 +1,75 @@
/**
* The per-import report both record importers fill (#1215): counts, the row
* errors in file order and a short preview, each bounded so a file of twenty
* thousand broken rows answers with a report of a fixed size.
*/
import type {
RecordImportResult,
RecordImportRowReason,
} from "@/lib/validations/record-import";
export const MAX_REPORTED_ERRORS = 500;
export const MAX_PREVIEW_ROWS = 20;
export type PreviewRow = Record<string, string | number | boolean | null>;
export class RecordImportReport {
rows = 0;
valid = 0;
inserted = 0;
duplicates = 0;
failed = 0;
private errors: RecordImportResult["errors"] = [];
private truncated = false;
private preview: PreviewRow[] = [];
constructor(
readonly kind: RecordImportResult["kind"],
readonly dryRun: boolean,
) {}
private note(line: number, reason: RecordImportRowReason, field?: string) {
if (this.errors.length < MAX_REPORTED_ERRORS) {
this.errors.push(field ? { line, reason, field } : { line, reason });
} else {
this.truncated = true;
}
}
fail(line: number, reason: RecordImportRowReason, field?: string) {
this.failed += 1;
this.note(line, reason, field);
}
duplicate(line: number) {
this.duplicates += 1;
this.note(line, "duplicate");
}
addPreview(row: PreviewRow) {
if (this.dryRun && this.preview.length < MAX_PREVIEW_ROWS) {
this.preview.push(row);
}
}
toResult(
extra: Pick<RecordImportResult, "markers"> = {},
): RecordImportResult {
// Errors arrive per chunk; a chunk's duplicates are found after its
// refusals, so put the list back in file order.
const errors = [...this.errors].sort((a, b) => a.line - b.line);
return {
kind: this.kind,
dryRun: this.dryRun,
rows: this.rows,
valid: this.valid,
inserted: this.inserted,
duplicates: this.duplicates,
failed: this.failed,
errors,
errorsTruncated: this.truncated,
...extra,
preview: this.preview,
};
}
}
src/lib/import/record-import-route.ts
+150
View File
@@ -0,0 +1,150 @@
/**
* The request side the two CSV record imports share (#1215): owner-only auth,
* the module gate, the rate buckets, the query parse, the content type and
* the streamed body with its byte cap. Each route supplies its schema and its
* importer; what a row becomes lives in `labs-csv.ts` / `medications-csv.ts`.
*
* Buckets: a dry run spends `import-preview:<id>` (30 an hour), because the
* mapping screen re-checks the file whenever a column choice changes; the
* real import spends the shared `import:<id>` bucket (5 an hour) every other
* importer uses.
*/
import type { NextRequest } from "next/server";
import type { z } from "zod/v4";
import { requireAuth } from "@/lib/api-handler";
import {
apiError,
apiSuccess,
getClientIp,
returnAllZodIssues,
} from "@/lib/api-response";
import { auditLog } from "@/lib/auth/audit";
import { CsvTooLargeError } from "@/lib/import/csv-stream";
import { annotate } from "@/lib/logging/context";
import { requireModuleEnabled } from "@/lib/modules/gate";
import type { ModuleKey } from "@/lib/modules/registry";
import { checkRateLimit } from "@/lib/rate-limit";
import { resolveUserTimezone } from "@/lib/tz/resolver";
import type { RecordImportResult } from "@/lib/validations/record-import";
export const RECORD_IMPORT_MAX_BYTES = 16 * 1024 * 1024;
const PREVIEW_PER_HOUR = 30;
const IMPORT_PER_HOUR = 5;
const HOUR_MS = 60 * 60 * 1000;
export async function handleRecordImport<
S extends z.ZodType<{ dryRun: boolean }>,
>(
request: NextRequest,
opts: {
kind: RecordImportResult["kind"];
module: ModuleKey;
schema: S;
run: (args: {
userId: string;
tz: string;
stream: ReadableStream<Uint8Array>;
maxBytes: number;
query: z.infer<S>;
}) => Promise<RecordImportResult>;
},
): Promise<Response> {
const { user } = await requireAuth();
const action = `import.${opts.kind}.csv`;
annotate({ action: { name: action } });
const gate = await requireModuleEnabled(user.id, opts.module);
if (!gate.enabled) return gate.response;
const parsed = opts.schema.safeParse(
Object.fromEntries(new URL(request.url).searchParams),
);
if (!parsed.success) return returnAllZodIssues(parsed.error, 422);
const query = parsed.data;
const rl = query.dryRun
? await checkRateLimit(
`import-preview:${user.id}`,
PREVIEW_PER_HOUR,
HOUR_MS,
)
: await checkRateLimit(`import:${user.id}`, IMPORT_PER_HOUR, HOUR_MS);
if (!rl.allowed) {
return apiError(
query.dryRun
? `Maximum ${PREVIEW_PER_HOUR} checks per hour`
: `Maximum ${IMPORT_PER_HOUR} imports per hour`,
429,
{ errorCode: "import.rate_limited" },
);
}
const contentType = (request.headers.get("content-type") ?? "").toLowerCase();
if (
!contentType.includes("text/csv") &&
!contentType.includes("text/plain")
) {
return apiError("Content-Type must be text/csv", 415, {
errorCode: "import.csv.content_type",
});
}
const declared = Number(request.headers.get("content-length") ?? "0");
if (Number.isFinite(declared) && declared > RECORD_IMPORT_MAX_BYTES) {
return apiError("CSV exceeds the 16 MB limit", 413, {
errorCode: "import.csv.too_large",
});
}
if (!request.body) {
return apiError("The CSV file is empty", 422, {
errorCode: "import.csv.empty",
});
}
const tz = await resolveUserTimezone(user.id);
let result: RecordImportResult;
try {
result = await opts.run({
userId: user.id,
tz,
stream: request.body,
maxBytes: RECORD_IMPORT_MAX_BYTES,
query,
});
} catch (err) {
if (err instanceof CsvTooLargeError) {
return apiError("CSV exceeds the 16 MB limit", 413, {
errorCode: "import.csv.too_large",
});
}
throw err;
}
if (result.rows === 0) {
return apiError("The CSV file has no data rows", 422, {
errorCode: "import.csv.empty",
});
}
if (!query.dryRun) {
await auditLog(action, {
userId: user.id,
ipAddress: getClientIp(request),
details: {
rows: result.rows,
inserted: result.inserted,
duplicates: result.duplicates,
failed: result.failed,
},
});
}
annotate({
meta: {
import_dry_run: query.dryRun,
import_rows: result.rows,
import_inserted: result.inserted,
import_duplicates: result.duplicates,
import_failed: result.failed,
},
});
return apiSuccess(result);
}
src/lib/openapi/error-codes.ts
+8 -1
View File
@@ -226,7 +226,14 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
"illness.episode.parent-not-found",
"illness.insights.invalid",
],
import: ["import.apple_health.busy", "import.write_failed"],
import: [
"import.apple_health.busy",
"import.csv.content_type",
"import.csv.empty",
"import.csv.too_large",
"import.rate_limited",
"import.write_failed",
],
ingest: ["ingest.rate_limited"],
insights: [
"insights.generate.budgetExceeded",
src/lib/openapi/routes/import.ts
+77
View File
@@ -23,6 +23,41 @@ import {
stdResponses,
} from "./shared";
import { isoDateTime } from "@/lib/validations/datetime";
import {
labsCsvQuerySchema,
medicationsCsvQuerySchema,
recordImportResultSchema,
} from "@/lib/validations/record-import";
const RECORD_IMPORT_COMMON =
"Body is the raw CSV file (`text/csv`, ≤ 16 MB), read as a stream: a UTF-8 byte-order mark is dropped, the separator is `,`, `;` or a tab, quoted fields may hold separators and line breaks. Every row is checked on its own and a refused row is reported by line with a reason, never guessed (a number or date that reads two ways is refused). Re-uploading the same file writes nothing: every row is reported as a duplicate. `dryRun=1` checks the whole file without writing and returns the first rows as they would be stored; it spends a separate bucket of 30 checks an hour, the import itself the shared 5-an-hour `import:` bucket. Owner only; refused with `module.disabled` when the module is switched off.";
function recordImportResponses(id: string) {
return {
"200": {
description: "Per-file and per-row outcome.",
content: {
"application/json": {
schema: dataEnvelope(recordImportResultSchema, id),
},
},
},
...stdResponses,
"403": {
description:
"`module.disabled`: the module the import writes to is switched off.",
content: { "application/json": { schema: errorEnvelope } },
},
"413": {
description: "`import.csv.too_large`: the file is larger than 16 MB.",
content: { "application/json": { schema: errorEnvelope } },
},
"415": {
description: "`import.csv.content_type`: the body is not `text/csv`.",
content: { "application/json": { schema: errorEnvelope } },
},
};
}
const csvImportRowResult = z
.object({
@@ -351,6 +386,48 @@ export const importPaths: NonNullable<ZodOpenApiObject["paths"]> = {
},
},
},
"/api/import/labs-csv": {
post: {
tags: ["Import"],
summary: "Import lab results from CSV",
description: `${RECORD_IMPORT_COMMON} Each row's name resolves to the account's marker the way a scanned report's rows do (name, alias, catalog name); a new analyte mints one marker from its first row. A numeric result needs a unit; one in another unit is converted when the conversion is known and keeps the printed value beside it, otherwise the row is refused with \`unit_mismatch\`. Any other text in the value column is a qualitative result. A date without a time is stored as that calendar date. At most 20 000 data rows.`,
requestParams: { query: labsCsvQuerySchema },
requestBody: {
required: true,
content: {
"text/csv": {
schema: {
type: "string",
example:
"Analyte;Value;Unit;Date;Range\nHbA1c;5,4;%;01.03.2025;4,0-5,6\nUrine protein;negative;;01.03.2025;",
},
},
},
},
responses: recordImportResponses("LabsCsvImportEnvelope"),
},
},
"/api/import/medications-csv": {
post: {
tags: ["Import"],
summary: "Import medications from CSV",
description: `${RECORD_IMPORT_COMMON} Maps name, dose (as written), daily intake times, an as-needed mark and a start and end date; every other rhythm is set up afterwards in the app. A row needs times unless it is marked as needed. Each row is checked against the create route's own schema; a dated window opens a course. Reminders stay off unless \`reminders=1\`. A medication the record holds under the same name and dose is a duplicate. At most 500 data rows.`,
requestParams: { query: medicationsCsvQuerySchema },
requestBody: {
required: true,
content: {
"text/csv": {
schema: {
type: "string",
example:
"name,dose,times,as needed,start,end\nRamipril,5 mg,08:00,,2025-01-01,\nIbuprofen,400 mg,,yes,,",
},
},
},
},
responses: recordImportResponses("MedicationsCsvImportEnvelope"),
},
},
"/api/import": {
post: {
tags: ["Import"],
src/lib/validations/record-import.ts
+241
View File
@@ -0,0 +1,241 @@
/**
* Query and response schemas of the CSV record imports for lab results and
* medications (#1215).
*
* The body of both routes is the raw CSV file (`text/csv`), streamed. How to
* read it travels in the query: the column each field comes from (0-based
* index into the file's columns), the separator, the decimal mark, the date
* order and the zone a time without an offset is read in. The client works
* these out with the person on the mapping screen; the server never guesses
* a column.
*/
import { z } from "zod/v4";
import { MAX_COLUMNS } from "@/lib/import/csv-stream";
import { isValidTimezone } from "@/lib/tz/format";
const column = z.coerce
.number()
.int()
.min(0)
.max(MAX_COLUMNS - 1)
.describe("0-based index of the file column this field is read from.");
const formatFields = {
dryRun: z
.enum(["1", "0", "true", "false"])
.default("0")
.transform((v) => v === "1" || v === "true")
.describe(
"`1` validates every row and reports what would happen, writing nothing.",
),
header: z
.enum(["1", "0"])
.default("1")
.transform((v) => v === "1")
.describe("`1` when the first row holds column names and is not data."),
delimiter: z
.enum(["auto", "comma", "semicolon", "tab"])
.default("auto")
.describe("Column separator; `auto` detects it from the first line."),
decimal: z
.enum(["dot", "comma"])
.default("dot")
.describe(
"Decimal mark of the numbers. A number that reads differently in the two conventions (`1,234` with `dot`) is refused per row as ambiguous.",
),
dateOrder: z
.enum(["iso", "dmy", "mdy"])
.default("iso")
.describe(
"`iso` = YYYY-MM-DD[THH:mm[:ss][offset]], `dmy` = DD.MM.YYYY (also / or -), `mdy` = MM/DD/YYYY. Two-digit years are refused.",
),
timezone: z
.string()
.max(64)
.refine(isValidTimezone, { message: "Unknown time zone" })
.optional()
.describe(
"IANA zone a date-time without an offset is read in. Defaults to the account's zone. A date without a time is stored as that calendar date.",
),
};
function distinctColumns(
value: Record<string, unknown>,
keys: readonly string[],
): boolean {
const used = keys
.map((k) => value[k])
.filter((v): v is number => typeof v === "number");
return new Set(used).size === used.length;
}
const LAB_COLUMN_KEYS = [
"analyte",
"value",
"unit",
"date",
"time",
"referenceLow",
"referenceHigh",
"referenceRange",
"panel",
"note",
] as const;
export const labsCsvQuerySchema = z
.object({
...formatFields,
analyte: column.describe("Column of the marker name (required)."),
value: column.describe(
"Column of the result (required). A number is a numeric result; any other text (`negative`, `<0.5`) is kept as a qualitative result.",
),
unit: column
.optional()
.describe("Column of the unit. Required for a numeric result."),
date: column.describe("Column of the date, or date and time (required)."),
time: column.optional().describe("Column of the time, when separate."),
referenceLow: column.optional(),
referenceHigh: column.optional(),
referenceRange: column
.optional()
.describe("Column of a printed range such as `4.0-5.6` or `< 200`."),
panel: column.optional(),
note: column.optional().describe("Column of a note; stored encrypted."),
})
.refine((v) => distinctColumns(v, LAB_COLUMN_KEYS), {
message: "Each column can feed one field",
path: ["analyte"],
});
export type LabsCsvQuery = z.infer<typeof labsCsvQuerySchema>;
const MEDICATION_COLUMN_KEYS = [
"name",
"dose",
"times",
"asNeeded",
"startsOn",
"endsOn",
] as const;
export const medicationsCsvQuerySchema = z
.object({
...formatFields,
name: column.describe("Column of the medication name (required)."),
dose: column.describe(
"Column of the dose as written, e.g. `10 mg` (required).",
),
times: column
.optional()
.describe(
"Column of daily intake times (`08:00`, or several: `08:00 20:00`, separated by space, comma, semicolon or `|`). A medication with times is taken every day at them; any other rhythm is set up afterwards in the app. A row needs times unless it is marked as needed.",
),
asNeeded: column
.optional()
.describe("Column marking an as-needed medication (yes/no)."),
startsOn: column.optional().describe("Column of the start date."),
endsOn: column.optional().describe("Column of the end date."),
reminders: z
.enum(["1", "0"])
.default("0")
.transform((v) => v === "1")
.describe(
"`1` turns reminders on for imported medications that have times. Off by default, so an import never starts reminding on its own.",
),
})
.refine((v) => distinctColumns(v, MEDICATION_COLUMN_KEYS), {
message: "Each column can feed one field",
path: ["name"],
});
export type MedicationsCsvQuery = z.infer<typeof medicationsCsvQuerySchema>;
export const RECORD_IMPORT_ROW_REASONS = [
"missing_value",
"invalid_number",
"ambiguous_number",
"missing_unit",
"invalid_date",
"invalid_time",
"ambiguous_date",
"future_date",
"text_too_long",
"unit_mismatch",
"invalid_reference_range",
"invalid_times",
"missing_times",
"invalid_yes_no",
"as_needed_with_times",
"end_before_start",
"invalid_medication",
"oversized_row",
"row_limit",
"duplicate",
] as const;
export type RecordImportRowReason = (typeof RECORD_IMPORT_ROW_REASONS)[number];
export const recordImportResultSchema = z
.object({
kind: z.enum(["labs", "medications"]),
dryRun: z.boolean(),
rows: z.number().int().describe("Data rows read (the header excluded)."),
valid: z.number().int().describe("Rows that passed every check."),
inserted: z
.number()
.int()
.describe("Rows written; in a dry run, the rows that would be."),
duplicates: z
.number()
.int()
.describe(
"Rows skipped because the record already holds them, or an earlier row of the file does. A re-upload of the same file therefore writes nothing.",
),
failed: z
.number()
.int()
.describe("Rows refused; each is listed in `errors`."),
errors: z
.array(
z.object({
line: z.number().int().describe("1-based line of the file."),
reason: z.enum(RECORD_IMPORT_ROW_REASONS),
field: z.string().optional().describe("The mapped field at fault."),
}),
)
.describe("Refused and duplicate rows, in file order, at most 500."),
errorsTruncated: z.boolean(),
markers: z
.object({
matched: z
.array(
z.object({
name: z.string().describe("The name as the file writes it."),
marker: z.string().describe("The marker it resolves to."),
how: z.enum(["name", "alias", "catalog"]),
}),
)
.describe(
"File names that resolve to an existing marker, at most 200.",
),
created: z
.array(z.string())
.describe("Names that become new markers, at most 200."),
})
.optional()
.describe("Lab imports only: how the file's names meet the markers."),
preview: z
.array(
z.record(
z.string(),
z.union([z.string(), z.number(), z.boolean(), z.null()]),
),
)
.describe(
"The first 20 valid rows as they will be stored (dry run only).",
),
})
.meta({ id: "RecordImportResult" });
export type RecordImportResult = z.infer<typeof recordImportResultSchema>;
src/proxy.ts
+1 -1
View File
@@ -865,6 +865,6 @@ export const config = {
// the matcher and `apiHandler` performs the proxy's duties for it. The
// alternation must stay equal to that list; `proxy-body-limit.test.ts`
// compiles this exactly as Next does and checks.
"/((?!_next/static|_next/image|favicon.ico|sw\\.js|manifest\\.json|robots\\.txt|sitemap\\.xml|api/(?:import(?:/csv|/apple-health-export|/health-connect-export)?|admin/import-apple-health-export|admin/backups/upload|documents/inbound|medications/intake/(?:dose-history-import|bulk)|labs/ocr/extract|medications/[^/]+/ingredients/extract|user/avatar|measurements/batch|workouts/batch|insights/ecg|mood-entries/bulk|cycle/day-logs/bulk)/?$|.*\\.(?:svg|png|jpg|jpeg|gif|webp|txt|xml)$).*)",
"/((?!_next/static|_next/image|favicon.ico|sw\\.js|manifest\\.json|robots\\.txt|sitemap\\.xml|api/(?:import(?:/csv|/labs-csv|/medications-csv|/apple-health-export|/health-connect-export)?|admin/import-apple-health-export|admin/backups/upload|documents/inbound|medications/intake/(?:dose-history-import|bulk)|labs/ocr/extract|medications/[^/]+/ingredients/extract|user/avatar|measurements/batch|workouts/batch|insights/ecg|mood-entries/bulk|cycle/day-logs/bulk)/?$|.*\\.(?:svg|png|jpg|jpeg|gif|webp|txt|xml)$).*)",
],
};
tests/integration/record-csv-import.test.ts
+251
View File
@@ -0,0 +1,251 @@
/**
* The lab and medication CSV imports against a real Postgres (#1215).
*
* - a dry run writes nothing (no reading, no marker, no medication) and
* reports what the real run then does;
* - names resolve to existing markers, new analytes mint one marker each,
* a reading in another unit is converted and keeps what the file said;
* - re-uploading the same file writes nothing: every row is a duplicate;
* - a refused row is reported by line and does not stop the rest;
* - medications take a daily schedule, reminders off, as-needed without
* schedule, and a dated window opens a course.
*/
import { beforeAll, describe, expect, it, vi } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import { textToStream } from "@/lib/import/csv-stream";
import { importLabsCsv } from "@/lib/import/labs-csv";
import { importMedicationsCsv } from "@/lib/import/medications-csv";
import {
labsCsvQuerySchema,
medicationsCsvQuerySchema,
} from "@/lib/validations/record-import";
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/jobs/boss-instance", () => ({
getGlobalBoss: vi.fn(() => null),
}));
const prisma = getPrismaClient();
const USER = "csv-import-user";
const TZ = "Europe/Berlin";
const LABS = [
"Analyt;Wert;Einheit;Datum;Referenz",
"Glucose;5,5;mmol/L;01.03.2025;3,9 - 5,5",
"Glukose;99;mg/dL;02.03.2025;",
"HbA1c;5,4;%;01.03.2025;4,0-5,6",
"Urinstatus Eiweiß;negativ;;01.03.2025;",
"Ferritin;1.234;ng/mL;01.03.2025;",
"Ferritin;85;;01.03.2025;",
"Neuer Marker;12;U/l;03.03.2025;",
].join("\n");
function labsQuery(dryRun: boolean) {
return labsCsvQuerySchema.parse({
analyte: "0",
value: "1",
unit: "2",
date: "3",
referenceRange: "4",
decimal: "comma",
dateOrder: "dmy",
dryRun: dryRun ? "1" : "0",
});
}
beforeAll(async () => {
await truncateAllTables(prisma);
await prisma.user.create({
data: { id: USER, username: USER, timezone: TZ },
});
await prisma.biomarker.create({
// No analyte key stored: the import derives it the way the write does.
data: { userId: USER, name: "Glucose", unit: "mg/dL" },
});
await prisma.biomarkerAlias.create({
data: {
userId: USER,
biomarkerId: (
await prisma.biomarker.findFirstOrThrow({ where: { userId: USER } })
).id,
name: "Glukose",
nameFolded: "glukose",
source: "MANUAL",
},
});
});
describe("lab CSV import", () => {
it("a dry run writes nothing and reports the plan", async () => {
const result = await importLabsCsv({
userId: USER,
tz: TZ,
stream: textToStream(LABS, 16),
maxBytes: 1_000_000,
query: labsQuery(true),
});
expect(result).toMatchObject({
rows: 7,
valid: 5,
inserted: 5,
failed: 2,
duplicates: 0,
});
expect(result.errors).toEqual([
{ line: 6, reason: "ambiguous_number", field: "value" },
{ line: 7, reason: "missing_unit", field: "unit" },
]);
expect(result.markers?.created.sort()).toEqual([
"HbA1c",
"Neuer Marker",
"Urinstatus Eiweiß",
]);
expect(result.markers?.matched.map((m) => m.how).sort()).toEqual([
"alias",
"name",
]);
expect(await prisma.labResult.count({ where: { userId: USER } })).toBe(0);
expect(await prisma.biomarker.count({ where: { userId: USER } })).toBe(1);
});
it("the import converts, mints and keeps what the file said", async () => {
const result = await importLabsCsv({
userId: USER,
tz: TZ,
stream: textToStream(LABS, 16),
maxBytes: 1_000_000,
query: labsQuery(false),
});
expect(result).toMatchObject({ inserted: 5, failed: 2 });
const glucose = await prisma.labResult.findMany({
where: { userId: USER, analyte: "Glucose" },
orderBy: { takenAt: "asc" },
});
expect(glucose).toHaveLength(2);
expect(glucose[0].unit).toBe("mg/dL");
expect(glucose[0].value).toBeCloseTo(99.1, 0);
expect(glucose[0].sourceValue).toBe(5.5);
expect(glucose[0].sourceUnit).toBe("mmol/L");
expect(glucose[0].source).toBe("IMPORT");
expect(glucose[1].value).toBe(99);
const text = await prisma.labResult.findFirstOrThrow({
where: { userId: USER, valueText: "negativ" },
});
expect(text.value).toBeNull();
expect(await prisma.biomarker.count({ where: { userId: USER } })).toBe(4);
});
it("re-uploading the same file writes nothing", async () => {
const before = await prisma.labResult.count({ where: { userId: USER } });
const result = await importLabsCsv({
userId: USER,
tz: TZ,
stream: textToStream(LABS, 7),
maxBytes: 1_000_000,
query: labsQuery(false),
});
expect(result).toMatchObject({ inserted: 0, duplicates: 5, failed: 2 });
expect(await prisma.labResult.count({ where: { userId: USER } })).toBe(
before,
);
});
it("refuses a unit that does not convert into the marker's", async () => {
const result = await importLabsCsv({
userId: USER,
tz: TZ,
stream: textToStream(
"Analyt;Wert;Einheit;Datum\nGlucose;7;mmHg;05.03.2025\n",
),
maxBytes: 1_000_000,
query: labsCsvQuerySchema.parse({
analyte: "0",
value: "1",
unit: "2",
date: "3",
decimal: "comma",
dateOrder: "dmy",
}),
});
expect(result.errors).toEqual([
{ line: 2, reason: "unit_mismatch", field: "unit" },
]);
});
});
describe("medication CSV import", () => {
const MEDS = [
"name,dose,times,as needed,start,end",
"Ramipril,5 mg,08:00,,2025-01-01,",
"Metformin,500 mg,08:00 20:00,,,",
"Ibuprofen,400 mg,,yes,,",
"Vitamin D,1000 IE,,,,",
"ramipril,5 MG,08:00,,,",
].join("\n");
const query = (dryRun: boolean) =>
medicationsCsvQuerySchema.parse({
name: "0",
dose: "1",
times: "2",
asNeeded: "3",
startsOn: "4",
endsOn: "5",
dryRun: dryRun ? "1" : "0",
});
it("checks, then imports, then skips everything on a second upload", async () => {
const dry = await importMedicationsCsv({
userId: USER,
tz: TZ,
stream: textToStream(MEDS),
maxBytes: 100_000,
query: query(true),
});
expect(dry).toMatchObject({
rows: 5,
inserted: 3,
duplicates: 1,
failed: 1,
});
expect(dry.errors).toEqual([
{ line: 5, reason: "missing_times", field: "times" },
{ line: 6, reason: "duplicate" },
]);
expect(await prisma.medication.count({ where: { userId: USER } })).toBe(0);
const real = await importMedicationsCsv({
userId: USER,
tz: TZ,
stream: textToStream(MEDS),
maxBytes: 100_000,
query: query(false),
});
expect(real).toMatchObject({ inserted: 3, duplicates: 1, failed: 1 });
const meds = await prisma.medication.findMany({
where: { userId: USER },
include: { schedules: true, courses: true },
orderBy: { name: "asc" },
});
const byName = new Map(meds.map((m) => [m.name, m]));
expect(byName.get("Metformin")?.schedules[0]).toMatchObject({
rrule: "FREQ=DAILY",
timesOfDay: ["08:00", "20:00"],
});
expect(byName.get("Metformin")?.notificationsEnabled).toBe(false);
expect(byName.get("Ibuprofen")).toMatchObject({ asNeeded: true });
expect(byName.get("Ibuprofen")?.schedules).toHaveLength(0);
expect(byName.get("Ramipril")?.courses).toHaveLength(1);
const again = await importMedicationsCsv({
userId: USER,
tz: TZ,
stream: textToStream(MEDS),
maxBytes: 100_000,
query: query(false),
});
expect(again).toMatchObject({ inserted: 0, duplicates: 4 });
});
});