feat(import): add CSV measurement import with per-row status and dry-run

A self-hoster migrating from a spreadsheet or a meter export overwhelmingly
has CSV, which neither the JSON nor the Apple Health import covers. Add
`POST /api/import/csv` reusing the existing write loop and rollup re-fold.

The documented header is `type,value,unit,measuredAt[,glucoseContext,notes,
externalId]`, order-independent. measuredAt must carry an explicit ISO-8601
offset — a row without one is skipped rather than guessed — and is bounded by
the shared entry-instant validator. Glucose accepts mmol/L (converted to
canonical mg/dL) and weight accepts lb (converted to kg); any other
non-canonical unit is skipped, never mis-stored. An optional externalId
column makes a re-upload idempotent; without it a re-upload duplicates.

The response carries a per-row status envelope so a 2000-row file with three
bad rows imports 1997 and names the three. `?dryRun=1` validates and previews
the outcome without writing. The parse, validation, and unit conversion live
in a pure module so they are unit-tested without a database.

Settings -> Export & Import gains a third CSV card mirroring the JSON and
Apple Health cards, with upload, paste, preview, and a downloadable example.
The new route is documented in the OpenAPI contract.
This commit is contained in:
Marc-André Bombeck committed 2026-06-14 23:28:22 +02:00
1 parent 82e2d7dfb1
commit fbdcb31202
15 files changed
+1666 -2

No files matched your search

+140
View File
@@ -5202,6 +5202,54 @@ paths:
"401": *a1
"422": *a3
"429": *a4
/api/import/csv:
post:
tags:
- Import
summary: Import measurements from CSV
description: "Bulk-import measurements from a CSV file (web cold-start escape hatch). Body is raw `text/csv` (≤ 16 MB, ≤
10 000 valid rows). Header (order-independent): `type,value,unit,measuredAt[,glucoseContext,notes,externalId]`.
`measuredAt` must carry an explicit ISO-8601 offset and is bounded (no future beyond a 5-min skew, no instant
before 1900). Glucose accepts `mmol/L` (converted to canonical `mg/dL`); weight accepts `lb` (converted to
`kg`). An `externalId` column makes re-upload idempotent (upsert on `(userId, type, source=IMPORT,
externalId)`); without it a re-upload duplicates. `?dryRun=1` validates + previews without writing. Shares the
5/hour `import:` rate bucket."
parameters:
- name: dryRun
in: query
required: false
schema:
type: string
enum:
- "1"
- "true"
description: When `1` / `true`, parse + validate + return the per-row envelope without writing.
requestBody:
required: true
content:
text/csv:
schema:
type: string
example: |-
type,value,unit,measuredAt,glucoseContext,notes,externalId
WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,
BLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:05:00+02:00,FASTING,,meter-001
responses:
"200":
description: Per-row import outcome.
content:
application/json:
schema:
$ref: "#/components/schemas/CsvImportResponseEnvelope"
"401": *a1
"413":
description: CSV exceeds the 16 MB limit.
content:
application/json:
schema:
$ref: "#/components/schemas/CsvImportTooLargeEnvelope"
"422": *a3
"429": *a4
components:
schemas:
PasskeyLoginVerifyRequest:
@@ -15570,6 +15618,98 @@ components:
- kind
additionalProperties: false
description: Outcome of the idempotent web `ai_full` grant. Safe to call on every AI-settings mount.
CsvImportResponseEnvelope:
type: object
properties:
data:
$ref: "#/components/schemas/CsvImportResponse"
error:
type: "null"
meta:
type: object
properties:
requestId:
type: string
additionalProperties: false
required:
- data
- error
additionalProperties: false
CsvImportResponse:
type: object
properties:
inserted:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
updated:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
skipped:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
total:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
description: Total data rows parsed (excludes header).
dryRun:
type: boolean
rows:
type: array
items:
$ref: "#/components/schemas/CsvImportRowResult"
required:
- inserted
- updated
- skipped
- total
- dryRun
- rows
additionalProperties: false
description: Per-row import outcome. Mirrors the batch-route per-entry envelope. In `dryRun` mode no write runs and
every valid row reports its projected `inserted` status.
CsvImportRowResult:
type: object
properties:
line:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
description: 1-based source line number (the header is line 1).
status:
type: string
enum:
- inserted
- updated
- skipped
reason:
description: Machine-readable reason for a non-inserted row (e.g. unknown_type, unknown_unit, missing_timezone_offset,
value_out_of_range, implausible_timestamp, duplicate).
type: string
required:
- line
- status
additionalProperties: false
CsvImportTooLargeEnvelope:
type: object
properties:
data:
type: "null"
error:
type: "null"
meta:
type: object
properties:
requestId:
type: string
additionalProperties: false
required:
- data
- error
additionalProperties: false
MedicationScheduleOutput:
type: object
properties:
+22
View File
@@ -482,6 +482,7 @@
"pulseLabel": "Puls (bpm)",
"valueWithUnit": "Wert ({unit})",
"timestamp": "Zeitpunkt",
"timestampBackdateHint": "Datum und Uhrzeit anpassen, um einen vergangenen Wert zu erfassen.",
"notesPlaceholder": "z.B. nach dem Essen",
"formReset": "Formular zurücksetzen",
"saveError": "Fehler beim Speichern",
@@ -3958,6 +3959,27 @@
"failed": "Der Import ist fehlgeschlagen. Bitte erneut versuchen.",
"readFailed": "Diese Datei konnte nicht gelesen werden.",
"resultSummary": "{measurements} Messwerte und {moods} Stimmungseinträge importiert, {skipped} übersprungen."
},
"csv": {
"title": "CSV-Datei",
"description": "Messwerte aus einer Tabelle oder einem Messgerät-Export importieren. Eine Zeile pro Wert.",
"pasteLabel": "CSV einfügen",
"fileInputLabel": "CSV-Datei zum Importieren auswählen",
"schemaHint": "Spalten: type, value, unit, measuredAt (mit Zeitzonen-Offset) sowie optional glucoseContext, notes, externalId.",
"docsLink": "Spaltenschema und Beispiel ansehen.",
"uploadFile": "Datei hochladen",
"downloadExample": "Beispiel herunterladen",
"preview": "Vorschau",
"import": "Importieren",
"empty": "Bitte vor dem Import mindestens eine CSV-Zeile hinzufügen.",
"rateLimited": "Zu viele Importe in dieser Stunde (Limit 5). Bitte später erneut versuchen.",
"tooLarge": "Diese Datei ist zu groß (Limit 16 MB).",
"invalidPayload": "Die CSV konnte nicht gelesen werden. Bitte die Kopfzeile prüfen und erneut versuchen.",
"failed": "Der Import ist fehlgeschlagen. Bitte erneut versuchen.",
"readFailed": "Diese Datei konnte nicht gelesen werden.",
"previewSummary": "Vorschau: {inserted} würden importiert, {skipped} übersprungen.",
"resultSummary": "{inserted} neu importiert und {updated} aktualisiert, {skipped} übersprungen.",
"rowError": "Zeile {line}: {reason}"
}
}
},
+23 -1
View File
@@ -482,6 +482,7 @@
"pulseLabel": "Pulse (bpm)",
"valueWithUnit": "Value ({unit})",
"timestamp": "Time",
"timestampBackdateHint": "Adjust the date and time to log a past reading.",
"notesPlaceholder": "e.g. after a meal",
"formReset": "Reset form",
"saveError": "Error saving",
@@ -3919,7 +3920,7 @@
},
"import": {
"heading": "Import",
"description": "Bring data into HealthLog from an Apple Health export or a JSON file. Imports skip rows that already exist, so re-running one is safe.",
"description": "Bring data into HealthLog from an Apple Health export, a JSON file, or a CSV. Imports skip rows that already exist, so re-running one is safe.",
"appleHealth": {
"title": "Apple Health",
"description": "Upload the export.zip from the Health app on iPhone (Profile → Export All Health Data).",
@@ -3958,6 +3959,27 @@
"failed": "The import failed. Please try again.",
"readFailed": "Couldn't read that file.",
"resultSummary": "Imported {measurements} measurements and {moods} mood entries, {skipped} skipped."
},
"csv": {
"title": "CSV file",
"description": "Import measurements from a spreadsheet or meter export. One row per reading.",
"pasteLabel": "Paste CSV",
"fileInputLabel": "Choose a CSV file to import",
"schemaHint": "Columns: type, value, unit, measuredAt (with timezone offset), and optional glucoseContext, notes, externalId.",
"docsLink": "See the column schema and an example.",
"uploadFile": "Upload file",
"downloadExample": "Download example",
"preview": "Preview",
"import": "Import",
"empty": "Add at least one CSV row before importing.",
"rateLimited": "Too many imports this hour (limit 5). Please try again later.",
"tooLarge": "That file is too large (limit 16 MB).",
"invalidPayload": "The CSV could not be parsed. Check the header row and try again.",
"failed": "The import failed. Please try again.",
"readFailed": "Couldn't read that file.",
"previewSummary": "Preview: {inserted} would import, {skipped} skipped.",
"resultSummary": "Imported {inserted} new and updated {updated}, {skipped} skipped.",
"rowError": "Line {line}: {reason}"
}
}
},
+22
View File
@@ -482,6 +482,7 @@
"pulseLabel": "Pulso (lpm)",
"valueWithUnit": "Valor ({unit})",
"timestamp": "Hora",
"timestampBackdateHint": "Ajusta la fecha y la hora para registrar una lectura pasada.",
"notesPlaceholder": "p. ej. después de comer",
"formReset": "Restablecer formulario",
"saveError": "Error al guardar",
@@ -3958,6 +3959,27 @@
"failed": "La importación falló. Inténtalo de nuevo.",
"readFailed": "No se pudo leer ese archivo.",
"resultSummary": "{measurements} mediciones y {moods} entradas de ánimo importadas, {skipped} omitidas."
},
"csv": {
"title": "Archivo CSV",
"description": "Importa mediciones desde una hoja de cálculo o la exportación de un medidor. Una fila por lectura.",
"pasteLabel": "Pegar CSV",
"fileInputLabel": "Elegir un archivo CSV para importar",
"schemaHint": "Columnas: type, value, unit, measuredAt (con desfase de zona horaria) y opcionales glucoseContext, notes, externalId.",
"docsLink": "Ver el esquema de columnas y un ejemplo.",
"uploadFile": "Subir archivo",
"downloadExample": "Descargar ejemplo",
"preview": "Vista previa",
"import": "Importar",
"empty": "Añade al menos una fila CSV antes de importar.",
"rateLimited": "Demasiadas importaciones esta hora (límite 5). Inténtalo más tarde.",
"tooLarge": "Ese archivo es demasiado grande (límite 16 MB).",
"invalidPayload": "No se pudo analizar el CSV. Revisa la fila de encabezado e inténtalo de nuevo.",
"failed": "La importación falló. Inténtalo de nuevo.",
"readFailed": "No se pudo leer ese archivo.",
"previewSummary": "Vista previa: {inserted} se importarían, {skipped} omitidas.",
"resultSummary": "{inserted} nuevas importadas y {updated} actualizadas, {skipped} omitidas.",
"rowError": "Línea {line}: {reason}"
}
}
},
+22
View File
@@ -482,6 +482,7 @@
"pulseLabel": "Pouls (bpm)",
"valueWithUnit": "Valeur ({unit})",
"timestamp": "Heure",
"timestampBackdateHint": "Ajustez la date et l'heure pour enregistrer un relevé passé.",
"notesPlaceholder": "p. ex. après le repas",
"formReset": "Réinitialiser le formulaire",
"saveError": "Erreur lors de l’enregistrement",
@@ -3958,6 +3959,27 @@
"failed": "L'import a échoué. Veuillez réessayer.",
"readFailed": "Impossible de lire ce fichier.",
"resultSummary": "{measurements} mesures et {moods} entrées d'humeur importées, {skipped} ignorées."
},
"csv": {
"title": "Fichier CSV",
"description": "Importez des mesures depuis un tableur ou l'export d'un appareil. Une ligne par relevé.",
"pasteLabel": "Coller le CSV",
"fileInputLabel": "Choisir un fichier CSV à importer",
"schemaHint": "Colonnes : type, value, unit, measuredAt (avec décalage horaire) et, en option, glucoseContext, notes, externalId.",
"docsLink": "Voir le schéma des colonnes et un exemple.",
"uploadFile": "Téléverser un fichier",
"downloadExample": "Télécharger un exemple",
"preview": "Aperçu",
"import": "Importer",
"empty": "Ajoutez au moins une ligne CSV avant d'importer.",
"rateLimited": "Trop d'imports cette heure (limite 5). Veuillez réessayer plus tard.",
"tooLarge": "Ce fichier est trop volumineux (limite 16 Mo).",
"invalidPayload": "Le CSV n'a pas pu être analysé. Vérifiez la ligne d'en-tête et réessayez.",
"failed": "L'import a échoué. Veuillez réessayer.",
"readFailed": "Impossible de lire ce fichier.",
"previewSummary": "Aperçu : {inserted} seraient importées, {skipped} ignorées.",
"resultSummary": "{inserted} nouvelles importées et {updated} mises à jour, {skipped} ignorées.",
"rowError": "Ligne {line} : {reason}"
}
}
},
+22
View File
@@ -482,6 +482,7 @@
"pulseLabel": "Pulsazioni (bpm)",
"valueWithUnit": "Valore ({unit})",
"timestamp": "Ora",
"timestampBackdateHint": "Regola data e ora per registrare una lettura passata.",
"notesPlaceholder": "es. dopo il pasto",
"formReset": "Reimposta modulo",
"saveError": "Errore durante il salvataggio",
@@ -3958,6 +3959,27 @@
"failed": "L'importazione non è riuscita. Riprova.",
"readFailed": "Impossibile leggere il file.",
"resultSummary": "{measurements} misurazioni e {moods} voci dell'umore importate, {skipped} saltate."
},
"csv": {
"title": "File CSV",
"description": "Importa misurazioni da un foglio di calcolo o dall'esportazione di un dispositivo. Una riga per lettura.",
"pasteLabel": "Incolla CSV",
"fileInputLabel": "Scegli un file CSV da importare",
"schemaHint": "Colonne: type, value, unit, measuredAt (con offset del fuso orario) e, facoltative, glucoseContext, notes, externalId.",
"docsLink": "Vedi lo schema delle colonne e un esempio.",
"uploadFile": "Carica file",
"downloadExample": "Scarica esempio",
"preview": "Anteprima",
"import": "Importa",
"empty": "Aggiungi almeno una riga CSV prima di importare.",
"rateLimited": "Troppe importazioni in questa ora (limite 5). Riprova più tardi.",
"tooLarge": "Questo file è troppo grande (limite 16 MB).",
"invalidPayload": "Impossibile analizzare il CSV. Controlla la riga di intestazione e riprova.",
"failed": "L'importazione non è riuscita. Riprova.",
"readFailed": "Impossibile leggere questo file.",
"previewSummary": "Anteprima: {inserted} verrebbero importate, {skipped} saltate.",
"resultSummary": "{inserted} nuove importate e {updated} aggiornate, {skipped} saltate.",
"rowError": "Riga {line}: {reason}"
}
}
},
+22
View File
@@ -482,6 +482,7 @@
"pulseLabel": "Tętno (uderz./min)",
"valueWithUnit": "Wartość ({unit})",
"timestamp": "Czas",
"timestampBackdateHint": "Dostosuj datę i godzinę, aby zapisać odczyt z przeszłości.",
"notesPlaceholder": "np. po jedzeniu",
"formReset": "Resetuj formularz",
"saveError": "Błąd zapisu",
@@ -3958,6 +3959,27 @@
"failed": "Import nie powiódł się. Spróbuj ponownie.",
"readFailed": "Nie można odczytać tego pliku.",
"resultSummary": "Zaimportowano {measurements} pomiarów i {moods} wpisów nastroju, pominięto {skipped}."
},
"csv": {
"title": "Plik CSV",
"description": "Importuj pomiary z arkusza lub eksportu z urządzenia. Jeden wiersz na odczyt.",
"pasteLabel": "Wklej CSV",
"fileInputLabel": "Wybierz plik CSV do zaimportowania",
"schemaHint": "Kolumny: type, value, unit, measuredAt (z przesunięciem strefy czasowej) oraz opcjonalnie glucoseContext, notes, externalId.",
"docsLink": "Zobacz schemat kolumn i przykład.",
"uploadFile": "Prześlij plik",
"downloadExample": "Pobierz przykład",
"preview": "Podgląd",
"import": "Importuj",
"empty": "Dodaj co najmniej jeden wiersz CSV przed importem.",
"rateLimited": "Zbyt wiele importów w tej godzinie (limit 5). Spróbuj ponownie później.",
"tooLarge": "Ten plik jest za duży (limit 16 MB).",
"invalidPayload": "Nie udało się przetworzyć pliku CSV. Sprawdź wiersz nagłówka i spróbuj ponownie.",
"failed": "Import nie powiódł się. Spróbuj ponownie.",
"readFailed": "Nie udało się odczytać tego pliku.",
"previewSummary": "Podgląd: {inserted} zostanie zaimportowanych, pominięto {skipped}.",
"resultSummary": "Zaimportowano {inserted} nowych i zaktualizowano {updated}, pominięto {skipped}.",
"rowError": "Wiersz {line}: {reason}"
}
}
},
@@ -0,0 +1,189 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
vi.mock("@/lib/api-handler", () => ({
apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
requireAuth: vi.fn(async () => ({
user: { id: "u-1" },
session: { id: "s-1" },
})),
}));
vi.mock("@/lib/db", () => ({
prisma: {
measurement: { create: vi.fn(), upsert: vi.fn() },
},
}));
vi.mock("@/lib/rate-limit", () => ({
checkRateLimit: vi.fn(),
}));
vi.mock("@/lib/auth/audit", () => ({
auditLog: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/logging/context", () => ({
annotate: vi.fn(),
}));
vi.mock("@/lib/rollups/measurement-rollups", async () => {
const actual = await vi.importActual<
typeof import("@/lib/rollups/measurement-rollups")
>("@/lib/rollups/measurement-rollups");
return {
...actual,
recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined),
};
});
import { NextRequest } from "next/server";
import { POST } from "../route";
import { checkRateLimit } from "@/lib/rate-limit";
import { prisma } from "@/lib/db";
import { recomputeBucketsForMeasurement } from "@/lib/rollups/measurement-rollups";
const HEADER = "type,value,unit,measuredAt,glucoseContext,notes,externalId";
function csvRequest(csv: string, dryRun = false) {
return new NextRequest(
`http://localhost/api/import/csv${dryRun ? "?dryRun=1" : ""}`,
{
method: "POST",
body: csv,
headers: { "content-type": "text/csv" },
},
);
}
interface CsvEnvelope {
data: {
inserted: number;
updated: number;
skipped: number;
total: number;
dryRun: boolean;
rows: Array<{ line: number; status: string; reason?: string }>;
} | null;
error: string | null;
}
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(checkRateLimit).mockResolvedValue({
allowed: true,
remaining: 4,
resetAt: new Date(Date.now() + 1000),
} as never);
});
describe("POST /api/import/csv — rate limit", () => {
it("returns 429 against the shared import bucket when exhausted", async () => {
vi.mocked(checkRateLimit).mockResolvedValue({
allowed: false,
remaining: 0,
resetAt: new Date(Date.now() + 1000),
} as never);
const res = await POST(csvRequest([HEADER, ""].join("\n")));
expect(res.status).toBe(429);
expect(checkRateLimit).toHaveBeenCalledWith(
expect.stringContaining("import:u-1"),
5,
60 * 60 * 1000,
);
});
});
describe("POST /api/import/csv — fatal header error", () => {
it("returns 422 when a required column is missing", async () => {
const res = await POST(csvRequest("type,value,unit\nWEIGHT,80,kg"));
expect(res.status).toBe(422);
expect(vi.mocked(prisma.measurement.create)).not.toHaveBeenCalled();
});
});
describe("POST /api/import/csv — write loop + per-row envelope", () => {
it("inserts valid rows, skips invalid ones, returns per-row status", async () => {
vi.mocked(prisma.measurement.create).mockResolvedValue({} as never);
const res = await POST(
csvRequest(
[
HEADER,
"WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,", // inserted
"NOPE,1,kg,2026-05-01T08:00:00Z,,,", // skipped unknown_type
"WEIGHT,80,kg,2026-05-01T08:00:00,,,", // skipped missing offset
].join("\n"),
),
);
expect(res.status).toBe(200);
const body = (await res.json()) as CsvEnvelope;
expect(body.data?.inserted).toBe(1);
expect(body.data?.skipped).toBe(2);
expect(body.data?.total).toBe(3);
expect(body.data?.dryRun).toBe(false);
const reasons = body.data?.rows.map((r) => r.reason);
expect(reasons).toContain("unknown_type");
expect(reasons).toContain("missing_timezone_offset");
// One create call for the single valid row.
expect(vi.mocked(prisma.measurement.create)).toHaveBeenCalledTimes(1);
// Rollup re-fold fired for the touched (type, day).
expect(vi.mocked(recomputeBucketsForMeasurement)).toHaveBeenCalled();
});
it("dryRun previews without writing and reports projected inserts", async () => {
const res = await POST(
csvRequest(
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,"].join("\n"),
true,
),
);
expect(res.status).toBe(200);
const body = (await res.json()) as CsvEnvelope;
expect(body.data?.dryRun).toBe(true);
expect(body.data?.inserted).toBe(1);
expect(body.data?.rows[0].status).toBe("inserted");
expect(vi.mocked(prisma.measurement.create)).not.toHaveBeenCalled();
expect(vi.mocked(recomputeBucketsForMeasurement)).not.toHaveBeenCalled();
});
it("upserts an externalId row and surfaces it as updated when it already existed", async () => {
const created = new Date("2026-05-01T08:00:00Z");
const bumped = new Date("2026-05-02T08:00:00Z");
vi.mocked(prisma.measurement.upsert).mockResolvedValue({
createdAt: created,
updatedAt: bumped,
} as never);
const res = await POST(
csvRequest(
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,ext-1"].join("\n"),
),
);
const body = (await res.json()) as CsvEnvelope;
expect(body.data?.updated).toBe(1);
expect(body.data?.inserted).toBe(0);
expect(body.data?.rows[0].status).toBe("updated");
expect(vi.mocked(prisma.measurement.upsert)).toHaveBeenCalledTimes(1);
});
it("counts a unique-constraint duplicate as skipped/duplicate", async () => {
vi.mocked(prisma.measurement.create).mockRejectedValue(
new Error("Unique constraint failed"),
);
const res = await POST(
csvRequest(
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,"].join("\n"),
),
);
const body = (await res.json()) as CsvEnvelope;
expect(body.data?.inserted).toBe(0);
expect(body.data?.skipped).toBe(1);
expect(body.data?.rows[0]).toMatchObject({
status: "skipped",
reason: "duplicate",
});
});
});
+254
View File
@@ -0,0 +1,254 @@
import { prisma } from "@/lib/db";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { annotate } from "@/lib/logging/context";
import { auditLog } from "@/lib/auth/audit";
import { apiSuccess, apiError, getClientIp } from "@/lib/api-response";
import { checkRateLimit } from "@/lib/rate-limit";
import { NextRequest } from "next/server";
import {
parseCsvMeasurements,
type CsvRowResult,
} from "@/lib/import/csv-measurements";
import {
collapseToTypeDayKeys,
recomputeBucketsForMeasurement,
} from "@/lib/rollups/measurement-rollups";
import type {
MeasurementType,
GlucoseContext,
} from "@/generated/prisma/client";
/**
* CSV measurement import (v1.17.1).
*
* The cold-start escape hatch for self-hosters migrating from a spreadsheet
* or another tracker. Reuses the `/api/import` write loop + rollup re-fold;
* the parse + per-row validation + unit conversion lives in the pure
* `parseCsvMeasurements` module so it is unit-tested without a database.
*
* Body is `text/csv` (or `text/plain`) raw text — NOT JSON. Capped at 16 MB
* (the same ceiling as the JSON import; a 16 MB CSV is > 100 000 rows). Rows
* beyond the 10 000 ceiling are rejected so the rollup re-fold stays bounded.
*
* `?dryRun=1` parses + validates + returns the per-row status envelope
* WITHOUT writing — a trust win for a bulk cold-start import.
*
* Response: `{ inserted, updated, skipped, total, dryRun, rows: [{line,
* status, reason}] }`, mirroring the batch route's per-entry envelope.
*/
const MAX_CSV_BYTES = 16 * 1024 * 1024;
const MAX_ROWS = 10000;
export const POST = apiHandler(async (request: NextRequest) => {
const { user } = await requireAuth();
annotate({ action: { name: "import.csv.upload" } });
const dryRun =
new URL(request.url).searchParams.get("dryRun") === "1" ||
new URL(request.url).searchParams.get("dryRun") === "true";
// Same blast-radius reasoning as the JSON import — share the bucket so a
// CSV upload counts against the same 5/hour quota.
const rl = await checkRateLimit(`import:${user.id}`, 5, 60 * 60 * 1000);
if (!rl.allowed) {
return apiError("Maximum 5 imports per hour", 429);
}
// Raw text read with an explicit byte cap — `safeJson` is JSON-only, so we
// enforce the same `maxBytes` ceiling here against the Content-Length and
// the materialised body.
const declaredLength = Number(request.headers.get("content-length") ?? "0");
if (Number.isFinite(declaredLength) && declaredLength > MAX_CSV_BYTES) {
return apiError("CSV exceeds the 16 MB limit", 413);
}
let text: string;
try {
text = await request.text();
} catch {
return apiError("Could not read the request body", 400);
}
if (Buffer.byteLength(text, "utf8") > MAX_CSV_BYTES) {
return apiError("CSV exceeds the 16 MB limit", 413);
}
const parsed = parseCsvMeasurements(text);
if (parsed.fatal) {
return apiError(parsed.fatal.message, 422);
}
const okRows = parsed.rows.filter(
(r): r is CsvRowResult & { row: NonNullable<CsvRowResult["row"]> } =>
r.status === "ok" && r.row !== undefined,
);
if (okRows.length > MAX_ROWS) {
return apiError(
`CSV has ${okRows.length} valid rows — the limit is ${MAX_ROWS} per import`,
422,
);
}
const userId = user.id;
let inserted = 0;
let updated = 0;
let skipped = parsed.rows.filter((r) => r.status === "skipped").length;
const touchedMeasurements: Array<{
type: MeasurementType;
measuredAt: Date;
}> = [];
// Per-line write outcome for rows that reached the DB. Keyed by source line
// so the response envelope can surface `inserted` / `updated` / `duplicate`
// distinctly without polluting the parse-time `CsvRowReason`.
const writeOutcome = new Map<number, "inserted" | "updated" | "duplicate">();
if (!dryRun) {
for (const result of okRows) {
const m = result.row;
try {
if (m.externalId) {
// Idempotent re-import keyed on the source-stable id — re-uploading
// the same file updates in place rather than minting duplicates.
const res = await prisma.measurement.upsert({
where: {
userId_type_source_externalId: {
userId,
type: m.type as MeasurementType,
source: "IMPORT",
externalId: m.externalId,
},
},
update: {
value: m.value,
unit: m.unit,
measuredAt: m.measuredAt,
notes: m.notes ?? null,
glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null,
},
create: {
userId,
type: m.type as MeasurementType,
value: m.value,
unit: m.unit,
source: "IMPORT",
externalId: m.externalId,
measuredAt: m.measuredAt,
notes: m.notes ?? null,
glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null,
},
select: { createdAt: true, updatedAt: true },
});
// A fresh create has createdAt === updatedAt; an in-place update
// bumps updatedAt past createdAt.
if (res.updatedAt.getTime() > res.createdAt.getTime()) {
updated++;
writeOutcome.set(result.line, "updated");
} else {
inserted++;
writeOutcome.set(result.line, "inserted");
}
} else {
await prisma.measurement.create({
data: {
userId,
type: m.type as MeasurementType,
value: m.value,
unit: m.unit,
source: "IMPORT",
measuredAt: m.measuredAt,
notes: m.notes ?? null,
glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null,
},
});
inserted++;
writeOutcome.set(result.line, "inserted");
}
touchedMeasurements.push({
type: m.type as MeasurementType,
measuredAt: m.measuredAt,
});
} catch {
// Unique-constraint violation (a NULL-externalId duplicate against an
// existing row) — count as skipped rather than failing the file.
skipped++;
writeOutcome.set(result.line, "duplicate");
}
}
// One bounded rollup re-fold per touched (type, day) — a 10 000-row CSV
// pays at most ~N (type, day) recomputes, not 10 000 per-row hooks.
// Best-effort: a populator hiccup never fails the importer.
if (touchedMeasurements.length > 0) {
try {
const keys = collapseToTypeDayKeys(touchedMeasurements);
for (const k of keys) {
await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt);
}
} catch (err) {
annotate({
meta: {
measurement_rollup_csv_import_failed: true,
measurement_rollup_csv_import_error:
err instanceof Error ? err.message : String(err),
},
});
}
}
await auditLog("import.csv.upload", {
userId,
ipAddress: getClientIp(request),
details: { inserted, updated, skipped },
});
} else {
// Dry-run preview: no write ran, so report the projected outcome —
// every valid parse-`ok` row would insert (an externalId row could land
// as an update, but that needs a DB read we deliberately skip here).
inserted = okRows.length;
}
const summary = {
inserted,
updated,
skipped,
total: parsed.rows.length,
dryRun,
// Per-row status, mirroring the batch route's per-entry envelope. The
// normalised payload is stripped — the client only needs line + status.
// Parse-time `ok` rows are refined by the write outcome: a row that hit a
// unique-constraint duplicate becomes `skipped`/`duplicate`; an upsert
// that landed on an existing row becomes `updated`. In `dryRun` mode no
// write ran, so every parse-`ok` row stays `inserted` (the projected
// outcome).
rows: parsed.rows.map((r) => {
if (r.status === "skipped") {
return {
line: r.line,
status: "skipped" as const,
...(r.reason ? { reason: r.reason } : {}),
};
}
const outcome = writeOutcome.get(r.line);
if (outcome === "duplicate") {
return { line: r.line, status: "skipped" as const, reason: "duplicate" };
}
if (outcome === "updated") {
return { line: r.line, status: "updated" as const };
}
return { line: r.line, status: "inserted" as const };
}),
};
annotate({
meta: {
import_csv_inserted: inserted,
import_csv_updated: updated,
import_csv_skipped: skipped,
import_csv_dry_run: dryRun,
},
});
return apiSuccess(summary);
});
@@ -27,7 +27,13 @@ vi.mock("next/navigation", () => ({
}));
import { I18nProvider } from "@/lib/i18n/context";
import { ImportPanel, EXAMPLE_IMPORT, parseImportJson } from "../import-panel";
import {
ImportPanel,
EXAMPLE_IMPORT,
EXAMPLE_CSV,
parseImportJson,
} from "../import-panel";
import { parseCsvMeasurements } from "@/lib/import/csv-measurements";
function render(node: React.ReactElement, locale: "en" | "de" = "en") {
const queryClient = new QueryClient({
@@ -46,6 +52,7 @@ describe("<ImportPanel> — SSR smoke", () => {
expect(html).toContain('id="settings-section-import-title"');
expect(html).toContain('data-testid="import-card-apple-health"');
expect(html).toContain('data-testid="import-card-json"');
expect(html).toContain('data-testid="import-card-csv"');
// Raw i18n keys never leak past the provider.
expect(html).not.toContain("settings.sections.export.import.");
});
@@ -58,6 +65,11 @@ describe("<ImportPanel> — SSR smoke", () => {
"import-json-choose-file",
"import-json-download-example",
"import-action-json",
"import-csv-textarea",
"import-csv-choose-file",
"import-csv-download-example",
"import-csv-preview",
"import-action-csv",
]) {
expect(html).toContain(`data-testid="${id}"`);
}
@@ -112,6 +124,19 @@ describe("EXAMPLE_IMPORT payload", () => {
});
});
describe("EXAMPLE_CSV", () => {
it("parses entirely to ok rows (example must not drift from the schema)", () => {
// Pin the clock well past the fixture timestamps so the entry-instant
// bound never rejects the example as future-dated.
const out = parseCsvMeasurements(EXAMPLE_CSV, {
now: new Date("2026-06-01T00:00:00Z").getTime(),
});
expect(out.fatal).toBeUndefined();
expect(out.rows.length).toBeGreaterThan(0);
expect(out.rows.every((r) => r.status === "ok")).toBe(true);
});
});
describe("parseImportJson guard", () => {
it("rejects an unparseable string before any network call", () => {
const result = parseImportJson("{ not json");
+253
View File
@@ -32,6 +32,7 @@ import {
CheckCircle2,
Download,
FileJson,
FileSpreadsheet,
Loader2,
Upload,
} from "lucide-react";
@@ -68,6 +69,7 @@ export function ImportPanel() {
<div className="grid gap-4 md:grid-cols-2">
<AppleHealthImportCard />
<JsonImportCard />
<CsvImportCard />
</div>
</section>
);
@@ -630,3 +632,254 @@ function JsonImportCard() {
</ImportCardShell>
);
}
// ─────────────────────────── CSV import ───────────────────────────
/**
* Downloadable CSV example. Header is order-independent server-side, but the
* example pins the documented order so the docs + the route schema never
* drift. Exported so a test can assert it stays a valid header.
*/
export const EXAMPLE_CSV = [
"type,value,unit,measuredAt,glucoseContext,notes,externalId",
"WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,",
"BLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:05:00+02:00,FASTING,,meter-001",
"BLOOD_PRESSURE_SYS,120,mmHg,2026-05-01T08:05:00+02:00,,,",
].join("\n");
interface CsvRowResult {
line: number;
status: "inserted" | "updated" | "skipped";
reason?: string;
}
interface CsvImportResult {
inserted: number;
updated: number;
skipped: number;
total: number;
dryRun: boolean;
rows: CsvRowResult[];
}
function CsvImportCard() {
const { t } = useTranslations();
const fileInputRef = useRef<HTMLInputElement>(null);
const [text, setText] = useState("");
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const [result, setResult] = useState<CsvImportResult | null>(null);
const textareaId = useId();
async function onFileChange(e: React.ChangeEvent<HTMLInputElement>) {
const file = e.target.files?.[0];
e.target.value = "";
if (!file) return;
setError(null);
setResult(null);
try {
const content = await file.text();
setText(content);
} catch {
setError(t("settings.sections.export.import.csv.readFailed"));
}
}
function downloadExample() {
const blob = new Blob([EXAMPLE_CSV], { type: "text/csv" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = "healthlog-import-example.csv";
document.body.appendChild(a);
a.click();
a.remove();
URL.revokeObjectURL(url);
}
async function send(dryRun: boolean) {
setError(null);
setResult(null);
if (text.trim().length === 0) {
setError(t("settings.sections.export.import.csv.empty"));
return;
}
setBusy(true);
try {
const res = await apiFetchRaw(`/api/import/csv${dryRun ? "?dryRun=1" : ""}`, {
method: "POST",
headers: { "Content-Type": "text/csv" },
credentials: "include",
body: text,
});
if (res.status === 429) {
setError(t("settings.sections.export.import.csv.rateLimited"));
return;
}
if (res.status === 413) {
setError(t("settings.sections.export.import.csv.tooLarge"));
return;
}
if (res.status === 422) {
const body = await res.json().catch(() => null);
setError(
body?.error ??
t("settings.sections.export.import.csv.invalidPayload"),
);
return;
}
if (!res.ok) {
setError(t("settings.sections.export.import.csv.failed"));
return;
}
const data = (await res.json()).data as CsvImportResult;
setResult(data);
} catch {
setError(t("settings.sections.export.import.csv.failed"));
} finally {
setBusy(false);
}
}
const errorRows = result?.rows.filter((r) => r.status === "skipped") ?? [];
return (
<ImportCardShell
testId="import-card-csv"
icon={FileSpreadsheet}
title={t("settings.sections.export.import.csv.title")}
description={t("settings.sections.export.import.csv.description")}
>
<div className="space-y-1.5">
<Label htmlFor={textareaId} className="text-xs">
{t("settings.sections.export.import.csv.pasteLabel")}
</Label>
<Textarea
id={textareaId}
data-testid="import-csv-textarea"
value={text}
onChange={(e) => setText(e.target.value)}
rows={5}
spellCheck={false}
placeholder="type,value,unit,measuredAt,…"
className="font-mono text-xs"
/>
</div>
<input
ref={fileInputRef}
type="file"
accept=".csv,text/csv"
className="sr-only"
aria-label={t("settings.sections.export.import.csv.fileInputLabel")}
onChange={onFileChange}
/>
<p className="text-muted-foreground text-xs">
{t("settings.sections.export.import.csv.schemaHint")}{" "}
<a
href="/docs/integrations/data-import"
className="text-primary underline underline-offset-2"
>
{t("settings.sections.export.import.csv.docsLink")}
</a>
</p>
<div aria-live="polite" className="space-y-2">
{result && (
<div data-testid="import-csv-result" className="space-y-1.5">
<p className="text-foreground flex items-start gap-2 text-xs">
<CheckCircle2
className="mt-0.5 h-3.5 w-3.5 shrink-0 text-emerald-500"
aria-hidden="true"
/>
<span>
{result.dryRun
? t("settings.sections.export.import.csv.previewSummary", {
inserted: result.inserted,
skipped: result.skipped,
})
: t("settings.sections.export.import.csv.resultSummary", {
inserted: result.inserted,
updated: result.updated,
skipped: result.skipped,
})}
</span>
</p>
{errorRows.length > 0 && (
<ul className="text-muted-foreground max-h-32 space-y-0.5 overflow-auto text-xs">
{errorRows.slice(0, 50).map((r) => (
<li key={r.line}>
{t("settings.sections.export.import.csv.rowError", {
line: r.line,
reason: r.reason ?? "skipped",
})}
</li>
))}
</ul>
)}
</div>
)}
{error && (
<p
role="alert"
className="text-destructive flex items-start gap-2 text-sm"
>
<AlertCircle
className="mt-0.5 h-3.5 w-3.5 shrink-0"
aria-hidden="true"
/>
<span>{error}</span>
</p>
)}
</div>
<div className="mt-auto flex flex-wrap items-center gap-3 pt-1">
<Button
type="button"
variant="outline"
size="sm"
onClick={() => fileInputRef.current?.click()}
data-testid="import-csv-choose-file"
>
<Upload className="h-3.5 w-3.5" />
{t("settings.sections.export.import.csv.uploadFile")}
</Button>
<Button
type="button"
variant="ghost"
size="sm"
onClick={downloadExample}
data-testid="import-csv-download-example"
>
<Download className="h-3.5 w-3.5" />
{t("settings.sections.export.import.csv.downloadExample")}
</Button>
<Button
type="button"
variant="secondary"
size="sm"
disabled={busy || text.trim().length === 0}
onClick={() => void send(true)}
data-testid="import-csv-preview"
>
{t("settings.sections.export.import.csv.preview")}
</Button>
<Button
type="button"
size="sm"
disabled={busy || text.trim().length === 0}
onClick={() => void send(false)}
data-testid="import-action-csv"
>
{busy ? (
<Loader2 className="h-3.5 w-3.5 animate-spin motion-reduce:animate-none" />
) : (
<FileSpreadsheet className="h-3.5 w-3.5" />
)}
{t("settings.sections.export.import.csv.import")}
</Button>
</div>
</ImportCardShell>
);
}
@@ -0,0 +1,204 @@
import { describe, it, expect } from "vitest";
import {
parseCsvMeasurements,
splitCsvRows,
} from "@/lib/import/csv-measurements";
// Pin the clock so the entry-instant bound is deterministic. All fixture
// timestamps sit comfortably in the past relative to this.
const NOW = new Date("2026-06-01T00:00:00.000Z").getTime();
const HEADER = "type,value,unit,measuredAt,glucoseContext,notes,externalId";
function parse(rows: string[]) {
return parseCsvMeasurements([HEADER, ...rows].join("\n"), { now: NOW });
}
describe("splitCsvRows", () => {
it("handles CRLF, quoted commas, escaped quotes, and a BOM", () => {
const csv =
"a,b,c\r\n1,\"x,y\",\"he said \"\"hi\"\"\"\r\n2,plain,z\r\n";
const grid = splitCsvRows(csv);
expect(grid).toEqual([
["a", "b", "c"],
["1", "x,y", 'he said "hi"'],
["2", "plain", "z"],
]);
});
it("drops trailing blank lines", () => {
expect(splitCsvRows("a,b\n1,2\n\n\n")).toEqual([
["a", "b"],
["1", "2"],
]);
});
});
describe("parseCsvMeasurements — header validation", () => {
it("is fatal when a required column is missing", () => {
const out = parseCsvMeasurements("type,value,unit\nWEIGHT,80,kg", {
now: NOW,
});
expect(out.fatal?.reason).toBe("missing_required_column");
expect(out.rows).toEqual([]);
});
it("is fatal on an empty file", () => {
const out = parseCsvMeasurements("", { now: NOW });
expect(out.fatal?.reason).toBe("missing_required_column");
});
it("accepts columns in any order", () => {
const out = parseCsvMeasurements(
["unit,measuredAt,type,value", "kg,2026-05-01T08:00:00Z,WEIGHT,80.5"].join(
"\n",
),
{ now: NOW },
);
expect(out.fatal).toBeUndefined();
expect(out.rows[0].status).toBe("ok");
expect(out.rows[0].row?.value).toBe(80.5);
});
});
describe("parseCsvMeasurements — per-row partial failure", () => {
it("imports the good rows and skips the bad ones with distinct reasons", () => {
const out = parse([
"WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,", // ok
"NOPE,1,kg,2026-05-01T08:00:00Z,,,", // unknown_type
"WEIGHT,9999,kg,2026-05-01T08:00:00Z,,,", // value_out_of_range
"WEIGHT,80,kg,2026-05-01T08:00:00,,,", // missing_timezone_offset
]);
const byLine = Object.fromEntries(
out.rows.map((r) => [r.line, { status: r.status, reason: r.reason }]),
);
expect(byLine[2]).toEqual({ status: "ok", reason: undefined });
expect(byLine[3]).toEqual({ status: "skipped", reason: "unknown_type" });
expect(byLine[4]).toEqual({
status: "skipped",
reason: "value_out_of_range",
});
expect(byLine[5]).toEqual({
status: "skipped",
reason: "missing_timezone_offset",
});
});
it("reports line numbers 1-based with the header as line 1", () => {
const out = parse(["WEIGHT,80,kg,2026-05-01T08:00:00Z,,,"]);
expect(out.rows[0].line).toBe(2);
});
});
describe("parseCsvMeasurements — unit conversion", () => {
it("converts glucose mmol/L to canonical mg/dL", () => {
const out = parse([
"BLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:00:00Z,FASTING,,",
]);
expect(out.rows[0].status).toBe("ok");
expect(out.rows[0].row?.unit).toBe("mg/dL");
expect(out.rows[0].row?.value).toBeCloseTo(5.3 * 18.016, 3);
});
it("converts weight lb to canonical kg", () => {
const out = parse(["WEIGHT,180,lb,2026-05-01T08:00:00Z,,,"]);
expect(out.rows[0].status).toBe("ok");
expect(out.rows[0].row?.unit).toBe("kg");
expect(out.rows[0].row?.value).toBeCloseTo(180 * 0.453592, 3);
});
it("accepts the canonical unit case-insensitively", () => {
const out = parse(["WEIGHT,80,KG,2026-05-01T08:00:00Z,,,"]);
expect(out.rows[0].status).toBe("ok");
expect(out.rows[0].row?.unit).toBe("kg");
});
it("skips an unrecognised unit rather than mis-storing", () => {
const out = parse(["WEIGHT,80,stone,2026-05-01T08:00:00Z,,,"]);
expect(out.rows[0]).toMatchObject({
status: "skipped",
reason: "unknown_unit",
});
});
});
describe("parseCsvMeasurements — timezone + entry-instant bound", () => {
it("skips a timestamp without an offset (no silent local interpretation)", () => {
const out = parse(["WEIGHT,80,kg,2026-05-01T08:00:00,,,"]);
expect(out.rows[0].reason).toBe("missing_timezone_offset");
});
it("accepts both Z and ±HH:MM offsets", () => {
const out = parse([
"WEIGHT,80,kg,2026-05-01T08:00:00Z,,,",
"WEIGHT,81,kg,2026-05-01T08:00:00+02:00,,,",
]);
expect(out.rows.map((r) => r.status)).toEqual(["ok", "ok"]);
});
it("rejects a future-dated row via the entry-instant bound", () => {
// 10 minutes past NOW — beyond the 5-min skew tolerance.
const future = new Date(NOW + 10 * 60 * 1000).toISOString();
const out = parse([`WEIGHT,80,kg,${future},,,`]);
expect(out.rows[0]).toMatchObject({
status: "skipped",
reason: "implausible_timestamp",
});
});
it("rejects a pre-1900 row", () => {
const out = parse(["WEIGHT,80,kg,1899-12-31T00:00:00Z,,,"]);
expect(out.rows[0]).toMatchObject({
status: "skipped",
reason: "implausible_timestamp",
});
});
});
describe("parseCsvMeasurements — glucose context", () => {
it("requires a context for BLOOD_GLUCOSE", () => {
const out = parse(["BLOOD_GLUCOSE,95,mg/dL,2026-05-01T08:00:00Z,,,"]);
expect(out.rows[0]).toMatchObject({
status: "skipped",
reason: "missing_glucose_context",
});
});
it("rejects a context on a non-glucose row", () => {
const out = parse(["WEIGHT,80,kg,2026-05-01T08:00:00Z,FASTING,,"]);
expect(out.rows[0]).toMatchObject({
status: "skipped",
reason: "unexpected_glucose_context",
});
});
it("rejects an unknown context value", () => {
const out = parse(["BLOOD_GLUCOSE,95,mg/dL,2026-05-01T08:00:00Z,LUNCH,,"]);
expect(out.rows[0]).toMatchObject({
status: "skipped",
reason: "invalid_glucose_context",
});
});
});
describe("parseCsvMeasurements — optional columns", () => {
it("carries externalId through and trims notes", () => {
const out = parse(["WEIGHT,80,kg,2026-05-01T08:00:00Z,, after run ,ext-9"]);
expect(out.rows[0].row).toMatchObject({
externalId: "ext-9",
notes: "after run",
});
});
it("works without the optional columns present at all", () => {
const out = parseCsvMeasurements(
["type,value,unit,measuredAt", "WEIGHT,80,kg,2026-05-01T08:00:00Z"].join(
"\n",
),
{ now: NOW },
);
expect(out.rows[0].status).toBe("ok");
expect(out.rows[0].row?.externalId).toBeUndefined();
});
});
+368
View File
@@ -0,0 +1,368 @@
/**
* CSV measurement import — parse + normalise + per-row validation (v1.17.1).
*
* A cold-start escape hatch for self-hosters migrating from a spreadsheet,
* a glucose-meter export, or another tracker. The genuine import gap left
* by the JSON (`/api/import`) and Apple-Health (`/api/import/apple-health-export`)
* routes is plain CSV, which neither covers.
*
* This module is the pure, side-effect-free core: it turns raw CSV text
* into a per-row result set carrying either a normalised measurement (ready
* for the `/api/import` write loop) or a `skipped` status with a stable
* machine-readable `reason`. The route (`/api/import/csv`) owns the DB write
* + rollup re-fold; everything testable without a database lives here.
*
* Documented column schema (header row required, order-independent):
*
* type,value,unit,measuredAt[,glucoseContext][,notes][,externalId]
*
* - `type` — a `measurementTypeEnum` value (e.g. WEIGHT). Unknown → skipped.
* - `value` — a number in the row's `unit`.
* - `unit` — the canonical unit for the type, OR a recognised alias:
* glucose mmol/L → mg/dL (× 18.016),
* weight lb → kg (× 0.453592).
* Any other non-canonical unit → skipped (never silently
* mis-stored).
* - `measuredAt` — ISO-8601 WITH an explicit offset (`Z` or ±HH:MM). A row
* without an offset is skipped — the importer never guesses
* a timezone. The instant is bounded by `validateEntryInstant`
* (no future beyond a 5-min skew, no instant before 1900).
* - `glucoseContext` — required for BLOOD_GLUCOSE, forbidden otherwise.
* - `notes` — free text, ≤ 200 chars.
* - `externalId` — optional source-stable id. When present, the write loop
* upserts on `(userId, type, source=IMPORT, externalId)` so a
* re-upload is idempotent. Absent → re-upload duplicates.
*/
import {
measurementTypeEnum,
glucoseContextEnum,
getUnitForType,
validateMeasurementRange,
MEASUREMENT_NOTES_MAX_LENGTH,
} from "@/lib/validations/measurement";
import { isPlausibleEntryInstant } from "@/lib/validations/entry-instant";
/** Canonical conversion factors keyed by `${type}:${loweredAlias}`. */
const GLUCOSE_MMOL_TO_MGDL = 18.016;
const LB_TO_KG = 0.453592;
/** A normalised, write-ready row in the `/api/import` measurement shape. */
export interface NormalisedMeasurementRow {
type: string;
value: number;
unit: string;
measuredAt: Date;
glucoseContext?: string;
notes?: string;
externalId?: string;
}
/** Stable machine-readable skip reasons, mirroring the batch route's vocabulary. */
export type CsvRowReason =
| "missing_required_column"
| "unknown_type"
| "invalid_value"
| "value_out_of_range"
| "unknown_unit"
| "missing_timezone_offset"
| "invalid_timestamp"
| "implausible_timestamp"
| "missing_glucose_context"
| "unexpected_glucose_context"
| "invalid_glucose_context"
| "notes_too_long"
| "external_id_too_long"
| "wrong_column_count";
export type CsvRowStatus = "ok" | "skipped";
export interface CsvRowResult {
/** 1-based source line number (header is line 1, first data row is line 2). */
line: number;
status: CsvRowStatus;
reason?: CsvRowReason;
/** Present only when `status === "ok"`. */
row?: NormalisedMeasurementRow;
}
export interface CsvParseOutcome {
/** A fatal, file-level error (e.g. missing required header). When set, `rows` is empty. */
fatal?: { reason: CsvRowReason; message: string };
rows: CsvRowResult[];
}
const REQUIRED_COLUMNS = ["type", "value", "unit", "measuredat"] as const;
const OPTIONAL_COLUMNS = ["glucosecontext", "notes", "externalid"] as const;
/**
* Split CSV text into rows of cells. Hand-rolled, zero-dependency, RFC-4180-ish:
* handles quoted fields, escaped quotes (`""`), commas + newlines inside quotes,
* a leading UTF-8 BOM, and CRLF / LF line endings. Trailing blank lines are
* dropped. Sufficient for the flat measurement schema; not a general CSV engine.
*/
export function splitCsvRows(text: string): string[][] {
// Strip a leading BOM so the first header cell isn't "type".
const input = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
const rows: string[][] = [];
let cell = "";
let row: string[] = [];
let inQuotes = false;
let sawAnyChar = false;
for (let i = 0; i < input.length; i++) {
const ch = input[i];
if (inQuotes) {
if (ch === '"') {
if (input[i + 1] === '"') {
cell += '"';
i++;
} else {
inQuotes = false;
}
} else {
cell += ch;
}
continue;
}
if (ch === '"') {
inQuotes = true;
sawAnyChar = true;
continue;
}
if (ch === ",") {
row.push(cell);
cell = "";
sawAnyChar = true;
continue;
}
if (ch === "\r") {
// Swallow; the \n (if any) finalises the row.
continue;
}
if (ch === "\n") {
row.push(cell);
rows.push(row);
row = [];
cell = "";
sawAnyChar = false;
continue;
}
cell += ch;
sawAnyChar = true;
}
// Flush a trailing row that did not end in a newline.
if (sawAnyChar || cell.length > 0 || row.length > 0) {
row.push(cell);
rows.push(row);
}
// Drop fully-empty rows (e.g. a trailing blank line that produced [""]).
return rows.filter((r) => !(r.length === 1 && r[0].trim() === ""));
}
/** True when an ISO-8601 datetime string carries an explicit offset (`Z` or ±HH:MM). */
function hasExplicitOffset(value: string): boolean {
// Offset is in the time portion, after the `T`. Match a trailing Z or ±HH:MM.
return /([zZ]|[+-]\d{2}:?\d{2})$/.test(value.trim());
}
/**
* Resolve the row's unit to the canonical unit + converted value, or return
* `null` when the unit is neither canonical nor a recognised alias.
*/
function convertToCanonicalUnit(
type: string,
value: number,
rawUnit: string,
): { value: number; unit: string } | null {
const canonical = getUnitForType(type);
const unit = rawUnit.trim();
// Case-insensitive match against the canonical unit (mg/dL vs MG/DL).
if (unit.toLowerCase() === canonical.toLowerCase()) {
return { value, unit: canonical };
}
const lower = unit.toLowerCase();
// Glucose: mmol/L → mg/dL.
if (type === "BLOOD_GLUCOSE" && (lower === "mmol/l" || lower === "mmol")) {
return { value: value * GLUCOSE_MMOL_TO_MGDL, unit: canonical };
}
// Weight: lb → kg.
if (
type === "WEIGHT" &&
(lower === "lb" || lower === "lbs" || lower === "pound" || lower === "pounds")
) {
return { value: value * LB_TO_KG, unit: canonical };
}
return null;
}
const KNOWN_TYPES = new Set(measurementTypeEnum.options as readonly string[]);
const KNOWN_GLUCOSE_CONTEXTS = new Set(
glucoseContextEnum.options as readonly string[],
);
/**
* Parse + validate + normalise a CSV measurement file. Returns a per-row
* result set: each data row is either `ok` with a normalised measurement or
* `skipped` with a stable reason. A missing required header column is the
* only fatal, file-level failure.
*
* `now` is injectable so tests pin the entry-instant clock.
*/
export function parseCsvMeasurements(
text: string,
opts: { now?: number } = {},
): CsvParseOutcome {
const now = opts.now ?? Date.now();
const grid = splitCsvRows(text);
if (grid.length === 0) {
return {
fatal: {
reason: "missing_required_column",
message: "The CSV is empty — a header row is required.",
},
rows: [],
};
}
const header = grid[0].map((h) => h.trim().toLowerCase());
const colIndex = new Map<string, number>();
header.forEach((name, idx) => {
if (!colIndex.has(name)) colIndex.set(name, idx);
});
const missing = REQUIRED_COLUMNS.filter((c) => !colIndex.has(c));
if (missing.length > 0) {
return {
fatal: {
reason: "missing_required_column",
message: `Missing required column(s): ${missing.join(", ")}. Required header: ${REQUIRED_COLUMNS.join(",")}[,${OPTIONAL_COLUMNS.join(",")}].`,
},
rows: [],
};
}
const idxType = colIndex.get("type")!;
const idxValue = colIndex.get("value")!;
const idxUnit = colIndex.get("unit")!;
const idxMeasuredAt = colIndex.get("measuredat")!;
const idxContext = colIndex.get("glucosecontext");
const idxNotes = colIndex.get("notes");
const idxExternalId = colIndex.get("externalid");
const rows: CsvRowResult[] = [];
for (let r = 1; r < grid.length; r++) {
const cells = grid[r];
const line = r + 1; // 1-based, header is line 1.
const cellAt = (idx: number | undefined): string =>
idx === undefined ? "" : (cells[idx] ?? "").trim();
const type = cellAt(idxType);
const rawValue = cellAt(idxValue);
const rawUnit = cellAt(idxUnit);
const rawMeasuredAt = cellAt(idxMeasuredAt);
const rawContext = cellAt(idxContext);
const rawNotes = idxNotes === undefined ? "" : (cells[idxNotes] ?? "");
const rawExternalId = cellAt(idxExternalId);
const skip = (reason: CsvRowReason): void => {
rows.push({ line, status: "skipped", reason });
};
if (!KNOWN_TYPES.has(type)) {
skip("unknown_type");
continue;
}
const value = Number(rawValue);
if (rawValue === "" || !Number.isFinite(value)) {
skip("invalid_value");
continue;
}
const converted = convertToCanonicalUnit(type, value, rawUnit);
if (converted === null) {
skip("unknown_unit");
continue;
}
if (validateMeasurementRange(type, converted.value) !== null) {
skip("value_out_of_range");
continue;
}
if (!rawMeasuredAt) {
skip("invalid_timestamp");
continue;
}
if (!hasExplicitOffset(rawMeasuredAt)) {
skip("missing_timezone_offset");
continue;
}
const measuredAt = new Date(rawMeasuredAt);
if (Number.isNaN(measuredAt.getTime())) {
skip("invalid_timestamp");
continue;
}
if (!isPlausibleEntryInstant(measuredAt, { now })) {
skip("implausible_timestamp");
continue;
}
// Glucose context: required for BLOOD_GLUCOSE, forbidden otherwise.
let glucoseContext: string | undefined;
if (type === "BLOOD_GLUCOSE") {
if (!rawContext) {
skip("missing_glucose_context");
continue;
}
if (!KNOWN_GLUCOSE_CONTEXTS.has(rawContext)) {
skip("invalid_glucose_context");
continue;
}
glucoseContext = rawContext;
} else if (rawContext) {
skip("unexpected_glucose_context");
continue;
}
const notes = rawNotes.trim();
if (notes.length > MEASUREMENT_NOTES_MAX_LENGTH) {
skip("notes_too_long");
continue;
}
if (rawExternalId.length > 120) {
skip("external_id_too_long");
continue;
}
rows.push({
line,
status: "ok",
row: {
type,
value: converted.value,
unit: converted.unit,
measuredAt,
...(glucoseContext ? { glucoseContext } : {}),
...(notes ? { notes } : {}),
...(rawExternalId ? { externalId: rawExternalId } : {}),
},
});
}
return { rows };
}
/** Column order for the downloadable example + docs. */
export const CSV_EXAMPLE_COLUMNS = [
"type",
"value",
"unit",
"measuredAt",
"glucoseContext",
"notes",
"externalId",
] as const;
+97
View File
@@ -0,0 +1,97 @@
/**
* OpenAPI route table — CSV measurement import (v1.17.1).
*
* The CSV importer is the cold-start escape hatch for self-hosters migrating
* from a spreadsheet or another tracker. The JSON (`/api/import`) and Apple-
* Health (`/api/import/apple-health-export`) routes predate the contract
* surface and are intentionally not documented here; this module documents
* the one new endpoint so the wire contract stays drift-free (CI gate). iOS
* does not call it — its canonical ingest is `POST /api/measurements/batch`.
*/
import type { ZodOpenApiObject } from "zod-openapi";
import { z } from "zod/v4";
import { dataEnvelope, stdResponses } from "./shared";
const csvImportRowResult = z
.object({
line: z
.number()
.int()
.describe("1-based source line number (the header is line 1)."),
status: z.enum(["inserted", "updated", "skipped"]),
reason: z
.string()
.optional()
.describe(
"Machine-readable reason for a non-inserted row (e.g. unknown_type, unknown_unit, missing_timezone_offset, value_out_of_range, implausible_timestamp, duplicate).",
),
})
.meta({ id: "CsvImportRowResult" });
const csvImportResponse = z
.object({
inserted: z.number().int(),
updated: z.number().int(),
skipped: z.number().int(),
total: z.number().int().describe("Total data rows parsed (excludes header)."),
dryRun: z.boolean(),
rows: z.array(csvImportRowResult),
})
.meta({
id: "CsvImportResponse",
description:
"Per-row import outcome. Mirrors the batch-route per-entry envelope. In `dryRun` mode no write runs and every valid row reports its projected `inserted` status.",
});
export const importPaths: NonNullable<ZodOpenApiObject["paths"]> = {
"/api/import/csv": {
post: {
tags: ["Import"],
summary: "Import measurements from CSV",
description:
"Bulk-import measurements from a CSV file (web cold-start escape hatch). Body is raw `text/csv` (≤ 16 MB, ≤ 10 000 valid rows). Header (order-independent): `type,value,unit,measuredAt[,glucoseContext,notes,externalId]`. `measuredAt` must carry an explicit ISO-8601 offset and is bounded (no future beyond a 5-min skew, no instant before 1900). Glucose accepts `mmol/L` (converted to canonical `mg/dL`); weight accepts `lb` (converted to `kg`). An `externalId` column makes re-upload idempotent (upsert on `(userId, type, source=IMPORT, externalId)`); without it a re-upload duplicates. `?dryRun=1` validates + previews without writing. Shares the 5/hour `import:` rate bucket.",
parameters: [
{
name: "dryRun",
in: "query",
required: false,
schema: { type: "string", enum: ["1", "true"] },
description:
"When `1` / `true`, parse + validate + return the per-row envelope without writing.",
},
],
requestBody: {
required: true,
content: {
"text/csv": {
schema: {
type: "string",
example:
"type,value,unit,measuredAt,glucoseContext,notes,externalId\nWEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,\nBLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:05:00+02:00,FASTING,,meter-001",
},
},
},
},
responses: {
"200": {
description: "Per-row import outcome.",
content: {
"application/json": {
schema: dataEnvelope(csvImportResponse, "CsvImportResponseEnvelope"),
},
},
},
...stdResponses,
"413": {
description: "CSV exceeds the 16 MB limit.",
content: {
"application/json": {
schema: dataEnvelope(z.null(), "CsvImportTooLargeEnvelope"),
},
},
},
},
},
},
};
+2
View File
@@ -28,6 +28,7 @@ import { consentPaths } from "./consent";
import { cyclePaths } from "./cycle";
import { devicePaths } from "./devices";
import { healthRecordPaths } from "./health-record";
import { importPaths } from "./import";
import { insightsPaths } from "./insights";
import { measurementPaths } from "./measurements";
import { medicationPaths, medicationResource } from "./medications";
@@ -57,6 +58,7 @@ export const openApiPaths: NonNullable<ZodOpenApiObject["paths"]> = {
...moodPaths,
...settingsPaths,
...consentPaths,
...importPaths,
};
export const openApiComponents: NonNullable<ZodOpenApiObject["components"]> = {