From fbdcb312029dba0cf26c23931abbfaf59cfed258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 14 Jun 2026 23:28:22 +0200 Subject: [PATCH] feat(import): add CSV measurement import with per-row status and dry-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A self-hoster migrating from a spreadsheet or a meter export overwhelmingly has CSV, which neither the JSON nor the Apple Health import covers. Add `POST /api/import/csv` reusing the existing write loop and rollup re-fold. The documented header is `type,value,unit,measuredAt[,glucoseContext,notes, externalId]`, order-independent. measuredAt must carry an explicit ISO-8601 offset — a row without one is skipped rather than guessed — and is bounded by the shared entry-instant validator. Glucose accepts mmol/L (converted to canonical mg/dL) and weight accepts lb (converted to kg); any other non-canonical unit is skipped, never mis-stored. An optional externalId column makes a re-upload idempotent; without it a re-upload duplicates. The response carries a per-row status envelope so a 2000-row file with three bad rows imports 1997 and names the three. `?dryRun=1` validates and previews the outcome without writing. The parse, validation, and unit conversion live in a pure module so they are unit-tested without a database. Settings -> Export & Import gains a third CSV card mirroring the JSON and Apple Health cards, with upload, paste, preview, and a downloadable example. The new route is documented in the OpenAPI contract. --- docs/api/openapi.yaml | 140 +++++++ messages/de.json | 22 ++ messages/en.json | 24 +- messages/es.json | 22 ++ messages/fr.json | 22 ++ messages/it.json | 22 ++ messages/pl.json | 22 ++ .../api/import/csv/__tests__/route.test.ts | 189 +++++++++ src/app/api/import/csv/route.ts | 254 ++++++++++++ .../settings/__tests__/import-panel.test.tsx | 27 +- src/components/settings/import-panel.tsx | 253 ++++++++++++ .../import/__tests__/csv-measurements.test.ts | 204 ++++++++++ src/lib/import/csv-measurements.ts | 368 ++++++++++++++++++ src/lib/openapi/routes/import.ts | 97 +++++ src/lib/openapi/routes/index.ts | 2 + 15 files changed, 1666 insertions(+), 2 deletions(-) create mode 100644 src/app/api/import/csv/__tests__/route.test.ts create mode 100644 src/app/api/import/csv/route.ts create mode 100644 src/lib/import/__tests__/csv-measurements.test.ts create mode 100644 src/lib/import/csv-measurements.ts create mode 100644 src/lib/openapi/routes/import.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index bf628854e..d9a5901ed 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -5202,6 +5202,54 @@ paths: "401": *a1 "422": *a3 "429": *a4 + /api/import/csv: + post: + tags: + - Import + summary: Import measurements from CSV + description: "Bulk-import measurements from a CSV file (web cold-start escape hatch). Body is raw `text/csv` (≤ 16 MB, ≤ + 10 000 valid rows). Header (order-independent): `type,value,unit,measuredAt[,glucoseContext,notes,externalId]`. + `measuredAt` must carry an explicit ISO-8601 offset and is bounded (no future beyond a 5-min skew, no instant + before 1900). Glucose accepts `mmol/L` (converted to canonical `mg/dL`); weight accepts `lb` (converted to + `kg`). An `externalId` column makes re-upload idempotent (upsert on `(userId, type, source=IMPORT, + externalId)`); without it a re-upload duplicates. `?dryRun=1` validates + previews without writing. Shares the + 5/hour `import:` rate bucket." + parameters: + - name: dryRun + in: query + required: false + schema: + type: string + enum: + - "1" + - "true" + description: When `1` / `true`, parse + validate + return the per-row envelope without writing. + requestBody: + required: true + content: + text/csv: + schema: + type: string + example: |- + type,value,unit,measuredAt,glucoseContext,notes,externalId + WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning, + BLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:05:00+02:00,FASTING,,meter-001 + responses: + "200": + description: Per-row import outcome. + content: + application/json: + schema: + $ref: "#/components/schemas/CsvImportResponseEnvelope" + "401": *a1 + "413": + description: CSV exceeds the 16 MB limit. + content: + application/json: + schema: + $ref: "#/components/schemas/CsvImportTooLargeEnvelope" + "422": *a3 + "429": *a4 components: schemas: PasskeyLoginVerifyRequest: @@ -15570,6 +15618,98 @@ components: - kind additionalProperties: false description: Outcome of the idempotent web `ai_full` grant. Safe to call on every AI-settings mount. + CsvImportResponseEnvelope: + type: object + properties: + data: + $ref: "#/components/schemas/CsvImportResponse" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false + CsvImportResponse: + type: object + properties: + inserted: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + updated: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + skipped: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + total: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Total data rows parsed (excludes header). + dryRun: + type: boolean + rows: + type: array + items: + $ref: "#/components/schemas/CsvImportRowResult" + required: + - inserted + - updated + - skipped + - total + - dryRun + - rows + additionalProperties: false + description: Per-row import outcome. Mirrors the batch-route per-entry envelope. In `dryRun` mode no write runs and + every valid row reports its projected `inserted` status. + CsvImportRowResult: + type: object + properties: + line: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: 1-based source line number (the header is line 1). + status: + type: string + enum: + - inserted + - updated + - skipped + reason: + description: Machine-readable reason for a non-inserted row (e.g. unknown_type, unknown_unit, missing_timezone_offset, + value_out_of_range, implausible_timestamp, duplicate). + type: string + required: + - line + - status + additionalProperties: false + CsvImportTooLargeEnvelope: + type: object + properties: + data: + type: "null" + error: + type: "null" + meta: + type: object + properties: + requestId: + type: string + additionalProperties: false + required: + - data + - error + additionalProperties: false MedicationScheduleOutput: type: object properties: diff --git a/messages/de.json b/messages/de.json index 01784ec8b..840194598 100644 --- a/messages/de.json +++ b/messages/de.json @@ -482,6 +482,7 @@ "pulseLabel": "Puls (bpm)", "valueWithUnit": "Wert ({unit})", "timestamp": "Zeitpunkt", + "timestampBackdateHint": "Datum und Uhrzeit anpassen, um einen vergangenen Wert zu erfassen.", "notesPlaceholder": "z.B. nach dem Essen", "formReset": "Formular zurücksetzen", "saveError": "Fehler beim Speichern", @@ -3958,6 +3959,27 @@ "failed": "Der Import ist fehlgeschlagen. Bitte erneut versuchen.", "readFailed": "Diese Datei konnte nicht gelesen werden.", "resultSummary": "{measurements} Messwerte und {moods} Stimmungseinträge importiert, {skipped} übersprungen." + }, + "csv": { + "title": "CSV-Datei", + "description": "Messwerte aus einer Tabelle oder einem Messgerät-Export importieren. Eine Zeile pro Wert.", + "pasteLabel": "CSV einfügen", + "fileInputLabel": "CSV-Datei zum Importieren auswählen", + "schemaHint": "Spalten: type, value, unit, measuredAt (mit Zeitzonen-Offset) sowie optional glucoseContext, notes, externalId.", + "docsLink": "Spaltenschema und Beispiel ansehen.", + "uploadFile": "Datei hochladen", + "downloadExample": "Beispiel herunterladen", + "preview": "Vorschau", + "import": "Importieren", + "empty": "Bitte vor dem Import mindestens eine CSV-Zeile hinzufügen.", + "rateLimited": "Zu viele Importe in dieser Stunde (Limit 5). Bitte später erneut versuchen.", + "tooLarge": "Diese Datei ist zu groß (Limit 16 MB).", + "invalidPayload": "Die CSV konnte nicht gelesen werden. Bitte die Kopfzeile prüfen und erneut versuchen.", + "failed": "Der Import ist fehlgeschlagen. Bitte erneut versuchen.", + "readFailed": "Diese Datei konnte nicht gelesen werden.", + "previewSummary": "Vorschau: {inserted} würden importiert, {skipped} übersprungen.", + "resultSummary": "{inserted} neu importiert und {updated} aktualisiert, {skipped} übersprungen.", + "rowError": "Zeile {line}: {reason}" } } }, diff --git a/messages/en.json b/messages/en.json index a78cc1a24..29087aa94 100644 --- a/messages/en.json +++ b/messages/en.json @@ -482,6 +482,7 @@ "pulseLabel": "Pulse (bpm)", "valueWithUnit": "Value ({unit})", "timestamp": "Time", + "timestampBackdateHint": "Adjust the date and time to log a past reading.", "notesPlaceholder": "e.g. after a meal", "formReset": "Reset form", "saveError": "Error saving", @@ -3919,7 +3920,7 @@ }, "import": { "heading": "Import", - "description": "Bring data into HealthLog from an Apple Health export or a JSON file. Imports skip rows that already exist, so re-running one is safe.", + "description": "Bring data into HealthLog from an Apple Health export, a JSON file, or a CSV. Imports skip rows that already exist, so re-running one is safe.", "appleHealth": { "title": "Apple Health", "description": "Upload the export.zip from the Health app on iPhone (Profile → Export All Health Data).", @@ -3958,6 +3959,27 @@ "failed": "The import failed. Please try again.", "readFailed": "Couldn't read that file.", "resultSummary": "Imported {measurements} measurements and {moods} mood entries, {skipped} skipped." + }, + "csv": { + "title": "CSV file", + "description": "Import measurements from a spreadsheet or meter export. One row per reading.", + "pasteLabel": "Paste CSV", + "fileInputLabel": "Choose a CSV file to import", + "schemaHint": "Columns: type, value, unit, measuredAt (with timezone offset), and optional glucoseContext, notes, externalId.", + "docsLink": "See the column schema and an example.", + "uploadFile": "Upload file", + "downloadExample": "Download example", + "preview": "Preview", + "import": "Import", + "empty": "Add at least one CSV row before importing.", + "rateLimited": "Too many imports this hour (limit 5). Please try again later.", + "tooLarge": "That file is too large (limit 16 MB).", + "invalidPayload": "The CSV could not be parsed. Check the header row and try again.", + "failed": "The import failed. Please try again.", + "readFailed": "Couldn't read that file.", + "previewSummary": "Preview: {inserted} would import, {skipped} skipped.", + "resultSummary": "Imported {inserted} new and updated {updated}, {skipped} skipped.", + "rowError": "Line {line}: {reason}" } } }, diff --git a/messages/es.json b/messages/es.json index 07ef722cb..275305f32 100644 --- a/messages/es.json +++ b/messages/es.json @@ -482,6 +482,7 @@ "pulseLabel": "Pulso (lpm)", "valueWithUnit": "Valor ({unit})", "timestamp": "Hora", + "timestampBackdateHint": "Ajusta la fecha y la hora para registrar una lectura pasada.", "notesPlaceholder": "p. ej. después de comer", "formReset": "Restablecer formulario", "saveError": "Error al guardar", @@ -3958,6 +3959,27 @@ "failed": "La importación falló. Inténtalo de nuevo.", "readFailed": "No se pudo leer ese archivo.", "resultSummary": "{measurements} mediciones y {moods} entradas de ánimo importadas, {skipped} omitidas." + }, + "csv": { + "title": "Archivo CSV", + "description": "Importa mediciones desde una hoja de cálculo o la exportación de un medidor. Una fila por lectura.", + "pasteLabel": "Pegar CSV", + "fileInputLabel": "Elegir un archivo CSV para importar", + "schemaHint": "Columnas: type, value, unit, measuredAt (con desfase de zona horaria) y opcionales glucoseContext, notes, externalId.", + "docsLink": "Ver el esquema de columnas y un ejemplo.", + "uploadFile": "Subir archivo", + "downloadExample": "Descargar ejemplo", + "preview": "Vista previa", + "import": "Importar", + "empty": "Añade al menos una fila CSV antes de importar.", + "rateLimited": "Demasiadas importaciones esta hora (límite 5). Inténtalo más tarde.", + "tooLarge": "Ese archivo es demasiado grande (límite 16 MB).", + "invalidPayload": "No se pudo analizar el CSV. Revisa la fila de encabezado e inténtalo de nuevo.", + "failed": "La importación falló. Inténtalo de nuevo.", + "readFailed": "No se pudo leer ese archivo.", + "previewSummary": "Vista previa: {inserted} se importarían, {skipped} omitidas.", + "resultSummary": "{inserted} nuevas importadas y {updated} actualizadas, {skipped} omitidas.", + "rowError": "Línea {line}: {reason}" } } }, diff --git a/messages/fr.json b/messages/fr.json index acec48c77..4172998ed 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -482,6 +482,7 @@ "pulseLabel": "Pouls (bpm)", "valueWithUnit": "Valeur ({unit})", "timestamp": "Heure", + "timestampBackdateHint": "Ajustez la date et l'heure pour enregistrer un relevé passé.", "notesPlaceholder": "p. ex. après le repas", "formReset": "Réinitialiser le formulaire", "saveError": "Erreur lors de l’enregistrement", @@ -3958,6 +3959,27 @@ "failed": "L'import a échoué. Veuillez réessayer.", "readFailed": "Impossible de lire ce fichier.", "resultSummary": "{measurements} mesures et {moods} entrées d'humeur importées, {skipped} ignorées." + }, + "csv": { + "title": "Fichier CSV", + "description": "Importez des mesures depuis un tableur ou l'export d'un appareil. Une ligne par relevé.", + "pasteLabel": "Coller le CSV", + "fileInputLabel": "Choisir un fichier CSV à importer", + "schemaHint": "Colonnes : type, value, unit, measuredAt (avec décalage horaire) et, en option, glucoseContext, notes, externalId.", + "docsLink": "Voir le schéma des colonnes et un exemple.", + "uploadFile": "Téléverser un fichier", + "downloadExample": "Télécharger un exemple", + "preview": "Aperçu", + "import": "Importer", + "empty": "Ajoutez au moins une ligne CSV avant d'importer.", + "rateLimited": "Trop d'imports cette heure (limite 5). Veuillez réessayer plus tard.", + "tooLarge": "Ce fichier est trop volumineux (limite 16 Mo).", + "invalidPayload": "Le CSV n'a pas pu être analysé. Vérifiez la ligne d'en-tête et réessayez.", + "failed": "L'import a échoué. Veuillez réessayer.", + "readFailed": "Impossible de lire ce fichier.", + "previewSummary": "Aperçu : {inserted} seraient importées, {skipped} ignorées.", + "resultSummary": "{inserted} nouvelles importées et {updated} mises à jour, {skipped} ignorées.", + "rowError": "Ligne {line} : {reason}" } } }, diff --git a/messages/it.json b/messages/it.json index e825a1770..ed63814f3 100644 --- a/messages/it.json +++ b/messages/it.json @@ -482,6 +482,7 @@ "pulseLabel": "Pulsazioni (bpm)", "valueWithUnit": "Valore ({unit})", "timestamp": "Ora", + "timestampBackdateHint": "Regola data e ora per registrare una lettura passata.", "notesPlaceholder": "es. dopo il pasto", "formReset": "Reimposta modulo", "saveError": "Errore durante il salvataggio", @@ -3958,6 +3959,27 @@ "failed": "L'importazione non è riuscita. Riprova.", "readFailed": "Impossibile leggere il file.", "resultSummary": "{measurements} misurazioni e {moods} voci dell'umore importate, {skipped} saltate." + }, + "csv": { + "title": "File CSV", + "description": "Importa misurazioni da un foglio di calcolo o dall'esportazione di un dispositivo. Una riga per lettura.", + "pasteLabel": "Incolla CSV", + "fileInputLabel": "Scegli un file CSV da importare", + "schemaHint": "Colonne: type, value, unit, measuredAt (con offset del fuso orario) e, facoltative, glucoseContext, notes, externalId.", + "docsLink": "Vedi lo schema delle colonne e un esempio.", + "uploadFile": "Carica file", + "downloadExample": "Scarica esempio", + "preview": "Anteprima", + "import": "Importa", + "empty": "Aggiungi almeno una riga CSV prima di importare.", + "rateLimited": "Troppe importazioni in questa ora (limite 5). Riprova più tardi.", + "tooLarge": "Questo file è troppo grande (limite 16 MB).", + "invalidPayload": "Impossibile analizzare il CSV. Controlla la riga di intestazione e riprova.", + "failed": "L'importazione non è riuscita. Riprova.", + "readFailed": "Impossibile leggere questo file.", + "previewSummary": "Anteprima: {inserted} verrebbero importate, {skipped} saltate.", + "resultSummary": "{inserted} nuove importate e {updated} aggiornate, {skipped} saltate.", + "rowError": "Riga {line}: {reason}" } } }, diff --git a/messages/pl.json b/messages/pl.json index b551d7b0b..ae31e8652 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -482,6 +482,7 @@ "pulseLabel": "Tętno (uderz./min)", "valueWithUnit": "Wartość ({unit})", "timestamp": "Czas", + "timestampBackdateHint": "Dostosuj datę i godzinę, aby zapisać odczyt z przeszłości.", "notesPlaceholder": "np. po jedzeniu", "formReset": "Resetuj formularz", "saveError": "Błąd zapisu", @@ -3958,6 +3959,27 @@ "failed": "Import nie powiódł się. Spróbuj ponownie.", "readFailed": "Nie można odczytać tego pliku.", "resultSummary": "Zaimportowano {measurements} pomiarów i {moods} wpisów nastroju, pominięto {skipped}." + }, + "csv": { + "title": "Plik CSV", + "description": "Importuj pomiary z arkusza lub eksportu z urządzenia. Jeden wiersz na odczyt.", + "pasteLabel": "Wklej CSV", + "fileInputLabel": "Wybierz plik CSV do zaimportowania", + "schemaHint": "Kolumny: type, value, unit, measuredAt (z przesunięciem strefy czasowej) oraz opcjonalnie glucoseContext, notes, externalId.", + "docsLink": "Zobacz schemat kolumn i przykład.", + "uploadFile": "Prześlij plik", + "downloadExample": "Pobierz przykład", + "preview": "Podgląd", + "import": "Importuj", + "empty": "Dodaj co najmniej jeden wiersz CSV przed importem.", + "rateLimited": "Zbyt wiele importów w tej godzinie (limit 5). Spróbuj ponownie później.", + "tooLarge": "Ten plik jest za duży (limit 16 MB).", + "invalidPayload": "Nie udało się przetworzyć pliku CSV. Sprawdź wiersz nagłówka i spróbuj ponownie.", + "failed": "Import nie powiódł się. Spróbuj ponownie.", + "readFailed": "Nie udało się odczytać tego pliku.", + "previewSummary": "Podgląd: {inserted} zostanie zaimportowanych, pominięto {skipped}.", + "resultSummary": "Zaimportowano {inserted} nowych i zaktualizowano {updated}, pominięto {skipped}.", + "rowError": "Wiersz {line}: {reason}" } } }, diff --git a/src/app/api/import/csv/__tests__/route.test.ts b/src/app/api/import/csv/__tests__/route.test.ts new file mode 100644 index 000000000..a522a3a08 --- /dev/null +++ b/src/app/api/import/csv/__tests__/route.test.ts @@ -0,0 +1,189 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +vi.mock("@/lib/api-handler", () => ({ + apiHandler: unknown>(fn: T) => fn, + requireAuth: vi.fn(async () => ({ + user: { id: "u-1" }, + session: { id: "s-1" }, + })), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { create: vi.fn(), upsert: vi.fn() }, + }, +})); + +vi.mock("@/lib/rate-limit", () => ({ + checkRateLimit: vi.fn(), +})); + +vi.mock("@/lib/auth/audit", () => ({ + auditLog: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/logging/context", () => ({ + annotate: vi.fn(), +})); + +vi.mock("@/lib/rollups/measurement-rollups", async () => { + const actual = await vi.importActual< + typeof import("@/lib/rollups/measurement-rollups") + >("@/lib/rollups/measurement-rollups"); + return { + ...actual, + recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined), + }; +}); + +import { NextRequest } from "next/server"; +import { POST } from "../route"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { prisma } from "@/lib/db"; +import { recomputeBucketsForMeasurement } from "@/lib/rollups/measurement-rollups"; + +const HEADER = "type,value,unit,measuredAt,glucoseContext,notes,externalId"; + +function csvRequest(csv: string, dryRun = false) { + return new NextRequest( + `http://localhost/api/import/csv${dryRun ? "?dryRun=1" : ""}`, + { + method: "POST", + body: csv, + headers: { "content-type": "text/csv" }, + }, + ); +} + +interface CsvEnvelope { + data: { + inserted: number; + updated: number; + skipped: number; + total: number; + dryRun: boolean; + rows: Array<{ line: number; status: string; reason?: string }>; + } | null; + error: string | null; +} + +beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(checkRateLimit).mockResolvedValue({ + allowed: true, + remaining: 4, + resetAt: new Date(Date.now() + 1000), + } as never); +}); + +describe("POST /api/import/csv — rate limit", () => { + it("returns 429 against the shared import bucket when exhausted", async () => { + vi.mocked(checkRateLimit).mockResolvedValue({ + allowed: false, + remaining: 0, + resetAt: new Date(Date.now() + 1000), + } as never); + + const res = await POST(csvRequest([HEADER, ""].join("\n"))); + expect(res.status).toBe(429); + expect(checkRateLimit).toHaveBeenCalledWith( + expect.stringContaining("import:u-1"), + 5, + 60 * 60 * 1000, + ); + }); +}); + +describe("POST /api/import/csv — fatal header error", () => { + it("returns 422 when a required column is missing", async () => { + const res = await POST(csvRequest("type,value,unit\nWEIGHT,80,kg")); + expect(res.status).toBe(422); + expect(vi.mocked(prisma.measurement.create)).not.toHaveBeenCalled(); + }); +}); + +describe("POST /api/import/csv — write loop + per-row envelope", () => { + it("inserts valid rows, skips invalid ones, returns per-row status", async () => { + vi.mocked(prisma.measurement.create).mockResolvedValue({} as never); + + const res = await POST( + csvRequest( + [ + HEADER, + "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,", // inserted + "NOPE,1,kg,2026-05-01T08:00:00Z,,,", // skipped unknown_type + "WEIGHT,80,kg,2026-05-01T08:00:00,,,", // skipped missing offset + ].join("\n"), + ), + ); + + expect(res.status).toBe(200); + const body = (await res.json()) as CsvEnvelope; + expect(body.data?.inserted).toBe(1); + expect(body.data?.skipped).toBe(2); + expect(body.data?.total).toBe(3); + expect(body.data?.dryRun).toBe(false); + const reasons = body.data?.rows.map((r) => r.reason); + expect(reasons).toContain("unknown_type"); + expect(reasons).toContain("missing_timezone_offset"); + // One create call for the single valid row. + expect(vi.mocked(prisma.measurement.create)).toHaveBeenCalledTimes(1); + // Rollup re-fold fired for the touched (type, day). + expect(vi.mocked(recomputeBucketsForMeasurement)).toHaveBeenCalled(); + }); + + it("dryRun previews without writing and reports projected inserts", async () => { + const res = await POST( + csvRequest( + [HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,"].join("\n"), + true, + ), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as CsvEnvelope; + expect(body.data?.dryRun).toBe(true); + expect(body.data?.inserted).toBe(1); + expect(body.data?.rows[0].status).toBe("inserted"); + expect(vi.mocked(prisma.measurement.create)).not.toHaveBeenCalled(); + expect(vi.mocked(recomputeBucketsForMeasurement)).not.toHaveBeenCalled(); + }); + + it("upserts an externalId row and surfaces it as updated when it already existed", async () => { + const created = new Date("2026-05-01T08:00:00Z"); + const bumped = new Date("2026-05-02T08:00:00Z"); + vi.mocked(prisma.measurement.upsert).mockResolvedValue({ + createdAt: created, + updatedAt: bumped, + } as never); + + const res = await POST( + csvRequest( + [HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,ext-1"].join("\n"), + ), + ); + const body = (await res.json()) as CsvEnvelope; + expect(body.data?.updated).toBe(1); + expect(body.data?.inserted).toBe(0); + expect(body.data?.rows[0].status).toBe("updated"); + expect(vi.mocked(prisma.measurement.upsert)).toHaveBeenCalledTimes(1); + }); + + it("counts a unique-constraint duplicate as skipped/duplicate", async () => { + vi.mocked(prisma.measurement.create).mockRejectedValue( + new Error("Unique constraint failed"), + ); + + const res = await POST( + csvRequest( + [HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,"].join("\n"), + ), + ); + const body = (await res.json()) as CsvEnvelope; + expect(body.data?.inserted).toBe(0); + expect(body.data?.skipped).toBe(1); + expect(body.data?.rows[0]).toMatchObject({ + status: "skipped", + reason: "duplicate", + }); + }); +}); diff --git a/src/app/api/import/csv/route.ts b/src/app/api/import/csv/route.ts new file mode 100644 index 000000000..f8c7f1b56 --- /dev/null +++ b/src/app/api/import/csv/route.ts @@ -0,0 +1,254 @@ +import { prisma } from "@/lib/db"; +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { annotate } from "@/lib/logging/context"; +import { auditLog } from "@/lib/auth/audit"; +import { apiSuccess, apiError, getClientIp } from "@/lib/api-response"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { NextRequest } from "next/server"; + +import { + parseCsvMeasurements, + type CsvRowResult, +} from "@/lib/import/csv-measurements"; +import { + collapseToTypeDayKeys, + recomputeBucketsForMeasurement, +} from "@/lib/rollups/measurement-rollups"; +import type { + MeasurementType, + GlucoseContext, +} from "@/generated/prisma/client"; + +/** + * CSV measurement import (v1.17.1). + * + * The cold-start escape hatch for self-hosters migrating from a spreadsheet + * or another tracker. Reuses the `/api/import` write loop + rollup re-fold; + * the parse + per-row validation + unit conversion lives in the pure + * `parseCsvMeasurements` module so it is unit-tested without a database. + * + * Body is `text/csv` (or `text/plain`) raw text — NOT JSON. Capped at 16 MB + * (the same ceiling as the JSON import; a 16 MB CSV is > 100 000 rows). Rows + * beyond the 10 000 ceiling are rejected so the rollup re-fold stays bounded. + * + * `?dryRun=1` parses + validates + returns the per-row status envelope + * WITHOUT writing — a trust win for a bulk cold-start import. + * + * Response: `{ inserted, updated, skipped, total, dryRun, rows: [{line, + * status, reason}] }`, mirroring the batch route's per-entry envelope. + */ + +const MAX_CSV_BYTES = 16 * 1024 * 1024; +const MAX_ROWS = 10000; + +export const POST = apiHandler(async (request: NextRequest) => { + const { user } = await requireAuth(); + annotate({ action: { name: "import.csv.upload" } }); + + const dryRun = + new URL(request.url).searchParams.get("dryRun") === "1" || + new URL(request.url).searchParams.get("dryRun") === "true"; + + // Same blast-radius reasoning as the JSON import — share the bucket so a + // CSV upload counts against the same 5/hour quota. + const rl = await checkRateLimit(`import:${user.id}`, 5, 60 * 60 * 1000); + if (!rl.allowed) { + return apiError("Maximum 5 imports per hour", 429); + } + + // Raw text read with an explicit byte cap — `safeJson` is JSON-only, so we + // enforce the same `maxBytes` ceiling here against the Content-Length and + // the materialised body. + const declaredLength = Number(request.headers.get("content-length") ?? "0"); + if (Number.isFinite(declaredLength) && declaredLength > MAX_CSV_BYTES) { + return apiError("CSV exceeds the 16 MB limit", 413); + } + let text: string; + try { + text = await request.text(); + } catch { + return apiError("Could not read the request body", 400); + } + if (Buffer.byteLength(text, "utf8") > MAX_CSV_BYTES) { + return apiError("CSV exceeds the 16 MB limit", 413); + } + + const parsed = parseCsvMeasurements(text); + if (parsed.fatal) { + return apiError(parsed.fatal.message, 422); + } + + const okRows = parsed.rows.filter( + (r): r is CsvRowResult & { row: NonNullable } => + r.status === "ok" && r.row !== undefined, + ); + if (okRows.length > MAX_ROWS) { + return apiError( + `CSV has ${okRows.length} valid rows — the limit is ${MAX_ROWS} per import`, + 422, + ); + } + + const userId = user.id; + let inserted = 0; + let updated = 0; + let skipped = parsed.rows.filter((r) => r.status === "skipped").length; + + const touchedMeasurements: Array<{ + type: MeasurementType; + measuredAt: Date; + }> = []; + + // Per-line write outcome for rows that reached the DB. Keyed by source line + // so the response envelope can surface `inserted` / `updated` / `duplicate` + // distinctly without polluting the parse-time `CsvRowReason`. + const writeOutcome = new Map(); + + if (!dryRun) { + for (const result of okRows) { + const m = result.row; + try { + if (m.externalId) { + // Idempotent re-import keyed on the source-stable id — re-uploading + // the same file updates in place rather than minting duplicates. + const res = await prisma.measurement.upsert({ + where: { + userId_type_source_externalId: { + userId, + type: m.type as MeasurementType, + source: "IMPORT", + externalId: m.externalId, + }, + }, + update: { + value: m.value, + unit: m.unit, + measuredAt: m.measuredAt, + notes: m.notes ?? null, + glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null, + }, + create: { + userId, + type: m.type as MeasurementType, + value: m.value, + unit: m.unit, + source: "IMPORT", + externalId: m.externalId, + measuredAt: m.measuredAt, + notes: m.notes ?? null, + glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null, + }, + select: { createdAt: true, updatedAt: true }, + }); + // A fresh create has createdAt === updatedAt; an in-place update + // bumps updatedAt past createdAt. + if (res.updatedAt.getTime() > res.createdAt.getTime()) { + updated++; + writeOutcome.set(result.line, "updated"); + } else { + inserted++; + writeOutcome.set(result.line, "inserted"); + } + } else { + await prisma.measurement.create({ + data: { + userId, + type: m.type as MeasurementType, + value: m.value, + unit: m.unit, + source: "IMPORT", + measuredAt: m.measuredAt, + notes: m.notes ?? null, + glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null, + }, + }); + inserted++; + writeOutcome.set(result.line, "inserted"); + } + touchedMeasurements.push({ + type: m.type as MeasurementType, + measuredAt: m.measuredAt, + }); + } catch { + // Unique-constraint violation (a NULL-externalId duplicate against an + // existing row) — count as skipped rather than failing the file. + skipped++; + writeOutcome.set(result.line, "duplicate"); + } + } + + // One bounded rollup re-fold per touched (type, day) — a 10 000-row CSV + // pays at most ~N (type, day) recomputes, not 10 000 per-row hooks. + // Best-effort: a populator hiccup never fails the importer. + if (touchedMeasurements.length > 0) { + try { + const keys = collapseToTypeDayKeys(touchedMeasurements); + for (const k of keys) { + await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt); + } + } catch (err) { + annotate({ + meta: { + measurement_rollup_csv_import_failed: true, + measurement_rollup_csv_import_error: + err instanceof Error ? err.message : String(err), + }, + }); + } + } + + await auditLog("import.csv.upload", { + userId, + ipAddress: getClientIp(request), + details: { inserted, updated, skipped }, + }); + } else { + // Dry-run preview: no write ran, so report the projected outcome — + // every valid parse-`ok` row would insert (an externalId row could land + // as an update, but that needs a DB read we deliberately skip here). + inserted = okRows.length; + } + + const summary = { + inserted, + updated, + skipped, + total: parsed.rows.length, + dryRun, + // Per-row status, mirroring the batch route's per-entry envelope. The + // normalised payload is stripped — the client only needs line + status. + // Parse-time `ok` rows are refined by the write outcome: a row that hit a + // unique-constraint duplicate becomes `skipped`/`duplicate`; an upsert + // that landed on an existing row becomes `updated`. In `dryRun` mode no + // write ran, so every parse-`ok` row stays `inserted` (the projected + // outcome). + rows: parsed.rows.map((r) => { + if (r.status === "skipped") { + return { + line: r.line, + status: "skipped" as const, + ...(r.reason ? { reason: r.reason } : {}), + }; + } + const outcome = writeOutcome.get(r.line); + if (outcome === "duplicate") { + return { line: r.line, status: "skipped" as const, reason: "duplicate" }; + } + if (outcome === "updated") { + return { line: r.line, status: "updated" as const }; + } + return { line: r.line, status: "inserted" as const }; + }), + }; + + annotate({ + meta: { + import_csv_inserted: inserted, + import_csv_updated: updated, + import_csv_skipped: skipped, + import_csv_dry_run: dryRun, + }, + }); + + return apiSuccess(summary); +}); diff --git a/src/components/settings/__tests__/import-panel.test.tsx b/src/components/settings/__tests__/import-panel.test.tsx index 39e0469e6..4448f86cb 100644 --- a/src/components/settings/__tests__/import-panel.test.tsx +++ b/src/components/settings/__tests__/import-panel.test.tsx @@ -27,7 +27,13 @@ vi.mock("next/navigation", () => ({ })); import { I18nProvider } from "@/lib/i18n/context"; -import { ImportPanel, EXAMPLE_IMPORT, parseImportJson } from "../import-panel"; +import { + ImportPanel, + EXAMPLE_IMPORT, + EXAMPLE_CSV, + parseImportJson, +} from "../import-panel"; +import { parseCsvMeasurements } from "@/lib/import/csv-measurements"; function render(node: React.ReactElement, locale: "en" | "de" = "en") { const queryClient = new QueryClient({ @@ -46,6 +52,7 @@ describe(" — SSR smoke", () => { expect(html).toContain('id="settings-section-import-title"'); expect(html).toContain('data-testid="import-card-apple-health"'); expect(html).toContain('data-testid="import-card-json"'); + expect(html).toContain('data-testid="import-card-csv"'); // Raw i18n keys never leak past the provider. expect(html).not.toContain("settings.sections.export.import."); }); @@ -58,6 +65,11 @@ describe(" — SSR smoke", () => { "import-json-choose-file", "import-json-download-example", "import-action-json", + "import-csv-textarea", + "import-csv-choose-file", + "import-csv-download-example", + "import-csv-preview", + "import-action-csv", ]) { expect(html).toContain(`data-testid="${id}"`); } @@ -112,6 +124,19 @@ describe("EXAMPLE_IMPORT payload", () => { }); }); +describe("EXAMPLE_CSV", () => { + it("parses entirely to ok rows (example must not drift from the schema)", () => { + // Pin the clock well past the fixture timestamps so the entry-instant + // bound never rejects the example as future-dated. + const out = parseCsvMeasurements(EXAMPLE_CSV, { + now: new Date("2026-06-01T00:00:00Z").getTime(), + }); + expect(out.fatal).toBeUndefined(); + expect(out.rows.length).toBeGreaterThan(0); + expect(out.rows.every((r) => r.status === "ok")).toBe(true); + }); +}); + describe("parseImportJson guard", () => { it("rejects an unparseable string before any network call", () => { const result = parseImportJson("{ not json"); diff --git a/src/components/settings/import-panel.tsx b/src/components/settings/import-panel.tsx index 7380a5844..2c0f8a1b0 100644 --- a/src/components/settings/import-panel.tsx +++ b/src/components/settings/import-panel.tsx @@ -32,6 +32,7 @@ import { CheckCircle2, Download, FileJson, + FileSpreadsheet, Loader2, Upload, } from "lucide-react"; @@ -68,6 +69,7 @@ export function ImportPanel() {
+
); @@ -630,3 +632,254 @@ function JsonImportCard() { ); } + +// ─────────────────────────── CSV import ─────────────────────────── + +/** + * Downloadable CSV example. Header is order-independent server-side, but the + * example pins the documented order so the docs + the route schema never + * drift. Exported so a test can assert it stays a valid header. + */ +export const EXAMPLE_CSV = [ + "type,value,unit,measuredAt,glucoseContext,notes,externalId", + "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,", + "BLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:05:00+02:00,FASTING,,meter-001", + "BLOOD_PRESSURE_SYS,120,mmHg,2026-05-01T08:05:00+02:00,,,", +].join("\n"); + +interface CsvRowResult { + line: number; + status: "inserted" | "updated" | "skipped"; + reason?: string; +} + +interface CsvImportResult { + inserted: number; + updated: number; + skipped: number; + total: number; + dryRun: boolean; + rows: CsvRowResult[]; +} + +function CsvImportCard() { + const { t } = useTranslations(); + const fileInputRef = useRef(null); + const [text, setText] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [result, setResult] = useState(null); + const textareaId = useId(); + + async function onFileChange(e: React.ChangeEvent) { + const file = e.target.files?.[0]; + e.target.value = ""; + if (!file) return; + setError(null); + setResult(null); + try { + const content = await file.text(); + setText(content); + } catch { + setError(t("settings.sections.export.import.csv.readFailed")); + } + } + + function downloadExample() { + const blob = new Blob([EXAMPLE_CSV], { type: "text/csv" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "healthlog-import-example.csv"; + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); + } + + async function send(dryRun: boolean) { + setError(null); + setResult(null); + if (text.trim().length === 0) { + setError(t("settings.sections.export.import.csv.empty")); + return; + } + setBusy(true); + try { + const res = await apiFetchRaw(`/api/import/csv${dryRun ? "?dryRun=1" : ""}`, { + method: "POST", + headers: { "Content-Type": "text/csv" }, + credentials: "include", + body: text, + }); + if (res.status === 429) { + setError(t("settings.sections.export.import.csv.rateLimited")); + return; + } + if (res.status === 413) { + setError(t("settings.sections.export.import.csv.tooLarge")); + return; + } + if (res.status === 422) { + const body = await res.json().catch(() => null); + setError( + body?.error ?? + t("settings.sections.export.import.csv.invalidPayload"), + ); + return; + } + if (!res.ok) { + setError(t("settings.sections.export.import.csv.failed")); + return; + } + const data = (await res.json()).data as CsvImportResult; + setResult(data); + } catch { + setError(t("settings.sections.export.import.csv.failed")); + } finally { + setBusy(false); + } + } + + const errorRows = result?.rows.filter((r) => r.status === "skipped") ?? []; + + return ( + +
+ +