mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 12:32:43 +00:00
feat(import): add CSV measurement import with per-row status and dry-run
A self-hoster migrating from a spreadsheet or a meter export overwhelmingly has CSV, which neither the JSON nor the Apple Health import covers. Add `POST /api/import/csv` reusing the existing write loop and rollup re-fold. The documented header is `type,value,unit,measuredAt[,glucoseContext,notes, externalId]`, order-independent. measuredAt must carry an explicit ISO-8601 offset — a row without one is skipped rather than guessed — and is bounded by the shared entry-instant validator. Glucose accepts mmol/L (converted to canonical mg/dL) and weight accepts lb (converted to kg); any other non-canonical unit is skipped, never mis-stored. An optional externalId column makes a re-upload idempotent; without it a re-upload duplicates. The response carries a per-row status envelope so a 2000-row file with three bad rows imports 1997 and names the three. `?dryRun=1` validates and previews the outcome without writing. The parse, validation, and unit conversion live in a pure module so they are unit-tested without a database. Settings -> Export & Import gains a third CSV card mirroring the JSON and Apple Health cards, with upload, paste, preview, and a downloadable example. The new route is documented in the OpenAPI contract.
This commit is contained in:
1 parent
82e2d7dfb1
commit
fbdcb31202
15 files changed
+1666
-2
No files matched your search
@@ -5202,6 +5202,54 @@ paths:
|
||||
"401": *a1
|
||||
"422": *a3
|
||||
"429": *a4
|
||||
/api/import/csv:
|
||||
post:
|
||||
tags:
|
||||
- Import
|
||||
summary: Import measurements from CSV
|
||||
description: "Bulk-import measurements from a CSV file (web cold-start escape hatch). Body is raw `text/csv` (≤ 16 MB, ≤
|
||||
10 000 valid rows). Header (order-independent): `type,value,unit,measuredAt[,glucoseContext,notes,externalId]`.
|
||||
`measuredAt` must carry an explicit ISO-8601 offset and is bounded (no future beyond a 5-min skew, no instant
|
||||
before 1900). Glucose accepts `mmol/L` (converted to canonical `mg/dL`); weight accepts `lb` (converted to
|
||||
`kg`). An `externalId` column makes re-upload idempotent (upsert on `(userId, type, source=IMPORT,
|
||||
externalId)`); without it a re-upload duplicates. `?dryRun=1` validates + previews without writing. Shares the
|
||||
5/hour `import:` rate bucket."
|
||||
parameters:
|
||||
- name: dryRun
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
enum:
|
||||
- "1"
|
||||
- "true"
|
||||
description: When `1` / `true`, parse + validate + return the per-row envelope without writing.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
text/csv:
|
||||
schema:
|
||||
type: string
|
||||
example: |-
|
||||
type,value,unit,measuredAt,glucoseContext,notes,externalId
|
||||
WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,
|
||||
BLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:05:00+02:00,FASTING,,meter-001
|
||||
responses:
|
||||
"200":
|
||||
description: Per-row import outcome.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/CsvImportResponseEnvelope"
|
||||
"401": *a1
|
||||
"413":
|
||||
description: CSV exceeds the 16 MB limit.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/CsvImportTooLargeEnvelope"
|
||||
"422": *a3
|
||||
"429": *a4
|
||||
components:
|
||||
schemas:
|
||||
PasskeyLoginVerifyRequest:
|
||||
@@ -15570,6 +15618,98 @@ components:
|
||||
- kind
|
||||
additionalProperties: false
|
||||
description: Outcome of the idempotent web `ai_full` grant. Safe to call on every AI-settings mount.
|
||||
CsvImportResponseEnvelope:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
$ref: "#/components/schemas/CsvImportResponse"
|
||||
error:
|
||||
type: "null"
|
||||
meta:
|
||||
type: object
|
||||
properties:
|
||||
requestId:
|
||||
type: string
|
||||
additionalProperties: false
|
||||
required:
|
||||
- data
|
||||
- error
|
||||
additionalProperties: false
|
||||
CsvImportResponse:
|
||||
type: object
|
||||
properties:
|
||||
inserted:
|
||||
type: integer
|
||||
minimum: -9007199254740991
|
||||
maximum: 9007199254740991
|
||||
updated:
|
||||
type: integer
|
||||
minimum: -9007199254740991
|
||||
maximum: 9007199254740991
|
||||
skipped:
|
||||
type: integer
|
||||
minimum: -9007199254740991
|
||||
maximum: 9007199254740991
|
||||
total:
|
||||
type: integer
|
||||
minimum: -9007199254740991
|
||||
maximum: 9007199254740991
|
||||
description: Total data rows parsed (excludes header).
|
||||
dryRun:
|
||||
type: boolean
|
||||
rows:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/CsvImportRowResult"
|
||||
required:
|
||||
- inserted
|
||||
- updated
|
||||
- skipped
|
||||
- total
|
||||
- dryRun
|
||||
- rows
|
||||
additionalProperties: false
|
||||
description: Per-row import outcome. Mirrors the batch-route per-entry envelope. In `dryRun` mode no write runs and
|
||||
every valid row reports its projected `inserted` status.
|
||||
CsvImportRowResult:
|
||||
type: object
|
||||
properties:
|
||||
line:
|
||||
type: integer
|
||||
minimum: -9007199254740991
|
||||
maximum: 9007199254740991
|
||||
description: 1-based source line number (the header is line 1).
|
||||
status:
|
||||
type: string
|
||||
enum:
|
||||
- inserted
|
||||
- updated
|
||||
- skipped
|
||||
reason:
|
||||
description: Machine-readable reason for a non-inserted row (e.g. unknown_type, unknown_unit, missing_timezone_offset,
|
||||
value_out_of_range, implausible_timestamp, duplicate).
|
||||
type: string
|
||||
required:
|
||||
- line
|
||||
- status
|
||||
additionalProperties: false
|
||||
CsvImportTooLargeEnvelope:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
type: "null"
|
||||
error:
|
||||
type: "null"
|
||||
meta:
|
||||
type: object
|
||||
properties:
|
||||
requestId:
|
||||
type: string
|
||||
additionalProperties: false
|
||||
required:
|
||||
- data
|
||||
- error
|
||||
additionalProperties: false
|
||||
MedicationScheduleOutput:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
@@ -482,6 +482,7 @@
|
||||
"pulseLabel": "Puls (bpm)",
|
||||
"valueWithUnit": "Wert ({unit})",
|
||||
"timestamp": "Zeitpunkt",
|
||||
"timestampBackdateHint": "Datum und Uhrzeit anpassen, um einen vergangenen Wert zu erfassen.",
|
||||
"notesPlaceholder": "z.B. nach dem Essen",
|
||||
"formReset": "Formular zurücksetzen",
|
||||
"saveError": "Fehler beim Speichern",
|
||||
@@ -3958,6 +3959,27 @@
|
||||
"failed": "Der Import ist fehlgeschlagen. Bitte erneut versuchen.",
|
||||
"readFailed": "Diese Datei konnte nicht gelesen werden.",
|
||||
"resultSummary": "{measurements} Messwerte und {moods} Stimmungseinträge importiert, {skipped} übersprungen."
|
||||
},
|
||||
"csv": {
|
||||
"title": "CSV-Datei",
|
||||
"description": "Messwerte aus einer Tabelle oder einem Messgerät-Export importieren. Eine Zeile pro Wert.",
|
||||
"pasteLabel": "CSV einfügen",
|
||||
"fileInputLabel": "CSV-Datei zum Importieren auswählen",
|
||||
"schemaHint": "Spalten: type, value, unit, measuredAt (mit Zeitzonen-Offset) sowie optional glucoseContext, notes, externalId.",
|
||||
"docsLink": "Spaltenschema und Beispiel ansehen.",
|
||||
"uploadFile": "Datei hochladen",
|
||||
"downloadExample": "Beispiel herunterladen",
|
||||
"preview": "Vorschau",
|
||||
"import": "Importieren",
|
||||
"empty": "Bitte vor dem Import mindestens eine CSV-Zeile hinzufügen.",
|
||||
"rateLimited": "Zu viele Importe in dieser Stunde (Limit 5). Bitte später erneut versuchen.",
|
||||
"tooLarge": "Diese Datei ist zu groß (Limit 16 MB).",
|
||||
"invalidPayload": "Die CSV konnte nicht gelesen werden. Bitte die Kopfzeile prüfen und erneut versuchen.",
|
||||
"failed": "Der Import ist fehlgeschlagen. Bitte erneut versuchen.",
|
||||
"readFailed": "Diese Datei konnte nicht gelesen werden.",
|
||||
"previewSummary": "Vorschau: {inserted} würden importiert, {skipped} übersprungen.",
|
||||
"resultSummary": "{inserted} neu importiert und {updated} aktualisiert, {skipped} übersprungen.",
|
||||
"rowError": "Zeile {line}: {reason}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
+23
-1
@@ -482,6 +482,7 @@
|
||||
"pulseLabel": "Pulse (bpm)",
|
||||
"valueWithUnit": "Value ({unit})",
|
||||
"timestamp": "Time",
|
||||
"timestampBackdateHint": "Adjust the date and time to log a past reading.",
|
||||
"notesPlaceholder": "e.g. after a meal",
|
||||
"formReset": "Reset form",
|
||||
"saveError": "Error saving",
|
||||
@@ -3919,7 +3920,7 @@
|
||||
},
|
||||
"import": {
|
||||
"heading": "Import",
|
||||
"description": "Bring data into HealthLog from an Apple Health export or a JSON file. Imports skip rows that already exist, so re-running one is safe.",
|
||||
"description": "Bring data into HealthLog from an Apple Health export, a JSON file, or a CSV. Imports skip rows that already exist, so re-running one is safe.",
|
||||
"appleHealth": {
|
||||
"title": "Apple Health",
|
||||
"description": "Upload the export.zip from the Health app on iPhone (Profile → Export All Health Data).",
|
||||
@@ -3958,6 +3959,27 @@
|
||||
"failed": "The import failed. Please try again.",
|
||||
"readFailed": "Couldn't read that file.",
|
||||
"resultSummary": "Imported {measurements} measurements and {moods} mood entries, {skipped} skipped."
|
||||
},
|
||||
"csv": {
|
||||
"title": "CSV file",
|
||||
"description": "Import measurements from a spreadsheet or meter export. One row per reading.",
|
||||
"pasteLabel": "Paste CSV",
|
||||
"fileInputLabel": "Choose a CSV file to import",
|
||||
"schemaHint": "Columns: type, value, unit, measuredAt (with timezone offset), and optional glucoseContext, notes, externalId.",
|
||||
"docsLink": "See the column schema and an example.",
|
||||
"uploadFile": "Upload file",
|
||||
"downloadExample": "Download example",
|
||||
"preview": "Preview",
|
||||
"import": "Import",
|
||||
"empty": "Add at least one CSV row before importing.",
|
||||
"rateLimited": "Too many imports this hour (limit 5). Please try again later.",
|
||||
"tooLarge": "That file is too large (limit 16 MB).",
|
||||
"invalidPayload": "The CSV could not be parsed. Check the header row and try again.",
|
||||
"failed": "The import failed. Please try again.",
|
||||
"readFailed": "Couldn't read that file.",
|
||||
"previewSummary": "Preview: {inserted} would import, {skipped} skipped.",
|
||||
"resultSummary": "Imported {inserted} new and updated {updated}, {skipped} skipped.",
|
||||
"rowError": "Line {line}: {reason}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -482,6 +482,7 @@
|
||||
"pulseLabel": "Pulso (lpm)",
|
||||
"valueWithUnit": "Valor ({unit})",
|
||||
"timestamp": "Hora",
|
||||
"timestampBackdateHint": "Ajusta la fecha y la hora para registrar una lectura pasada.",
|
||||
"notesPlaceholder": "p. ej. después de comer",
|
||||
"formReset": "Restablecer formulario",
|
||||
"saveError": "Error al guardar",
|
||||
@@ -3958,6 +3959,27 @@
|
||||
"failed": "La importación falló. Inténtalo de nuevo.",
|
||||
"readFailed": "No se pudo leer ese archivo.",
|
||||
"resultSummary": "{measurements} mediciones y {moods} entradas de ánimo importadas, {skipped} omitidas."
|
||||
},
|
||||
"csv": {
|
||||
"title": "Archivo CSV",
|
||||
"description": "Importa mediciones desde una hoja de cálculo o la exportación de un medidor. Una fila por lectura.",
|
||||
"pasteLabel": "Pegar CSV",
|
||||
"fileInputLabel": "Elegir un archivo CSV para importar",
|
||||
"schemaHint": "Columnas: type, value, unit, measuredAt (con desfase de zona horaria) y opcionales glucoseContext, notes, externalId.",
|
||||
"docsLink": "Ver el esquema de columnas y un ejemplo.",
|
||||
"uploadFile": "Subir archivo",
|
||||
"downloadExample": "Descargar ejemplo",
|
||||
"preview": "Vista previa",
|
||||
"import": "Importar",
|
||||
"empty": "Añade al menos una fila CSV antes de importar.",
|
||||
"rateLimited": "Demasiadas importaciones esta hora (límite 5). Inténtalo más tarde.",
|
||||
"tooLarge": "Ese archivo es demasiado grande (límite 16 MB).",
|
||||
"invalidPayload": "No se pudo analizar el CSV. Revisa la fila de encabezado e inténtalo de nuevo.",
|
||||
"failed": "La importación falló. Inténtalo de nuevo.",
|
||||
"readFailed": "No se pudo leer ese archivo.",
|
||||
"previewSummary": "Vista previa: {inserted} se importarían, {skipped} omitidas.",
|
||||
"resultSummary": "{inserted} nuevas importadas y {updated} actualizadas, {skipped} omitidas.",
|
||||
"rowError": "Línea {line}: {reason}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -482,6 +482,7 @@
|
||||
"pulseLabel": "Pouls (bpm)",
|
||||
"valueWithUnit": "Valeur ({unit})",
|
||||
"timestamp": "Heure",
|
||||
"timestampBackdateHint": "Ajustez la date et l'heure pour enregistrer un relevé passé.",
|
||||
"notesPlaceholder": "p. ex. après le repas",
|
||||
"formReset": "Réinitialiser le formulaire",
|
||||
"saveError": "Erreur lors de l’enregistrement",
|
||||
@@ -3958,6 +3959,27 @@
|
||||
"failed": "L'import a échoué. Veuillez réessayer.",
|
||||
"readFailed": "Impossible de lire ce fichier.",
|
||||
"resultSummary": "{measurements} mesures et {moods} entrées d'humeur importées, {skipped} ignorées."
|
||||
},
|
||||
"csv": {
|
||||
"title": "Fichier CSV",
|
||||
"description": "Importez des mesures depuis un tableur ou l'export d'un appareil. Une ligne par relevé.",
|
||||
"pasteLabel": "Coller le CSV",
|
||||
"fileInputLabel": "Choisir un fichier CSV à importer",
|
||||
"schemaHint": "Colonnes : type, value, unit, measuredAt (avec décalage horaire) et, en option, glucoseContext, notes, externalId.",
|
||||
"docsLink": "Voir le schéma des colonnes et un exemple.",
|
||||
"uploadFile": "Téléverser un fichier",
|
||||
"downloadExample": "Télécharger un exemple",
|
||||
"preview": "Aperçu",
|
||||
"import": "Importer",
|
||||
"empty": "Ajoutez au moins une ligne CSV avant d'importer.",
|
||||
"rateLimited": "Trop d'imports cette heure (limite 5). Veuillez réessayer plus tard.",
|
||||
"tooLarge": "Ce fichier est trop volumineux (limite 16 Mo).",
|
||||
"invalidPayload": "Le CSV n'a pas pu être analysé. Vérifiez la ligne d'en-tête et réessayez.",
|
||||
"failed": "L'import a échoué. Veuillez réessayer.",
|
||||
"readFailed": "Impossible de lire ce fichier.",
|
||||
"previewSummary": "Aperçu : {inserted} seraient importées, {skipped} ignorées.",
|
||||
"resultSummary": "{inserted} nouvelles importées et {updated} mises à jour, {skipped} ignorées.",
|
||||
"rowError": "Ligne {line} : {reason}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -482,6 +482,7 @@
|
||||
"pulseLabel": "Pulsazioni (bpm)",
|
||||
"valueWithUnit": "Valore ({unit})",
|
||||
"timestamp": "Ora",
|
||||
"timestampBackdateHint": "Regola data e ora per registrare una lettura passata.",
|
||||
"notesPlaceholder": "es. dopo il pasto",
|
||||
"formReset": "Reimposta modulo",
|
||||
"saveError": "Errore durante il salvataggio",
|
||||
@@ -3958,6 +3959,27 @@
|
||||
"failed": "L'importazione non è riuscita. Riprova.",
|
||||
"readFailed": "Impossibile leggere il file.",
|
||||
"resultSummary": "{measurements} misurazioni e {moods} voci dell'umore importate, {skipped} saltate."
|
||||
},
|
||||
"csv": {
|
||||
"title": "File CSV",
|
||||
"description": "Importa misurazioni da un foglio di calcolo o dall'esportazione di un dispositivo. Una riga per lettura.",
|
||||
"pasteLabel": "Incolla CSV",
|
||||
"fileInputLabel": "Scegli un file CSV da importare",
|
||||
"schemaHint": "Colonne: type, value, unit, measuredAt (con offset del fuso orario) e, facoltative, glucoseContext, notes, externalId.",
|
||||
"docsLink": "Vedi lo schema delle colonne e un esempio.",
|
||||
"uploadFile": "Carica file",
|
||||
"downloadExample": "Scarica esempio",
|
||||
"preview": "Anteprima",
|
||||
"import": "Importa",
|
||||
"empty": "Aggiungi almeno una riga CSV prima di importare.",
|
||||
"rateLimited": "Troppe importazioni in questa ora (limite 5). Riprova più tardi.",
|
||||
"tooLarge": "Questo file è troppo grande (limite 16 MB).",
|
||||
"invalidPayload": "Impossibile analizzare il CSV. Controlla la riga di intestazione e riprova.",
|
||||
"failed": "L'importazione non è riuscita. Riprova.",
|
||||
"readFailed": "Impossibile leggere questo file.",
|
||||
"previewSummary": "Anteprima: {inserted} verrebbero importate, {skipped} saltate.",
|
||||
"resultSummary": "{inserted} nuove importate e {updated} aggiornate, {skipped} saltate.",
|
||||
"rowError": "Riga {line}: {reason}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -482,6 +482,7 @@
|
||||
"pulseLabel": "Tętno (uderz./min)",
|
||||
"valueWithUnit": "Wartość ({unit})",
|
||||
"timestamp": "Czas",
|
||||
"timestampBackdateHint": "Dostosuj datę i godzinę, aby zapisać odczyt z przeszłości.",
|
||||
"notesPlaceholder": "np. po jedzeniu",
|
||||
"formReset": "Resetuj formularz",
|
||||
"saveError": "Błąd zapisu",
|
||||
@@ -3958,6 +3959,27 @@
|
||||
"failed": "Import nie powiódł się. Spróbuj ponownie.",
|
||||
"readFailed": "Nie można odczytać tego pliku.",
|
||||
"resultSummary": "Zaimportowano {measurements} pomiarów i {moods} wpisów nastroju, pominięto {skipped}."
|
||||
},
|
||||
"csv": {
|
||||
"title": "Plik CSV",
|
||||
"description": "Importuj pomiary z arkusza lub eksportu z urządzenia. Jeden wiersz na odczyt.",
|
||||
"pasteLabel": "Wklej CSV",
|
||||
"fileInputLabel": "Wybierz plik CSV do zaimportowania",
|
||||
"schemaHint": "Kolumny: type, value, unit, measuredAt (z przesunięciem strefy czasowej) oraz opcjonalnie glucoseContext, notes, externalId.",
|
||||
"docsLink": "Zobacz schemat kolumn i przykład.",
|
||||
"uploadFile": "Prześlij plik",
|
||||
"downloadExample": "Pobierz przykład",
|
||||
"preview": "Podgląd",
|
||||
"import": "Importuj",
|
||||
"empty": "Dodaj co najmniej jeden wiersz CSV przed importem.",
|
||||
"rateLimited": "Zbyt wiele importów w tej godzinie (limit 5). Spróbuj ponownie później.",
|
||||
"tooLarge": "Ten plik jest za duży (limit 16 MB).",
|
||||
"invalidPayload": "Nie udało się przetworzyć pliku CSV. Sprawdź wiersz nagłówka i spróbuj ponownie.",
|
||||
"failed": "Import nie powiódł się. Spróbuj ponownie.",
|
||||
"readFailed": "Nie udało się odczytać tego pliku.",
|
||||
"previewSummary": "Podgląd: {inserted} zostanie zaimportowanych, pominięto {skipped}.",
|
||||
"resultSummary": "Zaimportowano {inserted} nowych i zaktualizowano {updated}, pominięto {skipped}.",
|
||||
"rowError": "Wiersz {line}: {reason}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
|
||||
vi.mock("@/lib/api-handler", () => ({
|
||||
apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
|
||||
requireAuth: vi.fn(async () => ({
|
||||
user: { id: "u-1" },
|
||||
session: { id: "s-1" },
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: { create: vi.fn(), upsert: vi.fn() },
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/rate-limit", () => ({
|
||||
checkRateLimit: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/auth/audit", () => ({
|
||||
auditLog: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({
|
||||
annotate: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/rollups/measurement-rollups", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("@/lib/rollups/measurement-rollups")
|
||||
>("@/lib/rollups/measurement-rollups");
|
||||
return {
|
||||
...actual,
|
||||
recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
});
|
||||
|
||||
import { NextRequest } from "next/server";
|
||||
import { POST } from "../route";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { recomputeBucketsForMeasurement } from "@/lib/rollups/measurement-rollups";
|
||||
|
||||
const HEADER = "type,value,unit,measuredAt,glucoseContext,notes,externalId";
|
||||
|
||||
function csvRequest(csv: string, dryRun = false) {
|
||||
return new NextRequest(
|
||||
`http://localhost/api/import/csv${dryRun ? "?dryRun=1" : ""}`,
|
||||
{
|
||||
method: "POST",
|
||||
body: csv,
|
||||
headers: { "content-type": "text/csv" },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
interface CsvEnvelope {
|
||||
data: {
|
||||
inserted: number;
|
||||
updated: number;
|
||||
skipped: number;
|
||||
total: number;
|
||||
dryRun: boolean;
|
||||
rows: Array<{ line: number; status: string; reason?: string }>;
|
||||
} | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(checkRateLimit).mockResolvedValue({
|
||||
allowed: true,
|
||||
remaining: 4,
|
||||
resetAt: new Date(Date.now() + 1000),
|
||||
} as never);
|
||||
});
|
||||
|
||||
describe("POST /api/import/csv — rate limit", () => {
|
||||
it("returns 429 against the shared import bucket when exhausted", async () => {
|
||||
vi.mocked(checkRateLimit).mockResolvedValue({
|
||||
allowed: false,
|
||||
remaining: 0,
|
||||
resetAt: new Date(Date.now() + 1000),
|
||||
} as never);
|
||||
|
||||
const res = await POST(csvRequest([HEADER, ""].join("\n")));
|
||||
expect(res.status).toBe(429);
|
||||
expect(checkRateLimit).toHaveBeenCalledWith(
|
||||
expect.stringContaining("import:u-1"),
|
||||
5,
|
||||
60 * 60 * 1000,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /api/import/csv — fatal header error", () => {
|
||||
it("returns 422 when a required column is missing", async () => {
|
||||
const res = await POST(csvRequest("type,value,unit\nWEIGHT,80,kg"));
|
||||
expect(res.status).toBe(422);
|
||||
expect(vi.mocked(prisma.measurement.create)).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /api/import/csv — write loop + per-row envelope", () => {
|
||||
it("inserts valid rows, skips invalid ones, returns per-row status", async () => {
|
||||
vi.mocked(prisma.measurement.create).mockResolvedValue({} as never);
|
||||
|
||||
const res = await POST(
|
||||
csvRequest(
|
||||
[
|
||||
HEADER,
|
||||
"WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,", // inserted
|
||||
"NOPE,1,kg,2026-05-01T08:00:00Z,,,", // skipped unknown_type
|
||||
"WEIGHT,80,kg,2026-05-01T08:00:00,,,", // skipped missing offset
|
||||
].join("\n"),
|
||||
),
|
||||
);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as CsvEnvelope;
|
||||
expect(body.data?.inserted).toBe(1);
|
||||
expect(body.data?.skipped).toBe(2);
|
||||
expect(body.data?.total).toBe(3);
|
||||
expect(body.data?.dryRun).toBe(false);
|
||||
const reasons = body.data?.rows.map((r) => r.reason);
|
||||
expect(reasons).toContain("unknown_type");
|
||||
expect(reasons).toContain("missing_timezone_offset");
|
||||
// One create call for the single valid row.
|
||||
expect(vi.mocked(prisma.measurement.create)).toHaveBeenCalledTimes(1);
|
||||
// Rollup re-fold fired for the touched (type, day).
|
||||
expect(vi.mocked(recomputeBucketsForMeasurement)).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("dryRun previews without writing and reports projected inserts", async () => {
|
||||
const res = await POST(
|
||||
csvRequest(
|
||||
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,"].join("\n"),
|
||||
true,
|
||||
),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as CsvEnvelope;
|
||||
expect(body.data?.dryRun).toBe(true);
|
||||
expect(body.data?.inserted).toBe(1);
|
||||
expect(body.data?.rows[0].status).toBe("inserted");
|
||||
expect(vi.mocked(prisma.measurement.create)).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(recomputeBucketsForMeasurement)).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("upserts an externalId row and surfaces it as updated when it already existed", async () => {
|
||||
const created = new Date("2026-05-01T08:00:00Z");
|
||||
const bumped = new Date("2026-05-02T08:00:00Z");
|
||||
vi.mocked(prisma.measurement.upsert).mockResolvedValue({
|
||||
createdAt: created,
|
||||
updatedAt: bumped,
|
||||
} as never);
|
||||
|
||||
const res = await POST(
|
||||
csvRequest(
|
||||
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,ext-1"].join("\n"),
|
||||
),
|
||||
);
|
||||
const body = (await res.json()) as CsvEnvelope;
|
||||
expect(body.data?.updated).toBe(1);
|
||||
expect(body.data?.inserted).toBe(0);
|
||||
expect(body.data?.rows[0].status).toBe("updated");
|
||||
expect(vi.mocked(prisma.measurement.upsert)).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("counts a unique-constraint duplicate as skipped/duplicate", async () => {
|
||||
vi.mocked(prisma.measurement.create).mockRejectedValue(
|
||||
new Error("Unique constraint failed"),
|
||||
);
|
||||
|
||||
const res = await POST(
|
||||
csvRequest(
|
||||
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,"].join("\n"),
|
||||
),
|
||||
);
|
||||
const body = (await res.json()) as CsvEnvelope;
|
||||
expect(body.data?.inserted).toBe(0);
|
||||
expect(body.data?.skipped).toBe(1);
|
||||
expect(body.data?.rows[0]).toMatchObject({
|
||||
status: "skipped",
|
||||
reason: "duplicate",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,254 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, requireAuth } from "@/lib/api-handler";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { apiSuccess, apiError, getClientIp } from "@/lib/api-response";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
import {
|
||||
parseCsvMeasurements,
|
||||
type CsvRowResult,
|
||||
} from "@/lib/import/csv-measurements";
|
||||
import {
|
||||
collapseToTypeDayKeys,
|
||||
recomputeBucketsForMeasurement,
|
||||
} from "@/lib/rollups/measurement-rollups";
|
||||
import type {
|
||||
MeasurementType,
|
||||
GlucoseContext,
|
||||
} from "@/generated/prisma/client";
|
||||
|
||||
/**
|
||||
* CSV measurement import (v1.17.1).
|
||||
*
|
||||
* The cold-start escape hatch for self-hosters migrating from a spreadsheet
|
||||
* or another tracker. Reuses the `/api/import` write loop + rollup re-fold;
|
||||
* the parse + per-row validation + unit conversion lives in the pure
|
||||
* `parseCsvMeasurements` module so it is unit-tested without a database.
|
||||
*
|
||||
* Body is `text/csv` (or `text/plain`) raw text — NOT JSON. Capped at 16 MB
|
||||
* (the same ceiling as the JSON import; a 16 MB CSV is > 100 000 rows). Rows
|
||||
* beyond the 10 000 ceiling are rejected so the rollup re-fold stays bounded.
|
||||
*
|
||||
* `?dryRun=1` parses + validates + returns the per-row status envelope
|
||||
* WITHOUT writing — a trust win for a bulk cold-start import.
|
||||
*
|
||||
* Response: `{ inserted, updated, skipped, total, dryRun, rows: [{line,
|
||||
* status, reason}] }`, mirroring the batch route's per-entry envelope.
|
||||
*/
|
||||
|
||||
const MAX_CSV_BYTES = 16 * 1024 * 1024;
|
||||
const MAX_ROWS = 10000;
|
||||
|
||||
export const POST = apiHandler(async (request: NextRequest) => {
|
||||
const { user } = await requireAuth();
|
||||
annotate({ action: { name: "import.csv.upload" } });
|
||||
|
||||
const dryRun =
|
||||
new URL(request.url).searchParams.get("dryRun") === "1" ||
|
||||
new URL(request.url).searchParams.get("dryRun") === "true";
|
||||
|
||||
// Same blast-radius reasoning as the JSON import — share the bucket so a
|
||||
// CSV upload counts against the same 5/hour quota.
|
||||
const rl = await checkRateLimit(`import:${user.id}`, 5, 60 * 60 * 1000);
|
||||
if (!rl.allowed) {
|
||||
return apiError("Maximum 5 imports per hour", 429);
|
||||
}
|
||||
|
||||
// Raw text read with an explicit byte cap — `safeJson` is JSON-only, so we
|
||||
// enforce the same `maxBytes` ceiling here against the Content-Length and
|
||||
// the materialised body.
|
||||
const declaredLength = Number(request.headers.get("content-length") ?? "0");
|
||||
if (Number.isFinite(declaredLength) && declaredLength > MAX_CSV_BYTES) {
|
||||
return apiError("CSV exceeds the 16 MB limit", 413);
|
||||
}
|
||||
let text: string;
|
||||
try {
|
||||
text = await request.text();
|
||||
} catch {
|
||||
return apiError("Could not read the request body", 400);
|
||||
}
|
||||
if (Buffer.byteLength(text, "utf8") > MAX_CSV_BYTES) {
|
||||
return apiError("CSV exceeds the 16 MB limit", 413);
|
||||
}
|
||||
|
||||
const parsed = parseCsvMeasurements(text);
|
||||
if (parsed.fatal) {
|
||||
return apiError(parsed.fatal.message, 422);
|
||||
}
|
||||
|
||||
const okRows = parsed.rows.filter(
|
||||
(r): r is CsvRowResult & { row: NonNullable<CsvRowResult["row"]> } =>
|
||||
r.status === "ok" && r.row !== undefined,
|
||||
);
|
||||
if (okRows.length > MAX_ROWS) {
|
||||
return apiError(
|
||||
`CSV has ${okRows.length} valid rows — the limit is ${MAX_ROWS} per import`,
|
||||
422,
|
||||
);
|
||||
}
|
||||
|
||||
const userId = user.id;
|
||||
let inserted = 0;
|
||||
let updated = 0;
|
||||
let skipped = parsed.rows.filter((r) => r.status === "skipped").length;
|
||||
|
||||
const touchedMeasurements: Array<{
|
||||
type: MeasurementType;
|
||||
measuredAt: Date;
|
||||
}> = [];
|
||||
|
||||
// Per-line write outcome for rows that reached the DB. Keyed by source line
|
||||
// so the response envelope can surface `inserted` / `updated` / `duplicate`
|
||||
// distinctly without polluting the parse-time `CsvRowReason`.
|
||||
const writeOutcome = new Map<number, "inserted" | "updated" | "duplicate">();
|
||||
|
||||
if (!dryRun) {
|
||||
for (const result of okRows) {
|
||||
const m = result.row;
|
||||
try {
|
||||
if (m.externalId) {
|
||||
// Idempotent re-import keyed on the source-stable id — re-uploading
|
||||
// the same file updates in place rather than minting duplicates.
|
||||
const res = await prisma.measurement.upsert({
|
||||
where: {
|
||||
userId_type_source_externalId: {
|
||||
userId,
|
||||
type: m.type as MeasurementType,
|
||||
source: "IMPORT",
|
||||
externalId: m.externalId,
|
||||
},
|
||||
},
|
||||
update: {
|
||||
value: m.value,
|
||||
unit: m.unit,
|
||||
measuredAt: m.measuredAt,
|
||||
notes: m.notes ?? null,
|
||||
glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null,
|
||||
},
|
||||
create: {
|
||||
userId,
|
||||
type: m.type as MeasurementType,
|
||||
value: m.value,
|
||||
unit: m.unit,
|
||||
source: "IMPORT",
|
||||
externalId: m.externalId,
|
||||
measuredAt: m.measuredAt,
|
||||
notes: m.notes ?? null,
|
||||
glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null,
|
||||
},
|
||||
select: { createdAt: true, updatedAt: true },
|
||||
});
|
||||
// A fresh create has createdAt === updatedAt; an in-place update
|
||||
// bumps updatedAt past createdAt.
|
||||
if (res.updatedAt.getTime() > res.createdAt.getTime()) {
|
||||
updated++;
|
||||
writeOutcome.set(result.line, "updated");
|
||||
} else {
|
||||
inserted++;
|
||||
writeOutcome.set(result.line, "inserted");
|
||||
}
|
||||
} else {
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId,
|
||||
type: m.type as MeasurementType,
|
||||
value: m.value,
|
||||
unit: m.unit,
|
||||
source: "IMPORT",
|
||||
measuredAt: m.measuredAt,
|
||||
notes: m.notes ?? null,
|
||||
glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null,
|
||||
},
|
||||
});
|
||||
inserted++;
|
||||
writeOutcome.set(result.line, "inserted");
|
||||
}
|
||||
touchedMeasurements.push({
|
||||
type: m.type as MeasurementType,
|
||||
measuredAt: m.measuredAt,
|
||||
});
|
||||
} catch {
|
||||
// Unique-constraint violation (a NULL-externalId duplicate against an
|
||||
// existing row) — count as skipped rather than failing the file.
|
||||
skipped++;
|
||||
writeOutcome.set(result.line, "duplicate");
|
||||
}
|
||||
}
|
||||
|
||||
// One bounded rollup re-fold per touched (type, day) — a 10 000-row CSV
|
||||
// pays at most ~N (type, day) recomputes, not 10 000 per-row hooks.
|
||||
// Best-effort: a populator hiccup never fails the importer.
|
||||
if (touchedMeasurements.length > 0) {
|
||||
try {
|
||||
const keys = collapseToTypeDayKeys(touchedMeasurements);
|
||||
for (const k of keys) {
|
||||
await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt);
|
||||
}
|
||||
} catch (err) {
|
||||
annotate({
|
||||
meta: {
|
||||
measurement_rollup_csv_import_failed: true,
|
||||
measurement_rollup_csv_import_error:
|
||||
err instanceof Error ? err.message : String(err),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await auditLog("import.csv.upload", {
|
||||
userId,
|
||||
ipAddress: getClientIp(request),
|
||||
details: { inserted, updated, skipped },
|
||||
});
|
||||
} else {
|
||||
// Dry-run preview: no write ran, so report the projected outcome —
|
||||
// every valid parse-`ok` row would insert (an externalId row could land
|
||||
// as an update, but that needs a DB read we deliberately skip here).
|
||||
inserted = okRows.length;
|
||||
}
|
||||
|
||||
const summary = {
|
||||
inserted,
|
||||
updated,
|
||||
skipped,
|
||||
total: parsed.rows.length,
|
||||
dryRun,
|
||||
// Per-row status, mirroring the batch route's per-entry envelope. The
|
||||
// normalised payload is stripped — the client only needs line + status.
|
||||
// Parse-time `ok` rows are refined by the write outcome: a row that hit a
|
||||
// unique-constraint duplicate becomes `skipped`/`duplicate`; an upsert
|
||||
// that landed on an existing row becomes `updated`. In `dryRun` mode no
|
||||
// write ran, so every parse-`ok` row stays `inserted` (the projected
|
||||
// outcome).
|
||||
rows: parsed.rows.map((r) => {
|
||||
if (r.status === "skipped") {
|
||||
return {
|
||||
line: r.line,
|
||||
status: "skipped" as const,
|
||||
...(r.reason ? { reason: r.reason } : {}),
|
||||
};
|
||||
}
|
||||
const outcome = writeOutcome.get(r.line);
|
||||
if (outcome === "duplicate") {
|
||||
return { line: r.line, status: "skipped" as const, reason: "duplicate" };
|
||||
}
|
||||
if (outcome === "updated") {
|
||||
return { line: r.line, status: "updated" as const };
|
||||
}
|
||||
return { line: r.line, status: "inserted" as const };
|
||||
}),
|
||||
};
|
||||
|
||||
annotate({
|
||||
meta: {
|
||||
import_csv_inserted: inserted,
|
||||
import_csv_updated: updated,
|
||||
import_csv_skipped: skipped,
|
||||
import_csv_dry_run: dryRun,
|
||||
},
|
||||
});
|
||||
|
||||
return apiSuccess(summary);
|
||||
});
|
||||
@@ -27,7 +27,13 @@ vi.mock("next/navigation", () => ({
|
||||
}));
|
||||
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
import { ImportPanel, EXAMPLE_IMPORT, parseImportJson } from "../import-panel";
|
||||
import {
|
||||
ImportPanel,
|
||||
EXAMPLE_IMPORT,
|
||||
EXAMPLE_CSV,
|
||||
parseImportJson,
|
||||
} from "../import-panel";
|
||||
import { parseCsvMeasurements } from "@/lib/import/csv-measurements";
|
||||
|
||||
function render(node: React.ReactElement, locale: "en" | "de" = "en") {
|
||||
const queryClient = new QueryClient({
|
||||
@@ -46,6 +52,7 @@ describe("<ImportPanel> — SSR smoke", () => {
|
||||
expect(html).toContain('id="settings-section-import-title"');
|
||||
expect(html).toContain('data-testid="import-card-apple-health"');
|
||||
expect(html).toContain('data-testid="import-card-json"');
|
||||
expect(html).toContain('data-testid="import-card-csv"');
|
||||
// Raw i18n keys never leak past the provider.
|
||||
expect(html).not.toContain("settings.sections.export.import.");
|
||||
});
|
||||
@@ -58,6 +65,11 @@ describe("<ImportPanel> — SSR smoke", () => {
|
||||
"import-json-choose-file",
|
||||
"import-json-download-example",
|
||||
"import-action-json",
|
||||
"import-csv-textarea",
|
||||
"import-csv-choose-file",
|
||||
"import-csv-download-example",
|
||||
"import-csv-preview",
|
||||
"import-action-csv",
|
||||
]) {
|
||||
expect(html).toContain(`data-testid="${id}"`);
|
||||
}
|
||||
@@ -112,6 +124,19 @@ describe("EXAMPLE_IMPORT payload", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("EXAMPLE_CSV", () => {
|
||||
it("parses entirely to ok rows (example must not drift from the schema)", () => {
|
||||
// Pin the clock well past the fixture timestamps so the entry-instant
|
||||
// bound never rejects the example as future-dated.
|
||||
const out = parseCsvMeasurements(EXAMPLE_CSV, {
|
||||
now: new Date("2026-06-01T00:00:00Z").getTime(),
|
||||
});
|
||||
expect(out.fatal).toBeUndefined();
|
||||
expect(out.rows.length).toBeGreaterThan(0);
|
||||
expect(out.rows.every((r) => r.status === "ok")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseImportJson guard", () => {
|
||||
it("rejects an unparseable string before any network call", () => {
|
||||
const result = parseImportJson("{ not json");
|
||||
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
CheckCircle2,
|
||||
Download,
|
||||
FileJson,
|
||||
FileSpreadsheet,
|
||||
Loader2,
|
||||
Upload,
|
||||
} from "lucide-react";
|
||||
@@ -68,6 +69,7 @@ export function ImportPanel() {
|
||||
<div className="grid gap-4 md:grid-cols-2">
|
||||
<AppleHealthImportCard />
|
||||
<JsonImportCard />
|
||||
<CsvImportCard />
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
@@ -630,3 +632,254 @@ function JsonImportCard() {
|
||||
</ImportCardShell>
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────── CSV import ───────────────────────────
|
||||
|
||||
/**
|
||||
* Downloadable CSV example. Header is order-independent server-side, but the
|
||||
* example pins the documented order so the docs + the route schema never
|
||||
* drift. Exported so a test can assert it stays a valid header.
|
||||
*/
|
||||
export const EXAMPLE_CSV = [
|
||||
"type,value,unit,measuredAt,glucoseContext,notes,externalId",
|
||||
"WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,",
|
||||
"BLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:05:00+02:00,FASTING,,meter-001",
|
||||
"BLOOD_PRESSURE_SYS,120,mmHg,2026-05-01T08:05:00+02:00,,,",
|
||||
].join("\n");
|
||||
|
||||
interface CsvRowResult {
|
||||
line: number;
|
||||
status: "inserted" | "updated" | "skipped";
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
interface CsvImportResult {
|
||||
inserted: number;
|
||||
updated: number;
|
||||
skipped: number;
|
||||
total: number;
|
||||
dryRun: boolean;
|
||||
rows: CsvRowResult[];
|
||||
}
|
||||
|
||||
function CsvImportCard() {
|
||||
const { t } = useTranslations();
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const [text, setText] = useState("");
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [result, setResult] = useState<CsvImportResult | null>(null);
|
||||
const textareaId = useId();
|
||||
|
||||
async function onFileChange(e: React.ChangeEvent<HTMLInputElement>) {
|
||||
const file = e.target.files?.[0];
|
||||
e.target.value = "";
|
||||
if (!file) return;
|
||||
setError(null);
|
||||
setResult(null);
|
||||
try {
|
||||
const content = await file.text();
|
||||
setText(content);
|
||||
} catch {
|
||||
setError(t("settings.sections.export.import.csv.readFailed"));
|
||||
}
|
||||
}
|
||||
|
||||
function downloadExample() {
|
||||
const blob = new Blob([EXAMPLE_CSV], { type: "text/csv" });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = "healthlog-import-example.csv";
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
async function send(dryRun: boolean) {
|
||||
setError(null);
|
||||
setResult(null);
|
||||
if (text.trim().length === 0) {
|
||||
setError(t("settings.sections.export.import.csv.empty"));
|
||||
return;
|
||||
}
|
||||
setBusy(true);
|
||||
try {
|
||||
const res = await apiFetchRaw(`/api/import/csv${dryRun ? "?dryRun=1" : ""}`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "text/csv" },
|
||||
credentials: "include",
|
||||
body: text,
|
||||
});
|
||||
if (res.status === 429) {
|
||||
setError(t("settings.sections.export.import.csv.rateLimited"));
|
||||
return;
|
||||
}
|
||||
if (res.status === 413) {
|
||||
setError(t("settings.sections.export.import.csv.tooLarge"));
|
||||
return;
|
||||
}
|
||||
if (res.status === 422) {
|
||||
const body = await res.json().catch(() => null);
|
||||
setError(
|
||||
body?.error ??
|
||||
t("settings.sections.export.import.csv.invalidPayload"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!res.ok) {
|
||||
setError(t("settings.sections.export.import.csv.failed"));
|
||||
return;
|
||||
}
|
||||
const data = (await res.json()).data as CsvImportResult;
|
||||
setResult(data);
|
||||
} catch {
|
||||
setError(t("settings.sections.export.import.csv.failed"));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
const errorRows = result?.rows.filter((r) => r.status === "skipped") ?? [];
|
||||
|
||||
return (
|
||||
<ImportCardShell
|
||||
testId="import-card-csv"
|
||||
icon={FileSpreadsheet}
|
||||
title={t("settings.sections.export.import.csv.title")}
|
||||
description={t("settings.sections.export.import.csv.description")}
|
||||
>
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor={textareaId} className="text-xs">
|
||||
{t("settings.sections.export.import.csv.pasteLabel")}
|
||||
</Label>
|
||||
<Textarea
|
||||
id={textareaId}
|
||||
data-testid="import-csv-textarea"
|
||||
value={text}
|
||||
onChange={(e) => setText(e.target.value)}
|
||||
rows={5}
|
||||
spellCheck={false}
|
||||
placeholder="type,value,unit,measuredAt,…"
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".csv,text/csv"
|
||||
className="sr-only"
|
||||
aria-label={t("settings.sections.export.import.csv.fileInputLabel")}
|
||||
onChange={onFileChange}
|
||||
/>
|
||||
|
||||
<p className="text-muted-foreground text-xs">
|
||||
{t("settings.sections.export.import.csv.schemaHint")}{" "}
|
||||
<a
|
||||
href="/docs/integrations/data-import"
|
||||
className="text-primary underline underline-offset-2"
|
||||
>
|
||||
{t("settings.sections.export.import.csv.docsLink")}
|
||||
</a>
|
||||
</p>
|
||||
|
||||
<div aria-live="polite" className="space-y-2">
|
||||
{result && (
|
||||
<div data-testid="import-csv-result" className="space-y-1.5">
|
||||
<p className="text-foreground flex items-start gap-2 text-xs">
|
||||
<CheckCircle2
|
||||
className="mt-0.5 h-3.5 w-3.5 shrink-0 text-emerald-500"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<span>
|
||||
{result.dryRun
|
||||
? t("settings.sections.export.import.csv.previewSummary", {
|
||||
inserted: result.inserted,
|
||||
skipped: result.skipped,
|
||||
})
|
||||
: t("settings.sections.export.import.csv.resultSummary", {
|
||||
inserted: result.inserted,
|
||||
updated: result.updated,
|
||||
skipped: result.skipped,
|
||||
})}
|
||||
</span>
|
||||
</p>
|
||||
{errorRows.length > 0 && (
|
||||
<ul className="text-muted-foreground max-h-32 space-y-0.5 overflow-auto text-xs">
|
||||
{errorRows.slice(0, 50).map((r) => (
|
||||
<li key={r.line}>
|
||||
{t("settings.sections.export.import.csv.rowError", {
|
||||
line: r.line,
|
||||
reason: r.reason ?? "skipped",
|
||||
})}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{error && (
|
||||
<p
|
||||
role="alert"
|
||||
className="text-destructive flex items-start gap-2 text-sm"
|
||||
>
|
||||
<AlertCircle
|
||||
className="mt-0.5 h-3.5 w-3.5 shrink-0"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<span>{error}</span>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="mt-auto flex flex-wrap items-center gap-3 pt-1">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
data-testid="import-csv-choose-file"
|
||||
>
|
||||
<Upload className="h-3.5 w-3.5" />
|
||||
{t("settings.sections.export.import.csv.uploadFile")}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={downloadExample}
|
||||
data-testid="import-csv-download-example"
|
||||
>
|
||||
<Download className="h-3.5 w-3.5" />
|
||||
{t("settings.sections.export.import.csv.downloadExample")}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
disabled={busy || text.trim().length === 0}
|
||||
onClick={() => void send(true)}
|
||||
data-testid="import-csv-preview"
|
||||
>
|
||||
{t("settings.sections.export.import.csv.preview")}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
disabled={busy || text.trim().length === 0}
|
||||
onClick={() => void send(false)}
|
||||
data-testid="import-action-csv"
|
||||
>
|
||||
{busy ? (
|
||||
<Loader2 className="h-3.5 w-3.5 animate-spin motion-reduce:animate-none" />
|
||||
) : (
|
||||
<FileSpreadsheet className="h-3.5 w-3.5" />
|
||||
)}
|
||||
{t("settings.sections.export.import.csv.import")}
|
||||
</Button>
|
||||
</div>
|
||||
</ImportCardShell>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import {
|
||||
parseCsvMeasurements,
|
||||
splitCsvRows,
|
||||
} from "@/lib/import/csv-measurements";
|
||||
|
||||
// Pin the clock so the entry-instant bound is deterministic. All fixture
|
||||
// timestamps sit comfortably in the past relative to this.
|
||||
const NOW = new Date("2026-06-01T00:00:00.000Z").getTime();
|
||||
|
||||
const HEADER = "type,value,unit,measuredAt,glucoseContext,notes,externalId";
|
||||
|
||||
function parse(rows: string[]) {
|
||||
return parseCsvMeasurements([HEADER, ...rows].join("\n"), { now: NOW });
|
||||
}
|
||||
|
||||
describe("splitCsvRows", () => {
|
||||
it("handles CRLF, quoted commas, escaped quotes, and a BOM", () => {
|
||||
const csv =
|
||||
"a,b,c\r\n1,\"x,y\",\"he said \"\"hi\"\"\"\r\n2,plain,z\r\n";
|
||||
const grid = splitCsvRows(csv);
|
||||
expect(grid).toEqual([
|
||||
["a", "b", "c"],
|
||||
["1", "x,y", 'he said "hi"'],
|
||||
["2", "plain", "z"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("drops trailing blank lines", () => {
|
||||
expect(splitCsvRows("a,b\n1,2\n\n\n")).toEqual([
|
||||
["a", "b"],
|
||||
["1", "2"],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCsvMeasurements — header validation", () => {
|
||||
it("is fatal when a required column is missing", () => {
|
||||
const out = parseCsvMeasurements("type,value,unit\nWEIGHT,80,kg", {
|
||||
now: NOW,
|
||||
});
|
||||
expect(out.fatal?.reason).toBe("missing_required_column");
|
||||
expect(out.rows).toEqual([]);
|
||||
});
|
||||
|
||||
it("is fatal on an empty file", () => {
|
||||
const out = parseCsvMeasurements("", { now: NOW });
|
||||
expect(out.fatal?.reason).toBe("missing_required_column");
|
||||
});
|
||||
|
||||
it("accepts columns in any order", () => {
|
||||
const out = parseCsvMeasurements(
|
||||
["unit,measuredAt,type,value", "kg,2026-05-01T08:00:00Z,WEIGHT,80.5"].join(
|
||||
"\n",
|
||||
),
|
||||
{ now: NOW },
|
||||
);
|
||||
expect(out.fatal).toBeUndefined();
|
||||
expect(out.rows[0].status).toBe("ok");
|
||||
expect(out.rows[0].row?.value).toBe(80.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCsvMeasurements — per-row partial failure", () => {
|
||||
it("imports the good rows and skips the bad ones with distinct reasons", () => {
|
||||
const out = parse([
|
||||
"WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,", // ok
|
||||
"NOPE,1,kg,2026-05-01T08:00:00Z,,,", // unknown_type
|
||||
"WEIGHT,9999,kg,2026-05-01T08:00:00Z,,,", // value_out_of_range
|
||||
"WEIGHT,80,kg,2026-05-01T08:00:00,,,", // missing_timezone_offset
|
||||
]);
|
||||
const byLine = Object.fromEntries(
|
||||
out.rows.map((r) => [r.line, { status: r.status, reason: r.reason }]),
|
||||
);
|
||||
expect(byLine[2]).toEqual({ status: "ok", reason: undefined });
|
||||
expect(byLine[3]).toEqual({ status: "skipped", reason: "unknown_type" });
|
||||
expect(byLine[4]).toEqual({
|
||||
status: "skipped",
|
||||
reason: "value_out_of_range",
|
||||
});
|
||||
expect(byLine[5]).toEqual({
|
||||
status: "skipped",
|
||||
reason: "missing_timezone_offset",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports line numbers 1-based with the header as line 1", () => {
|
||||
const out = parse(["WEIGHT,80,kg,2026-05-01T08:00:00Z,,,"]);
|
||||
expect(out.rows[0].line).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCsvMeasurements — unit conversion", () => {
|
||||
it("converts glucose mmol/L to canonical mg/dL", () => {
|
||||
const out = parse([
|
||||
"BLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:00:00Z,FASTING,,",
|
||||
]);
|
||||
expect(out.rows[0].status).toBe("ok");
|
||||
expect(out.rows[0].row?.unit).toBe("mg/dL");
|
||||
expect(out.rows[0].row?.value).toBeCloseTo(5.3 * 18.016, 3);
|
||||
});
|
||||
|
||||
it("converts weight lb to canonical kg", () => {
|
||||
const out = parse(["WEIGHT,180,lb,2026-05-01T08:00:00Z,,,"]);
|
||||
expect(out.rows[0].status).toBe("ok");
|
||||
expect(out.rows[0].row?.unit).toBe("kg");
|
||||
expect(out.rows[0].row?.value).toBeCloseTo(180 * 0.453592, 3);
|
||||
});
|
||||
|
||||
it("accepts the canonical unit case-insensitively", () => {
|
||||
const out = parse(["WEIGHT,80,KG,2026-05-01T08:00:00Z,,,"]);
|
||||
expect(out.rows[0].status).toBe("ok");
|
||||
expect(out.rows[0].row?.unit).toBe("kg");
|
||||
});
|
||||
|
||||
it("skips an unrecognised unit rather than mis-storing", () => {
|
||||
const out = parse(["WEIGHT,80,stone,2026-05-01T08:00:00Z,,,"]);
|
||||
expect(out.rows[0]).toMatchObject({
|
||||
status: "skipped",
|
||||
reason: "unknown_unit",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCsvMeasurements — timezone + entry-instant bound", () => {
|
||||
it("skips a timestamp without an offset (no silent local interpretation)", () => {
|
||||
const out = parse(["WEIGHT,80,kg,2026-05-01T08:00:00,,,"]);
|
||||
expect(out.rows[0].reason).toBe("missing_timezone_offset");
|
||||
});
|
||||
|
||||
it("accepts both Z and ±HH:MM offsets", () => {
|
||||
const out = parse([
|
||||
"WEIGHT,80,kg,2026-05-01T08:00:00Z,,,",
|
||||
"WEIGHT,81,kg,2026-05-01T08:00:00+02:00,,,",
|
||||
]);
|
||||
expect(out.rows.map((r) => r.status)).toEqual(["ok", "ok"]);
|
||||
});
|
||||
|
||||
it("rejects a future-dated row via the entry-instant bound", () => {
|
||||
// 10 minutes past NOW — beyond the 5-min skew tolerance.
|
||||
const future = new Date(NOW + 10 * 60 * 1000).toISOString();
|
||||
const out = parse([`WEIGHT,80,kg,${future},,,`]);
|
||||
expect(out.rows[0]).toMatchObject({
|
||||
status: "skipped",
|
||||
reason: "implausible_timestamp",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a pre-1900 row", () => {
|
||||
const out = parse(["WEIGHT,80,kg,1899-12-31T00:00:00Z,,,"]);
|
||||
expect(out.rows[0]).toMatchObject({
|
||||
status: "skipped",
|
||||
reason: "implausible_timestamp",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCsvMeasurements — glucose context", () => {
|
||||
it("requires a context for BLOOD_GLUCOSE", () => {
|
||||
const out = parse(["BLOOD_GLUCOSE,95,mg/dL,2026-05-01T08:00:00Z,,,"]);
|
||||
expect(out.rows[0]).toMatchObject({
|
||||
status: "skipped",
|
||||
reason: "missing_glucose_context",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a context on a non-glucose row", () => {
|
||||
const out = parse(["WEIGHT,80,kg,2026-05-01T08:00:00Z,FASTING,,"]);
|
||||
expect(out.rows[0]).toMatchObject({
|
||||
status: "skipped",
|
||||
reason: "unexpected_glucose_context",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an unknown context value", () => {
|
||||
const out = parse(["BLOOD_GLUCOSE,95,mg/dL,2026-05-01T08:00:00Z,LUNCH,,"]);
|
||||
expect(out.rows[0]).toMatchObject({
|
||||
status: "skipped",
|
||||
reason: "invalid_glucose_context",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCsvMeasurements — optional columns", () => {
|
||||
it("carries externalId through and trims notes", () => {
|
||||
const out = parse(["WEIGHT,80,kg,2026-05-01T08:00:00Z,, after run ,ext-9"]);
|
||||
expect(out.rows[0].row).toMatchObject({
|
||||
externalId: "ext-9",
|
||||
notes: "after run",
|
||||
});
|
||||
});
|
||||
|
||||
it("works without the optional columns present at all", () => {
|
||||
const out = parseCsvMeasurements(
|
||||
["type,value,unit,measuredAt", "WEIGHT,80,kg,2026-05-01T08:00:00Z"].join(
|
||||
"\n",
|
||||
),
|
||||
{ now: NOW },
|
||||
);
|
||||
expect(out.rows[0].status).toBe("ok");
|
||||
expect(out.rows[0].row?.externalId).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,368 @@
|
||||
/**
|
||||
* CSV measurement import — parse + normalise + per-row validation (v1.17.1).
|
||||
*
|
||||
* A cold-start escape hatch for self-hosters migrating from a spreadsheet,
|
||||
* a glucose-meter export, or another tracker. The genuine import gap left
|
||||
* by the JSON (`/api/import`) and Apple-Health (`/api/import/apple-health-export`)
|
||||
* routes is plain CSV, which neither covers.
|
||||
*
|
||||
* This module is the pure, side-effect-free core: it turns raw CSV text
|
||||
* into a per-row result set carrying either a normalised measurement (ready
|
||||
* for the `/api/import` write loop) or a `skipped` status with a stable
|
||||
* machine-readable `reason`. The route (`/api/import/csv`) owns the DB write
|
||||
* + rollup re-fold; everything testable without a database lives here.
|
||||
*
|
||||
* Documented column schema (header row required, order-independent):
|
||||
*
|
||||
* type,value,unit,measuredAt[,glucoseContext][,notes][,externalId]
|
||||
*
|
||||
* - `type` — a `measurementTypeEnum` value (e.g. WEIGHT). Unknown → skipped.
|
||||
* - `value` — a number in the row's `unit`.
|
||||
* - `unit` — the canonical unit for the type, OR a recognised alias:
|
||||
* glucose mmol/L → mg/dL (× 18.016),
|
||||
* weight lb → kg (× 0.453592).
|
||||
* Any other non-canonical unit → skipped (never silently
|
||||
* mis-stored).
|
||||
* - `measuredAt` — ISO-8601 WITH an explicit offset (`Z` or ±HH:MM). A row
|
||||
* without an offset is skipped — the importer never guesses
|
||||
* a timezone. The instant is bounded by `validateEntryInstant`
|
||||
* (no future beyond a 5-min skew, no instant before 1900).
|
||||
* - `glucoseContext` — required for BLOOD_GLUCOSE, forbidden otherwise.
|
||||
* - `notes` — free text, ≤ 200 chars.
|
||||
* - `externalId` — optional source-stable id. When present, the write loop
|
||||
* upserts on `(userId, type, source=IMPORT, externalId)` so a
|
||||
* re-upload is idempotent. Absent → re-upload duplicates.
|
||||
*/
|
||||
import {
|
||||
measurementTypeEnum,
|
||||
glucoseContextEnum,
|
||||
getUnitForType,
|
||||
validateMeasurementRange,
|
||||
MEASUREMENT_NOTES_MAX_LENGTH,
|
||||
} from "@/lib/validations/measurement";
|
||||
import { isPlausibleEntryInstant } from "@/lib/validations/entry-instant";
|
||||
|
||||
/** Canonical conversion factors keyed by `${type}:${loweredAlias}`. */
|
||||
const GLUCOSE_MMOL_TO_MGDL = 18.016;
|
||||
const LB_TO_KG = 0.453592;
|
||||
|
||||
/** A normalised, write-ready row in the `/api/import` measurement shape. */
|
||||
export interface NormalisedMeasurementRow {
|
||||
type: string;
|
||||
value: number;
|
||||
unit: string;
|
||||
measuredAt: Date;
|
||||
glucoseContext?: string;
|
||||
notes?: string;
|
||||
externalId?: string;
|
||||
}
|
||||
|
||||
/** Stable machine-readable skip reasons, mirroring the batch route's vocabulary. */
|
||||
export type CsvRowReason =
|
||||
| "missing_required_column"
|
||||
| "unknown_type"
|
||||
| "invalid_value"
|
||||
| "value_out_of_range"
|
||||
| "unknown_unit"
|
||||
| "missing_timezone_offset"
|
||||
| "invalid_timestamp"
|
||||
| "implausible_timestamp"
|
||||
| "missing_glucose_context"
|
||||
| "unexpected_glucose_context"
|
||||
| "invalid_glucose_context"
|
||||
| "notes_too_long"
|
||||
| "external_id_too_long"
|
||||
| "wrong_column_count";
|
||||
|
||||
export type CsvRowStatus = "ok" | "skipped";
|
||||
|
||||
export interface CsvRowResult {
|
||||
/** 1-based source line number (header is line 1, first data row is line 2). */
|
||||
line: number;
|
||||
status: CsvRowStatus;
|
||||
reason?: CsvRowReason;
|
||||
/** Present only when `status === "ok"`. */
|
||||
row?: NormalisedMeasurementRow;
|
||||
}
|
||||
|
||||
export interface CsvParseOutcome {
|
||||
/** A fatal, file-level error (e.g. missing required header). When set, `rows` is empty. */
|
||||
fatal?: { reason: CsvRowReason; message: string };
|
||||
rows: CsvRowResult[];
|
||||
}
|
||||
|
||||
const REQUIRED_COLUMNS = ["type", "value", "unit", "measuredat"] as const;
|
||||
const OPTIONAL_COLUMNS = ["glucosecontext", "notes", "externalid"] as const;
|
||||
|
||||
/**
|
||||
* Split CSV text into rows of cells. Hand-rolled, zero-dependency, RFC-4180-ish:
|
||||
* handles quoted fields, escaped quotes (`""`), commas + newlines inside quotes,
|
||||
* a leading UTF-8 BOM, and CRLF / LF line endings. Trailing blank lines are
|
||||
* dropped. Sufficient for the flat measurement schema; not a general CSV engine.
|
||||
*/
|
||||
export function splitCsvRows(text: string): string[][] {
|
||||
// Strip a leading BOM so the first header cell isn't "type".
|
||||
const input = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
||||
const rows: string[][] = [];
|
||||
let cell = "";
|
||||
let row: string[] = [];
|
||||
let inQuotes = false;
|
||||
let sawAnyChar = false;
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
const ch = input[i];
|
||||
if (inQuotes) {
|
||||
if (ch === '"') {
|
||||
if (input[i + 1] === '"') {
|
||||
cell += '"';
|
||||
i++;
|
||||
} else {
|
||||
inQuotes = false;
|
||||
}
|
||||
} else {
|
||||
cell += ch;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (ch === '"') {
|
||||
inQuotes = true;
|
||||
sawAnyChar = true;
|
||||
continue;
|
||||
}
|
||||
if (ch === ",") {
|
||||
row.push(cell);
|
||||
cell = "";
|
||||
sawAnyChar = true;
|
||||
continue;
|
||||
}
|
||||
if (ch === "\r") {
|
||||
// Swallow; the \n (if any) finalises the row.
|
||||
continue;
|
||||
}
|
||||
if (ch === "\n") {
|
||||
row.push(cell);
|
||||
rows.push(row);
|
||||
row = [];
|
||||
cell = "";
|
||||
sawAnyChar = false;
|
||||
continue;
|
||||
}
|
||||
cell += ch;
|
||||
sawAnyChar = true;
|
||||
}
|
||||
// Flush a trailing row that did not end in a newline.
|
||||
if (sawAnyChar || cell.length > 0 || row.length > 0) {
|
||||
row.push(cell);
|
||||
rows.push(row);
|
||||
}
|
||||
// Drop fully-empty rows (e.g. a trailing blank line that produced [""]).
|
||||
return rows.filter((r) => !(r.length === 1 && r[0].trim() === ""));
|
||||
}
|
||||
|
||||
/** True when an ISO-8601 datetime string carries an explicit offset (`Z` or ±HH:MM). */
|
||||
function hasExplicitOffset(value: string): boolean {
|
||||
// Offset is in the time portion, after the `T`. Match a trailing Z or ±HH:MM.
|
||||
return /([zZ]|[+-]\d{2}:?\d{2})$/.test(value.trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the row's unit to the canonical unit + converted value, or return
|
||||
* `null` when the unit is neither canonical nor a recognised alias.
|
||||
*/
|
||||
function convertToCanonicalUnit(
|
||||
type: string,
|
||||
value: number,
|
||||
rawUnit: string,
|
||||
): { value: number; unit: string } | null {
|
||||
const canonical = getUnitForType(type);
|
||||
const unit = rawUnit.trim();
|
||||
// Case-insensitive match against the canonical unit (mg/dL vs MG/DL).
|
||||
if (unit.toLowerCase() === canonical.toLowerCase()) {
|
||||
return { value, unit: canonical };
|
||||
}
|
||||
const lower = unit.toLowerCase();
|
||||
// Glucose: mmol/L → mg/dL.
|
||||
if (type === "BLOOD_GLUCOSE" && (lower === "mmol/l" || lower === "mmol")) {
|
||||
return { value: value * GLUCOSE_MMOL_TO_MGDL, unit: canonical };
|
||||
}
|
||||
// Weight: lb → kg.
|
||||
if (
|
||||
type === "WEIGHT" &&
|
||||
(lower === "lb" || lower === "lbs" || lower === "pound" || lower === "pounds")
|
||||
) {
|
||||
return { value: value * LB_TO_KG, unit: canonical };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const KNOWN_TYPES = new Set(measurementTypeEnum.options as readonly string[]);
|
||||
const KNOWN_GLUCOSE_CONTEXTS = new Set(
|
||||
glucoseContextEnum.options as readonly string[],
|
||||
);
|
||||
|
||||
/**
|
||||
* Parse + validate + normalise a CSV measurement file. Returns a per-row
|
||||
* result set: each data row is either `ok` with a normalised measurement or
|
||||
* `skipped` with a stable reason. A missing required header column is the
|
||||
* only fatal, file-level failure.
|
||||
*
|
||||
* `now` is injectable so tests pin the entry-instant clock.
|
||||
*/
|
||||
export function parseCsvMeasurements(
|
||||
text: string,
|
||||
opts: { now?: number } = {},
|
||||
): CsvParseOutcome {
|
||||
const now = opts.now ?? Date.now();
|
||||
const grid = splitCsvRows(text);
|
||||
|
||||
if (grid.length === 0) {
|
||||
return {
|
||||
fatal: {
|
||||
reason: "missing_required_column",
|
||||
message: "The CSV is empty — a header row is required.",
|
||||
},
|
||||
rows: [],
|
||||
};
|
||||
}
|
||||
|
||||
const header = grid[0].map((h) => h.trim().toLowerCase());
|
||||
const colIndex = new Map<string, number>();
|
||||
header.forEach((name, idx) => {
|
||||
if (!colIndex.has(name)) colIndex.set(name, idx);
|
||||
});
|
||||
|
||||
const missing = REQUIRED_COLUMNS.filter((c) => !colIndex.has(c));
|
||||
if (missing.length > 0) {
|
||||
return {
|
||||
fatal: {
|
||||
reason: "missing_required_column",
|
||||
message: `Missing required column(s): ${missing.join(", ")}. Required header: ${REQUIRED_COLUMNS.join(",")}[,${OPTIONAL_COLUMNS.join(",")}].`,
|
||||
},
|
||||
rows: [],
|
||||
};
|
||||
}
|
||||
|
||||
const idxType = colIndex.get("type")!;
|
||||
const idxValue = colIndex.get("value")!;
|
||||
const idxUnit = colIndex.get("unit")!;
|
||||
const idxMeasuredAt = colIndex.get("measuredat")!;
|
||||
const idxContext = colIndex.get("glucosecontext");
|
||||
const idxNotes = colIndex.get("notes");
|
||||
const idxExternalId = colIndex.get("externalid");
|
||||
|
||||
const rows: CsvRowResult[] = [];
|
||||
|
||||
for (let r = 1; r < grid.length; r++) {
|
||||
const cells = grid[r];
|
||||
const line = r + 1; // 1-based, header is line 1.
|
||||
const cellAt = (idx: number | undefined): string =>
|
||||
idx === undefined ? "" : (cells[idx] ?? "").trim();
|
||||
|
||||
const type = cellAt(idxType);
|
||||
const rawValue = cellAt(idxValue);
|
||||
const rawUnit = cellAt(idxUnit);
|
||||
const rawMeasuredAt = cellAt(idxMeasuredAt);
|
||||
const rawContext = cellAt(idxContext);
|
||||
const rawNotes = idxNotes === undefined ? "" : (cells[idxNotes] ?? "");
|
||||
const rawExternalId = cellAt(idxExternalId);
|
||||
|
||||
const skip = (reason: CsvRowReason): void => {
|
||||
rows.push({ line, status: "skipped", reason });
|
||||
};
|
||||
|
||||
if (!KNOWN_TYPES.has(type)) {
|
||||
skip("unknown_type");
|
||||
continue;
|
||||
}
|
||||
|
||||
const value = Number(rawValue);
|
||||
if (rawValue === "" || !Number.isFinite(value)) {
|
||||
skip("invalid_value");
|
||||
continue;
|
||||
}
|
||||
|
||||
const converted = convertToCanonicalUnit(type, value, rawUnit);
|
||||
if (converted === null) {
|
||||
skip("unknown_unit");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (validateMeasurementRange(type, converted.value) !== null) {
|
||||
skip("value_out_of_range");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!rawMeasuredAt) {
|
||||
skip("invalid_timestamp");
|
||||
continue;
|
||||
}
|
||||
if (!hasExplicitOffset(rawMeasuredAt)) {
|
||||
skip("missing_timezone_offset");
|
||||
continue;
|
||||
}
|
||||
const measuredAt = new Date(rawMeasuredAt);
|
||||
if (Number.isNaN(measuredAt.getTime())) {
|
||||
skip("invalid_timestamp");
|
||||
continue;
|
||||
}
|
||||
if (!isPlausibleEntryInstant(measuredAt, { now })) {
|
||||
skip("implausible_timestamp");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Glucose context: required for BLOOD_GLUCOSE, forbidden otherwise.
|
||||
let glucoseContext: string | undefined;
|
||||
if (type === "BLOOD_GLUCOSE") {
|
||||
if (!rawContext) {
|
||||
skip("missing_glucose_context");
|
||||
continue;
|
||||
}
|
||||
if (!KNOWN_GLUCOSE_CONTEXTS.has(rawContext)) {
|
||||
skip("invalid_glucose_context");
|
||||
continue;
|
||||
}
|
||||
glucoseContext = rawContext;
|
||||
} else if (rawContext) {
|
||||
skip("unexpected_glucose_context");
|
||||
continue;
|
||||
}
|
||||
|
||||
const notes = rawNotes.trim();
|
||||
if (notes.length > MEASUREMENT_NOTES_MAX_LENGTH) {
|
||||
skip("notes_too_long");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (rawExternalId.length > 120) {
|
||||
skip("external_id_too_long");
|
||||
continue;
|
||||
}
|
||||
|
||||
rows.push({
|
||||
line,
|
||||
status: "ok",
|
||||
row: {
|
||||
type,
|
||||
value: converted.value,
|
||||
unit: converted.unit,
|
||||
measuredAt,
|
||||
...(glucoseContext ? { glucoseContext } : {}),
|
||||
...(notes ? { notes } : {}),
|
||||
...(rawExternalId ? { externalId: rawExternalId } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return { rows };
|
||||
}
|
||||
|
||||
/** Column order for the downloadable example + docs. */
|
||||
export const CSV_EXAMPLE_COLUMNS = [
|
||||
"type",
|
||||
"value",
|
||||
"unit",
|
||||
"measuredAt",
|
||||
"glucoseContext",
|
||||
"notes",
|
||||
"externalId",
|
||||
] as const;
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* OpenAPI route table — CSV measurement import (v1.17.1).
|
||||
*
|
||||
* The CSV importer is the cold-start escape hatch for self-hosters migrating
|
||||
* from a spreadsheet or another tracker. The JSON (`/api/import`) and Apple-
|
||||
* Health (`/api/import/apple-health-export`) routes predate the contract
|
||||
* surface and are intentionally not documented here; this module documents
|
||||
* the one new endpoint so the wire contract stays drift-free (CI gate). iOS
|
||||
* does not call it — its canonical ingest is `POST /api/measurements/batch`.
|
||||
*/
|
||||
import type { ZodOpenApiObject } from "zod-openapi";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import { dataEnvelope, stdResponses } from "./shared";
|
||||
|
||||
const csvImportRowResult = z
|
||||
.object({
|
||||
line: z
|
||||
.number()
|
||||
.int()
|
||||
.describe("1-based source line number (the header is line 1)."),
|
||||
status: z.enum(["inserted", "updated", "skipped"]),
|
||||
reason: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe(
|
||||
"Machine-readable reason for a non-inserted row (e.g. unknown_type, unknown_unit, missing_timezone_offset, value_out_of_range, implausible_timestamp, duplicate).",
|
||||
),
|
||||
})
|
||||
.meta({ id: "CsvImportRowResult" });
|
||||
|
||||
const csvImportResponse = z
|
||||
.object({
|
||||
inserted: z.number().int(),
|
||||
updated: z.number().int(),
|
||||
skipped: z.number().int(),
|
||||
total: z.number().int().describe("Total data rows parsed (excludes header)."),
|
||||
dryRun: z.boolean(),
|
||||
rows: z.array(csvImportRowResult),
|
||||
})
|
||||
.meta({
|
||||
id: "CsvImportResponse",
|
||||
description:
|
||||
"Per-row import outcome. Mirrors the batch-route per-entry envelope. In `dryRun` mode no write runs and every valid row reports its projected `inserted` status.",
|
||||
});
|
||||
|
||||
export const importPaths: NonNullable<ZodOpenApiObject["paths"]> = {
|
||||
"/api/import/csv": {
|
||||
post: {
|
||||
tags: ["Import"],
|
||||
summary: "Import measurements from CSV",
|
||||
description:
|
||||
"Bulk-import measurements from a CSV file (web cold-start escape hatch). Body is raw `text/csv` (≤ 16 MB, ≤ 10 000 valid rows). Header (order-independent): `type,value,unit,measuredAt[,glucoseContext,notes,externalId]`. `measuredAt` must carry an explicit ISO-8601 offset and is bounded (no future beyond a 5-min skew, no instant before 1900). Glucose accepts `mmol/L` (converted to canonical `mg/dL`); weight accepts `lb` (converted to `kg`). An `externalId` column makes re-upload idempotent (upsert on `(userId, type, source=IMPORT, externalId)`); without it a re-upload duplicates. `?dryRun=1` validates + previews without writing. Shares the 5/hour `import:` rate bucket.",
|
||||
parameters: [
|
||||
{
|
||||
name: "dryRun",
|
||||
in: "query",
|
||||
required: false,
|
||||
schema: { type: "string", enum: ["1", "true"] },
|
||||
description:
|
||||
"When `1` / `true`, parse + validate + return the per-row envelope without writing.",
|
||||
},
|
||||
],
|
||||
requestBody: {
|
||||
required: true,
|
||||
content: {
|
||||
"text/csv": {
|
||||
schema: {
|
||||
type: "string",
|
||||
example:
|
||||
"type,value,unit,measuredAt,glucoseContext,notes,externalId\nWEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,\nBLOOD_GLUCOSE,5.3,mmol/L,2026-05-01T08:05:00+02:00,FASTING,,meter-001",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
responses: {
|
||||
"200": {
|
||||
description: "Per-row import outcome.",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(csvImportResponse, "CsvImportResponseEnvelope"),
|
||||
},
|
||||
},
|
||||
},
|
||||
...stdResponses,
|
||||
"413": {
|
||||
description: "CSV exceeds the 16 MB limit.",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: dataEnvelope(z.null(), "CsvImportTooLargeEnvelope"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -28,6 +28,7 @@ import { consentPaths } from "./consent";
|
||||
import { cyclePaths } from "./cycle";
|
||||
import { devicePaths } from "./devices";
|
||||
import { healthRecordPaths } from "./health-record";
|
||||
import { importPaths } from "./import";
|
||||
import { insightsPaths } from "./insights";
|
||||
import { measurementPaths } from "./measurements";
|
||||
import { medicationPaths, medicationResource } from "./medications";
|
||||
@@ -57,6 +58,7 @@ export const openApiPaths: NonNullable<ZodOpenApiObject["paths"]> = {
|
||||
...moodPaths,
|
||||
...settingsPaths,
|
||||
...consentPaths,
|
||||
...importPaths,
|
||||
};
|
||||
|
||||
export const openApiComponents: NonNullable<ZodOpenApiObject["components"]> = {
|
||||
|
||||
Reference in new issue
Block a user