feat(import): add CSV measurement import with per-row status and dry-run

A self-hoster migrating from a spreadsheet or a meter export overwhelmingly
has CSV, which neither the JSON nor the Apple Health import covers. Add
`POST /api/import/csv` reusing the existing write loop and rollup re-fold.

The documented header is `type,value,unit,measuredAt[,glucoseContext,notes,
externalId]`, order-independent. measuredAt must carry an explicit ISO-8601
offset — a row without one is skipped rather than guessed — and is bounded by
the shared entry-instant validator. Glucose accepts mmol/L (converted to
canonical mg/dL) and weight accepts lb (converted to kg); any other
non-canonical unit is skipped, never mis-stored. An optional externalId
column makes a re-upload idempotent; without it a re-upload duplicates.

The response carries a per-row status envelope so a 2000-row file with three
bad rows imports 1997 and names the three. `?dryRun=1` validates and previews
the outcome without writing. The parse, validation, and unit conversion live
in a pure module so they are unit-tested without a database.

Settings -> Export & Import gains a third CSV card mirroring the JSON and
Apple Health cards, with upload, paste, preview, and a downloadable example.
The new route is documented in the OpenAPI contract.
This commit is contained in:
Marc-André Bombeck committed 2026-06-14 23:28:22 +02:00
1 parent 82e2d7dfb1
commit fbdcb31202
15 files changed
+1666 -2

No files matched your search

@@ -0,0 +1,189 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
vi.mock("@/lib/api-handler", () => ({
apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
requireAuth: vi.fn(async () => ({
user: { id: "u-1" },
session: { id: "s-1" },
})),
}));
vi.mock("@/lib/db", () => ({
prisma: {
measurement: { create: vi.fn(), upsert: vi.fn() },
},
}));
vi.mock("@/lib/rate-limit", () => ({
checkRateLimit: vi.fn(),
}));
vi.mock("@/lib/auth/audit", () => ({
auditLog: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/logging/context", () => ({
annotate: vi.fn(),
}));
vi.mock("@/lib/rollups/measurement-rollups", async () => {
const actual = await vi.importActual<
typeof import("@/lib/rollups/measurement-rollups")
>("@/lib/rollups/measurement-rollups");
return {
...actual,
recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined),
};
});
import { NextRequest } from "next/server";
import { POST } from "../route";
import { checkRateLimit } from "@/lib/rate-limit";
import { prisma } from "@/lib/db";
import { recomputeBucketsForMeasurement } from "@/lib/rollups/measurement-rollups";
const HEADER = "type,value,unit,measuredAt,glucoseContext,notes,externalId";
function csvRequest(csv: string, dryRun = false) {
return new NextRequest(
`http://localhost/api/import/csv${dryRun ? "?dryRun=1" : ""}`,
{
method: "POST",
body: csv,
headers: { "content-type": "text/csv" },
},
);
}
interface CsvEnvelope {
data: {
inserted: number;
updated: number;
skipped: number;
total: number;
dryRun: boolean;
rows: Array<{ line: number; status: string; reason?: string }>;
} | null;
error: string | null;
}
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(checkRateLimit).mockResolvedValue({
allowed: true,
remaining: 4,
resetAt: new Date(Date.now() + 1000),
} as never);
});
describe("POST /api/import/csv — rate limit", () => {
it("returns 429 against the shared import bucket when exhausted", async () => {
vi.mocked(checkRateLimit).mockResolvedValue({
allowed: false,
remaining: 0,
resetAt: new Date(Date.now() + 1000),
} as never);
const res = await POST(csvRequest([HEADER, ""].join("\n")));
expect(res.status).toBe(429);
expect(checkRateLimit).toHaveBeenCalledWith(
expect.stringContaining("import:u-1"),
5,
60 * 60 * 1000,
);
});
});
describe("POST /api/import/csv — fatal header error", () => {
it("returns 422 when a required column is missing", async () => {
const res = await POST(csvRequest("type,value,unit\nWEIGHT,80,kg"));
expect(res.status).toBe(422);
expect(vi.mocked(prisma.measurement.create)).not.toHaveBeenCalled();
});
});
describe("POST /api/import/csv — write loop + per-row envelope", () => {
it("inserts valid rows, skips invalid ones, returns per-row status", async () => {
vi.mocked(prisma.measurement.create).mockResolvedValue({} as never);
const res = await POST(
csvRequest(
[
HEADER,
"WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,morning,", // inserted
"NOPE,1,kg,2026-05-01T08:00:00Z,,,", // skipped unknown_type
"WEIGHT,80,kg,2026-05-01T08:00:00,,,", // skipped missing offset
].join("\n"),
),
);
expect(res.status).toBe(200);
const body = (await res.json()) as CsvEnvelope;
expect(body.data?.inserted).toBe(1);
expect(body.data?.skipped).toBe(2);
expect(body.data?.total).toBe(3);
expect(body.data?.dryRun).toBe(false);
const reasons = body.data?.rows.map((r) => r.reason);
expect(reasons).toContain("unknown_type");
expect(reasons).toContain("missing_timezone_offset");
// One create call for the single valid row.
expect(vi.mocked(prisma.measurement.create)).toHaveBeenCalledTimes(1);
// Rollup re-fold fired for the touched (type, day).
expect(vi.mocked(recomputeBucketsForMeasurement)).toHaveBeenCalled();
});
it("dryRun previews without writing and reports projected inserts", async () => {
const res = await POST(
csvRequest(
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,"].join("\n"),
true,
),
);
expect(res.status).toBe(200);
const body = (await res.json()) as CsvEnvelope;
expect(body.data?.dryRun).toBe(true);
expect(body.data?.inserted).toBe(1);
expect(body.data?.rows[0].status).toBe("inserted");
expect(vi.mocked(prisma.measurement.create)).not.toHaveBeenCalled();
expect(vi.mocked(recomputeBucketsForMeasurement)).not.toHaveBeenCalled();
});
it("upserts an externalId row and surfaces it as updated when it already existed", async () => {
const created = new Date("2026-05-01T08:00:00Z");
const bumped = new Date("2026-05-02T08:00:00Z");
vi.mocked(prisma.measurement.upsert).mockResolvedValue({
createdAt: created,
updatedAt: bumped,
} as never);
const res = await POST(
csvRequest(
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,ext-1"].join("\n"),
),
);
const body = (await res.json()) as CsvEnvelope;
expect(body.data?.updated).toBe(1);
expect(body.data?.inserted).toBe(0);
expect(body.data?.rows[0].status).toBe("updated");
expect(vi.mocked(prisma.measurement.upsert)).toHaveBeenCalledTimes(1);
});
it("counts a unique-constraint duplicate as skipped/duplicate", async () => {
vi.mocked(prisma.measurement.create).mockRejectedValue(
new Error("Unique constraint failed"),
);
const res = await POST(
csvRequest(
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,"].join("\n"),
),
);
const body = (await res.json()) as CsvEnvelope;
expect(body.data?.inserted).toBe(0);
expect(body.data?.skipped).toBe(1);
expect(body.data?.rows[0]).toMatchObject({
status: "skipped",
reason: "duplicate",
});
});
});
+254
View File
@@ -0,0 +1,254 @@
import { prisma } from "@/lib/db";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { annotate } from "@/lib/logging/context";
import { auditLog } from "@/lib/auth/audit";
import { apiSuccess, apiError, getClientIp } from "@/lib/api-response";
import { checkRateLimit } from "@/lib/rate-limit";
import { NextRequest } from "next/server";
import {
parseCsvMeasurements,
type CsvRowResult,
} from "@/lib/import/csv-measurements";
import {
collapseToTypeDayKeys,
recomputeBucketsForMeasurement,
} from "@/lib/rollups/measurement-rollups";
import type {
MeasurementType,
GlucoseContext,
} from "@/generated/prisma/client";
/**
* CSV measurement import (v1.17.1).
*
* The cold-start escape hatch for self-hosters migrating from a spreadsheet
* or another tracker. Reuses the `/api/import` write loop + rollup re-fold;
* the parse + per-row validation + unit conversion lives in the pure
* `parseCsvMeasurements` module so it is unit-tested without a database.
*
* Body is `text/csv` (or `text/plain`) raw text — NOT JSON. Capped at 16 MB
* (the same ceiling as the JSON import; a 16 MB CSV is > 100 000 rows). Rows
* beyond the 10 000 ceiling are rejected so the rollup re-fold stays bounded.
*
* `?dryRun=1` parses + validates + returns the per-row status envelope
* WITHOUT writing — a trust win for a bulk cold-start import.
*
* Response: `{ inserted, updated, skipped, total, dryRun, rows: [{line,
* status, reason}] }`, mirroring the batch route's per-entry envelope.
*/
const MAX_CSV_BYTES = 16 * 1024 * 1024;
const MAX_ROWS = 10000;
export const POST = apiHandler(async (request: NextRequest) => {
const { user } = await requireAuth();
annotate({ action: { name: "import.csv.upload" } });
const dryRun =
new URL(request.url).searchParams.get("dryRun") === "1" ||
new URL(request.url).searchParams.get("dryRun") === "true";
// Same blast-radius reasoning as the JSON import — share the bucket so a
// CSV upload counts against the same 5/hour quota.
const rl = await checkRateLimit(`import:${user.id}`, 5, 60 * 60 * 1000);
if (!rl.allowed) {
return apiError("Maximum 5 imports per hour", 429);
}
// Raw text read with an explicit byte cap — `safeJson` is JSON-only, so we
// enforce the same `maxBytes` ceiling here against the Content-Length and
// the materialised body.
const declaredLength = Number(request.headers.get("content-length") ?? "0");
if (Number.isFinite(declaredLength) && declaredLength > MAX_CSV_BYTES) {
return apiError("CSV exceeds the 16 MB limit", 413);
}
let text: string;
try {
text = await request.text();
} catch {
return apiError("Could not read the request body", 400);
}
if (Buffer.byteLength(text, "utf8") > MAX_CSV_BYTES) {
return apiError("CSV exceeds the 16 MB limit", 413);
}
const parsed = parseCsvMeasurements(text);
if (parsed.fatal) {
return apiError(parsed.fatal.message, 422);
}
const okRows = parsed.rows.filter(
(r): r is CsvRowResult & { row: NonNullable<CsvRowResult["row"]> } =>
r.status === "ok" && r.row !== undefined,
);
if (okRows.length > MAX_ROWS) {
return apiError(
`CSV has ${okRows.length} valid rows — the limit is ${MAX_ROWS} per import`,
422,
);
}
const userId = user.id;
let inserted = 0;
let updated = 0;
let skipped = parsed.rows.filter((r) => r.status === "skipped").length;
const touchedMeasurements: Array<{
type: MeasurementType;
measuredAt: Date;
}> = [];
// Per-line write outcome for rows that reached the DB. Keyed by source line
// so the response envelope can surface `inserted` / `updated` / `duplicate`
// distinctly without polluting the parse-time `CsvRowReason`.
const writeOutcome = new Map<number, "inserted" | "updated" | "duplicate">();
if (!dryRun) {
for (const result of okRows) {
const m = result.row;
try {
if (m.externalId) {
// Idempotent re-import keyed on the source-stable id — re-uploading
// the same file updates in place rather than minting duplicates.
const res = await prisma.measurement.upsert({
where: {
userId_type_source_externalId: {
userId,
type: m.type as MeasurementType,
source: "IMPORT",
externalId: m.externalId,
},
},
update: {
value: m.value,
unit: m.unit,
measuredAt: m.measuredAt,
notes: m.notes ?? null,
glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null,
},
create: {
userId,
type: m.type as MeasurementType,
value: m.value,
unit: m.unit,
source: "IMPORT",
externalId: m.externalId,
measuredAt: m.measuredAt,
notes: m.notes ?? null,
glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null,
},
select: { createdAt: true, updatedAt: true },
});
// A fresh create has createdAt === updatedAt; an in-place update
// bumps updatedAt past createdAt.
if (res.updatedAt.getTime() > res.createdAt.getTime()) {
updated++;
writeOutcome.set(result.line, "updated");
} else {
inserted++;
writeOutcome.set(result.line, "inserted");
}
} else {
await prisma.measurement.create({
data: {
userId,
type: m.type as MeasurementType,
value: m.value,
unit: m.unit,
source: "IMPORT",
measuredAt: m.measuredAt,
notes: m.notes ?? null,
glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null,
},
});
inserted++;
writeOutcome.set(result.line, "inserted");
}
touchedMeasurements.push({
type: m.type as MeasurementType,
measuredAt: m.measuredAt,
});
} catch {
// Unique-constraint violation (a NULL-externalId duplicate against an
// existing row) — count as skipped rather than failing the file.
skipped++;
writeOutcome.set(result.line, "duplicate");
}
}
// One bounded rollup re-fold per touched (type, day) — a 10 000-row CSV
// pays at most ~N (type, day) recomputes, not 10 000 per-row hooks.
// Best-effort: a populator hiccup never fails the importer.
if (touchedMeasurements.length > 0) {
try {
const keys = collapseToTypeDayKeys(touchedMeasurements);
for (const k of keys) {
await recomputeBucketsForMeasurement(userId, k.type, k.measuredAt);
}
} catch (err) {
annotate({
meta: {
measurement_rollup_csv_import_failed: true,
measurement_rollup_csv_import_error:
err instanceof Error ? err.message : String(err),
},
});
}
}
await auditLog("import.csv.upload", {
userId,
ipAddress: getClientIp(request),
details: { inserted, updated, skipped },
});
} else {
// Dry-run preview: no write ran, so report the projected outcome —
// every valid parse-`ok` row would insert (an externalId row could land
// as an update, but that needs a DB read we deliberately skip here).
inserted = okRows.length;
}
const summary = {
inserted,
updated,
skipped,
total: parsed.rows.length,
dryRun,
// Per-row status, mirroring the batch route's per-entry envelope. The
// normalised payload is stripped — the client only needs line + status.
// Parse-time `ok` rows are refined by the write outcome: a row that hit a
// unique-constraint duplicate becomes `skipped`/`duplicate`; an upsert
// that landed on an existing row becomes `updated`. In `dryRun` mode no
// write ran, so every parse-`ok` row stays `inserted` (the projected
// outcome).
rows: parsed.rows.map((r) => {
if (r.status === "skipped") {
return {
line: r.line,
status: "skipped" as const,
...(r.reason ? { reason: r.reason } : {}),
};
}
const outcome = writeOutcome.get(r.line);
if (outcome === "duplicate") {
return { line: r.line, status: "skipped" as const, reason: "duplicate" };
}
if (outcome === "updated") {
return { line: r.line, status: "updated" as const };
}
return { line: r.line, status: "inserted" as const };
}),
};
annotate({
meta: {
import_csv_inserted: inserted,
import_csv_updated: updated,
import_csv_skipped: skipped,
import_csv_dry_run: dryRun,
},
});
return apiSuccess(summary);
});