diff --git a/src/app/api/import/route.ts b/src/app/api/import/route.ts index 8931865f0..aedab2190 100644 --- a/src/app/api/import/route.ts +++ b/src/app/api/import/route.ts @@ -1,7 +1,8 @@ /** * `POST /api/import` — import user data (JSON format matching the export * structure). The route authenticates, meters and parses the file; the - * writers live in `src/lib/import/json/`. + * writers live in `src/lib/import/json/`. The app's own full backup and JSON + * export are read in their own terms first (`normalizeJsonImportFile`). */ import { apiHandler, requireAuth } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; @@ -28,6 +29,7 @@ import { } from "@/lib/validations/entry-instant"; import { isoDateTime } from "@/lib/validations/datetime"; import { fallbackMoodDateStart } from "@/lib/import/json/mood"; +import { normalizeJsonImportFile } from "@/lib/import/json/normalize"; import { runJsonImport } from "@/lib/import/json/run"; // Derived from canonical enum so round-trip export → import covers every @@ -120,7 +122,8 @@ export const POST = apiHandler(async (request: NextRequest) => { }); if (jsonError) return jsonError; - const parsed = importSchema.safeParse(body); + const normalized = normalizeJsonImportFile(body); + const parsed = importSchema.safeParse(normalized.file); if (!parsed.success) { return apiValidationError( parsed.error.issues[0]?.message ?? "Invalid import payload", @@ -129,12 +132,13 @@ export const POST = apiHandler(async (request: NextRequest) => { ); } - const { stats, writeFailed } = await runJsonImport({ + const { stats: written, writeFailed } = await runJsonImport({ userId: user.id, measurements: parsed.data.measurements, moodEntries: parsed.data.moodEntries, ipAddress: getClientIp(request), }); + const stats = { ...written, skipped: written.skipped + normalized.skipped }; if (writeFailed) { return apiError("Import write failed", 503, { diff --git a/src/lib/export/__tests__/full-backup-payload.test.ts b/src/lib/export/__tests__/full-backup-payload.test.ts index c418c854e..f0734dfb7 100644 --- a/src/lib/export/__tests__/full-backup-payload.test.ts +++ b/src/lib/export/__tests__/full-backup-payload.test.ts @@ -1002,10 +1002,14 @@ describe("buildFullBackupPayload — aggregation provenance", () => { ); const rows = payload.measurements as Array>; - // The portable arm carries the human-readable subset only — no - // externalSourceVersion, no glucoseContext, and no provenance either. + // The portable arm carries the human-readable subset only: none of the + // sync bookkeeping (source ids, their version, the provenance), but every + // attribute of the reading itself, its glucose context among them. expect(rows[0]).not.toHaveProperty("aggregationProvenance"); expect(rows[0]).not.toHaveProperty("externalSourceVersion"); + expect(rows[0]).not.toHaveProperty("externalId"); + expect(rows[0]).toHaveProperty("glucoseContext"); + expect(rows[0]).toHaveProperty("sleepStage"); }); }); diff --git a/src/lib/export/measurements-backup.ts b/src/lib/export/measurements-backup.ts index 63b17597d..1a0c63d48 100644 --- a/src/lib/export/measurements-backup.ts +++ b/src/lib/export/measurements-backup.ts @@ -91,14 +91,26 @@ function serialiseMeasurement( createdAt: measurement.createdAt.toISOString(), updatedAt: measurement.updatedAt.toISOString(), } - : { + : // The portable file is the human-readable subset: no tombstones, no + // ciphertext and none of the sync bookkeeping (the source's ids, its + // version, the aggregate's provenance). What the reading itself says + // stays: a glucose value without its fasting context, a sleep row + // without its stage or a range without its bounds reads as a + // different measurement once it is imported or restored elsewhere. + { id: measurement.id, type: measurement.type, value: measurement.value, + valueMin: measurement.valueMin, + valueMax: measurement.valueMax, unit: measurement.unit, measuredAt: measurement.measuredAt.toISOString(), source: measurement.source, notes: readNote(measurement.notesEncrypted, measurement.notes), + glucoseContext: measurement.glucoseContext, + sleepStage: measurement.sleepStage, + rhythmClassification: measurement.rhythmClassification, + deviceType: measurement.deviceType, deletedAt: measurement.deletedAt?.toISOString() ?? null, }; } diff --git a/src/lib/import/json/normalize.ts b/src/lib/import/json/normalize.ts new file mode 100644 index 000000000..a4ee9403b --- /dev/null +++ b/src/lib/import/json/normalize.ts @@ -0,0 +1,108 @@ +/** + * The JSON import reads what the app itself writes. + * + * Two of the app's files carry the import's two arrays: the full backup + * (`/api/export/full-backup`), at the top level, and the JSON export + * (`/api/export?format=json`), inside the response envelope's `data`. Neither + * was written for the import schema, and both spell an absent value the way + * their own readers expect: the backup writes `null`, the export writes an + * empty string. The import schema reads an absent value as a missing key, so + * a note, tag list, glucose context or external id the file left empty used + * to refuse the whole file. + * + * So the file is read in the exporter's terms before it is validated: + * - the envelope is opened when the arrays sit inside `data`; + * - every `null` or empty-string field of an entry is dropped, whatever the + * field is called, so a nullable column added to an export later cannot + * reopen the refusal; + * - a backup row that carries a `deletedAt` is the record of a deletion, + * not a reading, and is left out (and counted as skipped); + * - a measurement in a unit the app converts on every other way in (lb, + * °F, in, mmol/L, as the CSV import does) is converted to the stored + * unit, so a file kept in display units does not land as stored values. + * + * Validation stays where it was: the schema in the route still bounds every + * value and instant. + */ +import { resolveToCanonicalUnit } from "@/lib/measurements/unit-aliases"; + +export interface NormalizedJsonImportFile { + /** The file in the import schema's terms, ready for `safeParse`. */ + file: unknown; + /** Entries left out before validation: deletion records from a backup. */ + skipped: number; +} + +type Entry = Record; + +function isRecord(value: unknown): value is Entry { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function withoutAbsentFields(entry: Entry): Entry { + const out: Entry = {}; + for (const [key, value] of Object.entries(entry)) { + if (value === null || value === "") continue; + out[key] = value; + } + return out; +} + +function inStoredUnit(entry: Entry): Entry { + const { type, value, unit } = entry; + if ( + typeof type !== "string" || + typeof value !== "number" || + typeof unit !== "string" + ) { + return entry; + } + const canonical = resolveToCanonicalUnit(type, value, unit); + if (!canonical || canonical.unit === unit) return entry; + return { ...entry, value: canonical.value, unit: canonical.unit }; +} + +function readEntries( + value: unknown, + shape: (entry: Entry) => Entry, +): { entries: unknown; skipped: number } { + if (!Array.isArray(value)) return { entries: value, skipped: 0 }; + let skipped = 0; + const entries: unknown[] = []; + for (const raw of value) { + if (!isRecord(raw)) { + entries.push(raw); + continue; + } + if (typeof raw.deletedAt === "string" && raw.deletedAt !== "") { + skipped += 1; + continue; + } + entries.push(shape(withoutAbsentFields(raw))); + } + return { entries, skipped }; +} + +export function normalizeJsonImportFile( + body: unknown, +): NormalizedJsonImportFile { + if (!isRecord(body)) return { file: body, skipped: 0 }; + const source = + !("measurements" in body) && !("moodEntries" in body) && isRecord(body.data) + ? body.data + : body; + + const file: Entry = {}; + let skipped = 0; + if ("measurements" in source) { + const read = readEntries(source.measurements, inStoredUnit); + file.measurements = read.entries; + skipped += read.skipped; + } + if ("moodEntries" in source) { + const read = readEntries(source.moodEntries, (entry) => entry); + file.moodEntries = read.entries; + skipped += read.skipped; + } + return { file, skipped }; +} diff --git a/tests/integration/__golden__/backup-parity/export-portable.json b/tests/integration/__golden__/backup-parity/export-portable.json index b7076c404..82a5663bd 100644 --- a/tests/integration/__golden__/backup-parity/export-portable.json +++ b/tests/integration/__golden__/backup-parity/export-portable.json @@ -119,40 +119,64 @@ "id": "", "type": "BLOOD_GLUCOSE", "value": 5.4, + "valueMin": null, + "valueMax": null, "unit": "mmol/L", "measuredAt": "2026-07-03T06:45:00.000Z", "source": "MANUAL", "notes": null, + "glucoseContext": "FASTING", + "sleepStage": null, + "rhythmClassification": null, + "deviceType": null, "deletedAt": null }, { "id": "", "type": "SLEEP_DURATION", "value": 95, + "valueMin": null, + "valueMax": null, "unit": "min", "measuredAt": "2026-07-03T02:00:00.000Z", "source": "APPLE_HEALTH", "notes": null, + "glucoseContext": null, + "sleepStage": "DEEP", + "rhythmClassification": null, + "deviceType": null, "deletedAt": null }, { "id": "", "type": "WEIGHT", "value": 74.2, + "valueMin": 2.5, + "valueMax": 2.5, "unit": "kg", "measuredAt": "2026-07-01T07:00:00.000Z", "source": "MANUAL", "notes": "rt-Measurement-notesEncrypted", + "glucoseContext": null, + "sleepStage": null, + "rhythmClassification": null, + "deviceType": "rt-Measurement-deviceType", "deletedAt": null }, { "id": "", "type": "IRREGULAR_RHYTHM_NOTIFICATION", "value": 1, + "valueMin": null, + "valueMax": null, "unit": "event", "measuredAt": "2026-06-15T14:22:00.000Z", "source": "WITHINGS", "notes": null, + "glucoseContext": null, + "sleepStage": null, + "rhythmClassification": "IRREGULAR", + "deviceType": null, "deletedAt": null } ], @@ -1755,40 +1779,64 @@ "id": "", "type": "BLOOD_GLUCOSE", "value": 5.4, + "valueMin": null, + "valueMax": null, "unit": "mmol/L", "measuredAt": "2026-07-03T06:45:00.000Z", "source": "MANUAL", "notes": null, + "glucoseContext": "FASTING", + "sleepStage": null, + "rhythmClassification": null, + "deviceType": null, "deletedAt": null }, { "id": "", "type": "SLEEP_DURATION", "value": 95, + "valueMin": null, + "valueMax": null, "unit": "min", "measuredAt": "2026-07-03T02:00:00.000Z", "source": "APPLE_HEALTH", "notes": null, + "glucoseContext": null, + "sleepStage": "DEEP", + "rhythmClassification": null, + "deviceType": null, "deletedAt": null }, { "id": "", "type": "WEIGHT", "value": 74.2, + "valueMin": 2.5, + "valueMax": 2.5, "unit": "kg", "measuredAt": "2026-07-01T07:00:00.000Z", "source": "MANUAL", "notes": "rt-Measurement-notesEncrypted", + "glucoseContext": null, + "sleepStage": null, + "rhythmClassification": null, + "deviceType": "rt-Measurement-deviceType", "deletedAt": null }, { "id": "", "type": "IRREGULAR_RHYTHM_NOTIFICATION", "value": 1, + "valueMin": null, + "valueMax": null, "unit": "event", "measuredAt": "2026-06-15T14:22:00.000Z", "source": "WITHINGS", "notes": null, + "glucoseContext": null, + "sleepStage": null, + "rhythmClassification": "IRREGULAR", + "deviceType": null, "deletedAt": null } ], diff --git a/tests/integration/__golden__/backup-parity/restore-portable.json b/tests/integration/__golden__/backup-parity/restore-portable.json index 5b81b82e2..c5ca01def 100644 --- a/tests/integration/__golden__/backup-parity/restore-portable.json +++ b/tests/integration/__golden__/backup-parity/restore-portable.json @@ -513,7 +513,7 @@ "externalId": null, "externalSourceVersion": null, "aggregationProvenance": null, - "glucoseContext": null, + "glucoseContext": "FASTING", "sleepStage": null, "rhythmClassification": null, "deviceType": null, @@ -539,7 +539,7 @@ "aggregationProvenance": null, "glucoseContext": null, "sleepStage": null, - "rhythmClassification": null, + "rhythmClassification": "IRREGULAR", "deviceType": null, "syncVersion": 1, "deletedAt": null, @@ -562,7 +562,7 @@ "externalSourceVersion": null, "aggregationProvenance": null, "glucoseContext": null, - "sleepStage": null, + "sleepStage": "DEEP", "rhythmClassification": null, "deviceType": null, "syncVersion": 1, @@ -575,8 +575,8 @@ "userId": "round-trip-owner", "type": "WEIGHT", "value": 74.2, - "valueMin": null, - "valueMax": null, + "valueMin": 2.5, + "valueMax": 2.5, "unit": "kg", "source": "MANUAL", "measuredAt": "2026-07-01T07:00:00.000Z", @@ -588,7 +588,7 @@ "glucoseContext": null, "sleepStage": null, "rhythmClassification": null, - "deviceType": null, + "deviceType": "rt-Measurement-deviceType", "syncVersion": 1, "deletedAt": null, "createdAt": "", @@ -2512,40 +2512,64 @@ "id": "", "type": "BLOOD_GLUCOSE", "value": 5.4, + "valueMin": null, + "valueMax": null, "unit": "mmol/L", "measuredAt": "2026-07-03T06:45:00.000Z", "source": "MANUAL", "notes": null, + "glucoseContext": "FASTING", + "sleepStage": null, + "rhythmClassification": null, + "deviceType": null, "deletedAt": null }, { "id": "", "type": "SLEEP_DURATION", "value": 95, + "valueMin": null, + "valueMax": null, "unit": "min", "measuredAt": "2026-07-03T02:00:00.000Z", "source": "APPLE_HEALTH", "notes": null, + "glucoseContext": null, + "sleepStage": "DEEP", + "rhythmClassification": null, + "deviceType": null, "deletedAt": null }, { "id": "", "type": "WEIGHT", "value": 74.2, + "valueMin": 2.5, + "valueMax": 2.5, "unit": "kg", "measuredAt": "2026-07-01T07:00:00.000Z", "source": "MANUAL", "notes": "rt-Measurement-notesEncrypted", + "glucoseContext": null, + "sleepStage": null, + "rhythmClassification": null, + "deviceType": "rt-Measurement-deviceType", "deletedAt": null }, { "id": "", "type": "IRREGULAR_RHYTHM_NOTIFICATION", "value": 1, + "valueMin": null, + "valueMax": null, "unit": "event", "measuredAt": "2026-06-15T14:22:00.000Z", "source": "WITHINGS", "notes": null, + "glucoseContext": null, + "sleepStage": null, + "rhythmClassification": "IRREGULAR", + "deviceType": null, "deletedAt": null } ], diff --git a/tests/integration/json-import-own-exports.test.ts b/tests/integration/json-import-own-exports.test.ts new file mode 100644 index 000000000..4457c29e4 --- /dev/null +++ b/tests/integration/json-import-own-exports.test.ts @@ -0,0 +1,298 @@ +/** + * The JSON import takes the app's own files back. + * + * An account's full backup (`/api/export/full-backup`, the portable export) + * and its JSON export (`/api/export?format=json`) are written, as they are + * served, and imported into a fresh account. What the import carries (a + * measurement's type, value, unit, instant, glucose context and note; a mood + * entry's day, label, score, tags and instant) must come back as it was. + * + * Both files spell an empty field their own way, `null` in the backup and an + * empty string in the export. Before the import read files in the exporter's + * terms, the first empty note, tag list or glucose context refused the whole + * file with a 422. The portable backup also left a reading's own attributes + * out (a glucose value's fasting context among them), so the backup half + * pins those too. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { GET as exportJson } from "@/app/api/export/route"; +import { POST as importCsv } from "@/app/api/import/csv/route"; +import { GET as exportMeasurementsCsv } from "@/app/api/export/measurements/route"; +import { POST as importJson } from "@/app/api/import/route"; +import { encryptNote, readNote } from "@/lib/crypto/note-cipher"; +import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; + +import { callRoute } from "../kit/call-route"; +import { makeSession, makeUser } from "../kit/factories"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +const prisma = getPrismaClient(); + +const MEASUREMENTS = [ + { + type: "WEIGHT", + value: 81.4, + unit: "kg", + at: "2026-09-10T06:30:00.000Z", + note: "after the run", + }, + { + type: "WEIGHT", + value: 81.9, + unit: "kg", + at: "2026-09-11T06:30:00.000Z", + note: null, + }, + { + type: "PULSE", + value: 62, + unit: "bpm", + at: "2026-09-11T07:00:00.000Z", + note: null, + }, + { + type: "BLOOD_GLUCOSE", + value: 104, + unit: "mg/dL", + at: "2026-09-12T08:00:00.000Z", + note: null, + glucoseContext: "FASTING" as const, + }, + { + type: "BLOOD_GLUCOSE", + value: 141, + unit: "mg/dL", + at: "2026-09-12T13:00:00.000Z", + note: "after lunch", + glucoseContext: "POSTPRANDIAL" as const, + }, +] as const; + +const MOOD = [ + { + date: "2026-09-10", + mood: "GUT", + score: 4, + tags: null, + at: "2026-09-10T19:00:00.000Z", + }, + { + date: "2026-09-11", + mood: "OKAY", + score: 3, + tags: JSON.stringify(["work", "sleep"]), + at: "2026-09-11T20:30:00.000Z", + }, + { + date: "2026-09-12", + mood: "SUPER_GUT", + score: 5, + tags: null, + at: "2026-09-12T21:15:00.000Z", + }, +] as const; + +let ownerId: string; + +async function seedOwner(): Promise { + const owner = await makeUser({ + timezone: "Europe/Berlin", + glucoseUnit: "mmol/L" /* the JSON export stays in the stored unit */, + }); + ownerId = owner.id; + for (const m of MEASUREMENTS) { + await prisma.measurement.create({ + data: { + userId: ownerId, + type: m.type, + value: m.value, + unit: m.unit, + measuredAt: new Date(m.at), + source: "MANUAL", + notes: null, + notesEncrypted: encryptNote(m.note), + glucoseContext: "glucoseContext" in m ? m.glucoseContext : null, + }, + }); + } + // A reading deleted before the backup: its row is the record of a + // deletion, and the import leaves it out. + await prisma.measurement.create({ + data: { + userId: ownerId, + type: "WEIGHT", + value: 90, + unit: "kg", + measuredAt: new Date("2026-09-09T06:30:00.000Z"), + source: "MANUAL", + deletedAt: new Date("2026-09-09T07:00:00.000Z"), + }, + }); + for (const e of MOOD) { + await prisma.moodEntry.create({ + data: { + userId: ownerId, + date: e.date, + mood: e.mood, + score: e.score, + tags: e.tags, + source: "MANUAL", + moodLoggedAt: new Date(e.at), + }, + }); + } +} + +/** What an import carries, read back from an account. */ +async function carried(userId: string) { + const measurements = await prisma.measurement.findMany({ + where: { userId, deletedAt: null }, + orderBy: [{ measuredAt: "asc" }, { type: "asc" }], + }); + const mood = await prisma.moodEntry.findMany({ + where: { userId, deletedAt: null }, + orderBy: { moodLoggedAt: "asc" }, + }); + return { + measurements: measurements.map((m) => ({ + type: m.type, + value: m.value, + unit: m.unit, + measuredAt: m.measuredAt.toISOString(), + glucoseContext: m.glucoseContext, + note: readNote(m.notesEncrypted, m.notes), + })), + mood: mood.map((e) => ({ + date: e.date, + mood: e.mood, + score: e.score, + tags: e.tags, + loggedAt: e.moodLoggedAt.toISOString(), + })), + }; +} + +async function importInto(file: unknown) { + const user = await makeUser({ timezone: "Europe/Berlin" }); + const res = await callRoute<{ + data: { measurements: number; moodEntries: number; skipped: number }; + }>(importJson, { + method: "POST", + route: "/api/import", + body: file, + auth: await makeSession(user), + }); + return { userId: user.id, res }; +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await seedOwner(); +}); + +afterAll(async () => { + await truncateAllTables(prisma); +}); + +describe("the JSON import reads the app's own files", () => { + it("takes a full backup back, nulls and deletion records included", async () => { + let text = ""; + await streamFullBackupJson(prisma, ownerId, (chunk) => { + text += chunk; + }); + const backup = JSON.parse(text) as { + measurements: Array>; + moodEntries: Array>; + }; + // The file carries the shapes the import used to refuse. + expect(backup.measurements.some((m) => m.notes === null)).toBe(true); + expect(backup.moodEntries.some((e) => e.tags === null)).toBe(true); + + const { userId, res } = await importInto(backup); + expect(res.status).toBe(200); + const deletedInFile = backup.measurements.filter( + (m) => typeof m.deletedAt === "string", + ).length; + expect(res.data).toEqual({ + measurements: MEASUREMENTS.length, + moodEntries: MOOD.length, + skipped: deletedInFile, + }); + + expect(await carried(userId)).toEqual(await carried(ownerId)); + }); + + it("takes the JSON export back, empty strings and its envelope included", async () => { + const owner = await prisma.user.findUniqueOrThrow({ + where: { id: ownerId }, + }); + const exported = await callRoute(exportJson, { + method: "GET", + route: "/api/export", + query: { format: "json", type: "all" }, + auth: await makeSession(owner), + }); + expect(exported.status).toBe(200); + const file = exported.body as { + data: { + measurements: Array>; + moodEntries: Array>; + }; + }; + expect(file.data.measurements.some((m) => m.notes === "")).toBe(true); + expect(file.data.moodEntries.some((e) => e.tags === "")).toBe(true); + + const { userId, res } = await importInto(file); + expect(res.status).toBe(200); + expect(res.data).toMatchObject({ + measurements: MEASUREMENTS.length, + moodEntries: MOOD.length, + }); + + expect(await carried(userId)).toEqual(await carried(ownerId)); + }); + + it("takes the measurement CSV back into the CSV import, in display units too", async () => { + const owner = await prisma.user.findUniqueOrThrow({ + where: { id: ownerId }, + }); + const exported = await callRoute(exportMeasurementsCsv, { + method: "GET", + route: "/api/export/measurements", + auth: await makeSession(owner), + }); + expect(exported.status).toBe(200); + const csv = String(exported.body); + // The account reads glucose in mmol/L, so that is what the file says. + expect(csv).toMatch(/BLOOD_GLUCOSE,\d+(\.\d)?,mmol\/L/); + + const user = await makeUser({ timezone: "Europe/Berlin" }); + const res = await callRoute<{ inserted: number; skipped: number }>( + importCsv, + { + method: "POST", + route: "/api/import/csv", + text: csv, + headers: { "content-type": "text/csv" }, + auth: await makeSession(user), + }, + ); + expect(res.status).toBe(200); + expect(res.data).toMatchObject({ + inserted: MEASUREMENTS.length, + skipped: 0, + }); + + const back = (await carried(user.id)).measurements; + const original = (await carried(ownerId)).measurements; + expect(back).toHaveLength(original.length); + back.forEach((m, i) => { + const o = original[i]!; + // mmol/L at one decimal: the stored mg/dL value comes back within it. + const tolerance = m.type === "BLOOD_GLUCOSE" ? 0.05 * 18.0156 : 1e-9; + expect({ ...m, value: 0 }).toEqual({ ...o, value: 0 }); + expect(Math.abs(m.value - o.value)).toBeLessThanOrEqual(tolerance); + }); + }); +});