fix(import): take the app's own full backup and JSON export back

The JSON import refused the app's own files: the full backup writes an
empty note, tag list or glucose context as null and the JSON export as
an empty string, and the import schema read either as a value of the
wrong type. The file is now read in the exporter's terms before it is
validated: the export's envelope is opened, every null or empty field
is treated as absent, a backup row carrying a deletion is left out and
counted as skipped, and a unit the app converts on every other way in
is converted to the stored one.

The round trip also showed the portable backup dropping what a reading
itself says: glucose context, sleep stage, range bounds, rhythm and
device. They ride the portable file now, and a portable restore puts
them back. Sync bookkeeping stays out.

An integration test exports an account three ways (full backup, JSON
export, measurement CSV), imports each into a fresh account and compares
what the import carries.
This commit is contained in:
Marc-André Bombeck committed 2026-10-23 17:33:58 +02:00
1 parent 8a11b7bfc6
commit 3d26cee566
7 files changed
+510 -12

No files matched your search

+7 -3
View File
@@ -1,7 +1,8 @@
/**
* `POST /api/import` — import user data (JSON format matching the export
* structure). The route authenticates, meters and parses the file; the
* writers live in `src/lib/import/json/`.
* writers live in `src/lib/import/json/`. The app's own full backup and JSON
* export are read in their own terms first (`normalizeJsonImportFile`).
*/
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { annotate } from "@/lib/logging/context";
@@ -28,6 +29,7 @@ import {
} from "@/lib/validations/entry-instant";
import { isoDateTime } from "@/lib/validations/datetime";
import { fallbackMoodDateStart } from "@/lib/import/json/mood";
import { normalizeJsonImportFile } from "@/lib/import/json/normalize";
import { runJsonImport } from "@/lib/import/json/run";
// Derived from canonical enum so round-trip export → import covers every
@@ -120,7 +122,8 @@ export const POST = apiHandler(async (request: NextRequest) => {
});
if (jsonError) return jsonError;
const parsed = importSchema.safeParse(body);
const normalized = normalizeJsonImportFile(body);
const parsed = importSchema.safeParse(normalized.file);
if (!parsed.success) {
return apiValidationError(
parsed.error.issues[0]?.message ?? "Invalid import payload",
@@ -129,12 +132,13 @@ export const POST = apiHandler(async (request: NextRequest) => {
);
}
const { stats, writeFailed } = await runJsonImport({
const { stats: written, writeFailed } = await runJsonImport({
userId: user.id,
measurements: parsed.data.measurements,
moodEntries: parsed.data.moodEntries,
ipAddress: getClientIp(request),
});
const stats = { ...written, skipped: written.skipped + normalized.skipped };
if (writeFailed) {
return apiError("Import write failed", 503, {
@@ -1002,10 +1002,14 @@ describe("buildFullBackupPayload — aggregation provenance", () => {
);
const rows = payload.measurements as Array<Record<string, unknown>>;
// The portable arm carries the human-readable subset only — no
// externalSourceVersion, no glucoseContext, and no provenance either.
// The portable arm carries the human-readable subset only: none of the
// sync bookkeeping (source ids, their version, the provenance), but every
// attribute of the reading itself, its glucose context among them.
expect(rows[0]).not.toHaveProperty("aggregationProvenance");
expect(rows[0]).not.toHaveProperty("externalSourceVersion");
expect(rows[0]).not.toHaveProperty("externalId");
expect(rows[0]).toHaveProperty("glucoseContext");
expect(rows[0]).toHaveProperty("sleepStage");
});
});
+13 -1
View File
@@ -91,14 +91,26 @@ function serialiseMeasurement(
createdAt: measurement.createdAt.toISOString(),
updatedAt: measurement.updatedAt.toISOString(),
}
: {
: // The portable file is the human-readable subset: no tombstones, no
// ciphertext and none of the sync bookkeeping (the source's ids, its
// version, the aggregate's provenance). What the reading itself says
// stays: a glucose value without its fasting context, a sleep row
// without its stage or a range without its bounds reads as a
// different measurement once it is imported or restored elsewhere.
{
id: measurement.id,
type: measurement.type,
value: measurement.value,
valueMin: measurement.valueMin,
valueMax: measurement.valueMax,
unit: measurement.unit,
measuredAt: measurement.measuredAt.toISOString(),
source: measurement.source,
notes: readNote(measurement.notesEncrypted, measurement.notes),
glucoseContext: measurement.glucoseContext,
sleepStage: measurement.sleepStage,
rhythmClassification: measurement.rhythmClassification,
deviceType: measurement.deviceType,
deletedAt: measurement.deletedAt?.toISOString() ?? null,
};
}
+108
View File
@@ -0,0 +1,108 @@
/**
* The JSON import reads what the app itself writes.
*
* Two of the app's files carry the import's two arrays: the full backup
* (`/api/export/full-backup`), at the top level, and the JSON export
* (`/api/export?format=json`), inside the response envelope's `data`. Neither
* was written for the import schema, and both spell an absent value the way
* their own readers expect: the backup writes `null`, the export writes an
* empty string. The import schema reads an absent value as a missing key, so
* a note, tag list, glucose context or external id the file left empty used
* to refuse the whole file.
*
* So the file is read in the exporter's terms before it is validated:
* - the envelope is opened when the arrays sit inside `data`;
* - every `null` or empty-string field of an entry is dropped, whatever the
* field is called, so a nullable column added to an export later cannot
* reopen the refusal;
* - a backup row that carries a `deletedAt` is the record of a deletion,
* not a reading, and is left out (and counted as skipped);
* - a measurement in a unit the app converts on every other way in (lb,
* °F, in, mmol/L, as the CSV import does) is converted to the stored
* unit, so a file kept in display units does not land as stored values.
*
* Validation stays where it was: the schema in the route still bounds every
* value and instant.
*/
import { resolveToCanonicalUnit } from "@/lib/measurements/unit-aliases";
export interface NormalizedJsonImportFile {
/** The file in the import schema's terms, ready for `safeParse`. */
file: unknown;
/** Entries left out before validation: deletion records from a backup. */
skipped: number;
}
type Entry = Record<string, unknown>;
function isRecord(value: unknown): value is Entry {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function withoutAbsentFields(entry: Entry): Entry {
const out: Entry = {};
for (const [key, value] of Object.entries(entry)) {
if (value === null || value === "") continue;
out[key] = value;
}
return out;
}
function inStoredUnit(entry: Entry): Entry {
const { type, value, unit } = entry;
if (
typeof type !== "string" ||
typeof value !== "number" ||
typeof unit !== "string"
) {
return entry;
}
const canonical = resolveToCanonicalUnit(type, value, unit);
if (!canonical || canonical.unit === unit) return entry;
return { ...entry, value: canonical.value, unit: canonical.unit };
}
function readEntries(
value: unknown,
shape: (entry: Entry) => Entry,
): { entries: unknown; skipped: number } {
if (!Array.isArray(value)) return { entries: value, skipped: 0 };
let skipped = 0;
const entries: unknown[] = [];
for (const raw of value) {
if (!isRecord(raw)) {
entries.push(raw);
continue;
}
if (typeof raw.deletedAt === "string" && raw.deletedAt !== "") {
skipped += 1;
continue;
}
entries.push(shape(withoutAbsentFields(raw)));
}
return { entries, skipped };
}
export function normalizeJsonImportFile(
body: unknown,
): NormalizedJsonImportFile {
if (!isRecord(body)) return { file: body, skipped: 0 };
const source =
!("measurements" in body) && !("moodEntries" in body) && isRecord(body.data)
? body.data
: body;
const file: Entry = {};
let skipped = 0;
if ("measurements" in source) {
const read = readEntries(source.measurements, inStoredUnit);
file.measurements = read.entries;
skipped += read.skipped;
}
if ("moodEntries" in source) {
const read = readEntries(source.moodEntries, (entry) => entry);
file.moodEntries = read.entries;
skipped += read.skipped;
}
return { file, skipped };
}
@@ -119,40 +119,64 @@
"id": "<id:1>",
"type": "BLOOD_GLUCOSE",
"value": 5.4,
"valueMin": null,
"valueMax": null,
"unit": "mmol/L",
"measuredAt": "2026-07-03T06:45:00.000Z",
"source": "MANUAL",
"notes": null,
"glucoseContext": "FASTING",
"sleepStage": null,
"rhythmClassification": null,
"deviceType": null,
"deletedAt": null
},
{
"id": "<id:2>",
"type": "SLEEP_DURATION",
"value": 95,
"valueMin": null,
"valueMax": null,
"unit": "min",
"measuredAt": "2026-07-03T02:00:00.000Z",
"source": "APPLE_HEALTH",
"notes": null,
"glucoseContext": null,
"sleepStage": "DEEP",
"rhythmClassification": null,
"deviceType": null,
"deletedAt": null
},
{
"id": "<id:3>",
"type": "WEIGHT",
"value": 74.2,
"valueMin": 2.5,
"valueMax": 2.5,
"unit": "kg",
"measuredAt": "2026-07-01T07:00:00.000Z",
"source": "MANUAL",
"notes": "rt-Measurement-notesEncrypted",
"glucoseContext": null,
"sleepStage": null,
"rhythmClassification": null,
"deviceType": "rt-Measurement-deviceType",
"deletedAt": null
},
{
"id": "<id:4>",
"type": "IRREGULAR_RHYTHM_NOTIFICATION",
"value": 1,
"valueMin": null,
"valueMax": null,
"unit": "event",
"measuredAt": "2026-06-15T14:22:00.000Z",
"source": "WITHINGS",
"notes": null,
"glucoseContext": null,
"sleepStage": null,
"rhythmClassification": "IRREGULAR",
"deviceType": null,
"deletedAt": null
}
],
@@ -1755,40 +1779,64 @@
"id": "<id:1>",
"type": "BLOOD_GLUCOSE",
"value": 5.4,
"valueMin": null,
"valueMax": null,
"unit": "mmol/L",
"measuredAt": "2026-07-03T06:45:00.000Z",
"source": "MANUAL",
"notes": null,
"glucoseContext": "FASTING",
"sleepStage": null,
"rhythmClassification": null,
"deviceType": null,
"deletedAt": null
},
{
"id": "<id:2>",
"type": "SLEEP_DURATION",
"value": 95,
"valueMin": null,
"valueMax": null,
"unit": "min",
"measuredAt": "2026-07-03T02:00:00.000Z",
"source": "APPLE_HEALTH",
"notes": null,
"glucoseContext": null,
"sleepStage": "DEEP",
"rhythmClassification": null,
"deviceType": null,
"deletedAt": null
},
{
"id": "<id:3>",
"type": "WEIGHT",
"value": 74.2,
"valueMin": 2.5,
"valueMax": 2.5,
"unit": "kg",
"measuredAt": "2026-07-01T07:00:00.000Z",
"source": "MANUAL",
"notes": "rt-Measurement-notesEncrypted",
"glucoseContext": null,
"sleepStage": null,
"rhythmClassification": null,
"deviceType": "rt-Measurement-deviceType",
"deletedAt": null
},
{
"id": "<id:4>",
"type": "IRREGULAR_RHYTHM_NOTIFICATION",
"value": 1,
"valueMin": null,
"valueMax": null,
"unit": "event",
"measuredAt": "2026-06-15T14:22:00.000Z",
"source": "WITHINGS",
"notes": null,
"glucoseContext": null,
"sleepStage": null,
"rhythmClassification": "IRREGULAR",
"deviceType": null,
"deletedAt": null
}
],
@@ -513,7 +513,7 @@
"externalId": null,
"externalSourceVersion": null,
"aggregationProvenance": null,
"glucoseContext": null,
"glucoseContext": "FASTING",
"sleepStage": null,
"rhythmClassification": null,
"deviceType": null,
@@ -539,7 +539,7 @@
"aggregationProvenance": null,
"glucoseContext": null,
"sleepStage": null,
"rhythmClassification": null,
"rhythmClassification": "IRREGULAR",
"deviceType": null,
"syncVersion": 1,
"deletedAt": null,
@@ -562,7 +562,7 @@
"externalSourceVersion": null,
"aggregationProvenance": null,
"glucoseContext": null,
"sleepStage": null,
"sleepStage": "DEEP",
"rhythmClassification": null,
"deviceType": null,
"syncVersion": 1,
@@ -575,8 +575,8 @@
"userId": "round-trip-owner",
"type": "WEIGHT",
"value": 74.2,
"valueMin": null,
"valueMax": null,
"valueMin": 2.5,
"valueMax": 2.5,
"unit": "kg",
"source": "MANUAL",
"measuredAt": "2026-07-01T07:00:00.000Z",
@@ -588,7 +588,7 @@
"glucoseContext": null,
"sleepStage": null,
"rhythmClassification": null,
"deviceType": null,
"deviceType": "rt-Measurement-deviceType",
"syncVersion": 1,
"deletedAt": null,
"createdAt": "<now>",
@@ -2512,40 +2512,64 @@
"id": "<id:1>",
"type": "BLOOD_GLUCOSE",
"value": 5.4,
"valueMin": null,
"valueMax": null,
"unit": "mmol/L",
"measuredAt": "2026-07-03T06:45:00.000Z",
"source": "MANUAL",
"notes": null,
"glucoseContext": "FASTING",
"sleepStage": null,
"rhythmClassification": null,
"deviceType": null,
"deletedAt": null
},
{
"id": "<id:2>",
"type": "SLEEP_DURATION",
"value": 95,
"valueMin": null,
"valueMax": null,
"unit": "min",
"measuredAt": "2026-07-03T02:00:00.000Z",
"source": "APPLE_HEALTH",
"notes": null,
"glucoseContext": null,
"sleepStage": "DEEP",
"rhythmClassification": null,
"deviceType": null,
"deletedAt": null
},
{
"id": "<id:3>",
"type": "WEIGHT",
"value": 74.2,
"valueMin": 2.5,
"valueMax": 2.5,
"unit": "kg",
"measuredAt": "2026-07-01T07:00:00.000Z",
"source": "MANUAL",
"notes": "rt-Measurement-notesEncrypted",
"glucoseContext": null,
"sleepStage": null,
"rhythmClassification": null,
"deviceType": "rt-Measurement-deviceType",
"deletedAt": null
},
{
"id": "<id:4>",
"type": "IRREGULAR_RHYTHM_NOTIFICATION",
"value": 1,
"valueMin": null,
"valueMax": null,
"unit": "event",
"measuredAt": "2026-06-15T14:22:00.000Z",
"source": "WITHINGS",
"notes": null,
"glucoseContext": null,
"sleepStage": null,
"rhythmClassification": "IRREGULAR",
"deviceType": null,
"deletedAt": null
}
],
@@ -0,0 +1,298 @@
/**
* The JSON import takes the app's own files back.
*
* An account's full backup (`/api/export/full-backup`, the portable export)
* and its JSON export (`/api/export?format=json`) are written, as they are
* served, and imported into a fresh account. What the import carries (a
* measurement's type, value, unit, instant, glucose context and note; a mood
* entry's day, label, score, tags and instant) must come back as it was.
*
* Both files spell an empty field their own way, `null` in the backup and an
* empty string in the export. Before the import read files in the exporter's
* terms, the first empty note, tag list or glucose context refused the whole
* file with a 422. The portable backup also left a reading's own attributes
* out (a glucose value's fasting context among them), so the backup half
* pins those too.
*/
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { GET as exportJson } from "@/app/api/export/route";
import { POST as importCsv } from "@/app/api/import/csv/route";
import { GET as exportMeasurementsCsv } from "@/app/api/export/measurements/route";
import { POST as importJson } from "@/app/api/import/route";
import { encryptNote, readNote } from "@/lib/crypto/note-cipher";
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
import { callRoute } from "../kit/call-route";
import { makeSession, makeUser } from "../kit/factories";
import { getPrismaClient, truncateAllTables } from "./setup";
const prisma = getPrismaClient();
const MEASUREMENTS = [
{
type: "WEIGHT",
value: 81.4,
unit: "kg",
at: "2026-09-10T06:30:00.000Z",
note: "after the run",
},
{
type: "WEIGHT",
value: 81.9,
unit: "kg",
at: "2026-09-11T06:30:00.000Z",
note: null,
},
{
type: "PULSE",
value: 62,
unit: "bpm",
at: "2026-09-11T07:00:00.000Z",
note: null,
},
{
type: "BLOOD_GLUCOSE",
value: 104,
unit: "mg/dL",
at: "2026-09-12T08:00:00.000Z",
note: null,
glucoseContext: "FASTING" as const,
},
{
type: "BLOOD_GLUCOSE",
value: 141,
unit: "mg/dL",
at: "2026-09-12T13:00:00.000Z",
note: "after lunch",
glucoseContext: "POSTPRANDIAL" as const,
},
] as const;
const MOOD = [
{
date: "2026-09-10",
mood: "GUT",
score: 4,
tags: null,
at: "2026-09-10T19:00:00.000Z",
},
{
date: "2026-09-11",
mood: "OKAY",
score: 3,
tags: JSON.stringify(["work", "sleep"]),
at: "2026-09-11T20:30:00.000Z",
},
{
date: "2026-09-12",
mood: "SUPER_GUT",
score: 5,
tags: null,
at: "2026-09-12T21:15:00.000Z",
},
] as const;
let ownerId: string;
async function seedOwner(): Promise<void> {
const owner = await makeUser({
timezone: "Europe/Berlin",
glucoseUnit: "mmol/L" /* the JSON export stays in the stored unit */,
});
ownerId = owner.id;
for (const m of MEASUREMENTS) {
await prisma.measurement.create({
data: {
userId: ownerId,
type: m.type,
value: m.value,
unit: m.unit,
measuredAt: new Date(m.at),
source: "MANUAL",
notes: null,
notesEncrypted: encryptNote(m.note),
glucoseContext: "glucoseContext" in m ? m.glucoseContext : null,
},
});
}
// A reading deleted before the backup: its row is the record of a
// deletion, and the import leaves it out.
await prisma.measurement.create({
data: {
userId: ownerId,
type: "WEIGHT",
value: 90,
unit: "kg",
measuredAt: new Date("2026-09-09T06:30:00.000Z"),
source: "MANUAL",
deletedAt: new Date("2026-09-09T07:00:00.000Z"),
},
});
for (const e of MOOD) {
await prisma.moodEntry.create({
data: {
userId: ownerId,
date: e.date,
mood: e.mood,
score: e.score,
tags: e.tags,
source: "MANUAL",
moodLoggedAt: new Date(e.at),
},
});
}
}
/** What an import carries, read back from an account. */
async function carried(userId: string) {
const measurements = await prisma.measurement.findMany({
where: { userId, deletedAt: null },
orderBy: [{ measuredAt: "asc" }, { type: "asc" }],
});
const mood = await prisma.moodEntry.findMany({
where: { userId, deletedAt: null },
orderBy: { moodLoggedAt: "asc" },
});
return {
measurements: measurements.map((m) => ({
type: m.type,
value: m.value,
unit: m.unit,
measuredAt: m.measuredAt.toISOString(),
glucoseContext: m.glucoseContext,
note: readNote(m.notesEncrypted, m.notes),
})),
mood: mood.map((e) => ({
date: e.date,
mood: e.mood,
score: e.score,
tags: e.tags,
loggedAt: e.moodLoggedAt.toISOString(),
})),
};
}
async function importInto(file: unknown) {
const user = await makeUser({ timezone: "Europe/Berlin" });
const res = await callRoute<{
data: { measurements: number; moodEntries: number; skipped: number };
}>(importJson, {
method: "POST",
route: "/api/import",
body: file,
auth: await makeSession(user),
});
return { userId: user.id, res };
}
beforeAll(async () => {
await truncateAllTables(prisma);
await seedOwner();
});
afterAll(async () => {
await truncateAllTables(prisma);
});
describe("the JSON import reads the app's own files", () => {
it("takes a full backup back, nulls and deletion records included", async () => {
let text = "";
await streamFullBackupJson(prisma, ownerId, (chunk) => {
text += chunk;
});
const backup = JSON.parse(text) as {
measurements: Array<Record<string, unknown>>;
moodEntries: Array<Record<string, unknown>>;
};
// The file carries the shapes the import used to refuse.
expect(backup.measurements.some((m) => m.notes === null)).toBe(true);
expect(backup.moodEntries.some((e) => e.tags === null)).toBe(true);
const { userId, res } = await importInto(backup);
expect(res.status).toBe(200);
const deletedInFile = backup.measurements.filter(
(m) => typeof m.deletedAt === "string",
).length;
expect(res.data).toEqual({
measurements: MEASUREMENTS.length,
moodEntries: MOOD.length,
skipped: deletedInFile,
});
expect(await carried(userId)).toEqual(await carried(ownerId));
});
it("takes the JSON export back, empty strings and its envelope included", async () => {
const owner = await prisma.user.findUniqueOrThrow({
where: { id: ownerId },
});
const exported = await callRoute<unknown>(exportJson, {
method: "GET",
route: "/api/export",
query: { format: "json", type: "all" },
auth: await makeSession(owner),
});
expect(exported.status).toBe(200);
const file = exported.body as {
data: {
measurements: Array<Record<string, unknown>>;
moodEntries: Array<Record<string, unknown>>;
};
};
expect(file.data.measurements.some((m) => m.notes === "")).toBe(true);
expect(file.data.moodEntries.some((e) => e.tags === "")).toBe(true);
const { userId, res } = await importInto(file);
expect(res.status).toBe(200);
expect(res.data).toMatchObject({
measurements: MEASUREMENTS.length,
moodEntries: MOOD.length,
});
expect(await carried(userId)).toEqual(await carried(ownerId));
});
it("takes the measurement CSV back into the CSV import, in display units too", async () => {
const owner = await prisma.user.findUniqueOrThrow({
where: { id: ownerId },
});
const exported = await callRoute(exportMeasurementsCsv, {
method: "GET",
route: "/api/export/measurements",
auth: await makeSession(owner),
});
expect(exported.status).toBe(200);
const csv = String(exported.body);
// The account reads glucose in mmol/L, so that is what the file says.
expect(csv).toMatch(/BLOOD_GLUCOSE,\d+(\.\d)?,mmol\/L/);
const user = await makeUser({ timezone: "Europe/Berlin" });
const res = await callRoute<{ inserted: number; skipped: number }>(
importCsv,
{
method: "POST",
route: "/api/import/csv",
text: csv,
headers: { "content-type": "text/csv" },
auth: await makeSession(user),
},
);
expect(res.status).toBe(200);
expect(res.data).toMatchObject({
inserted: MEASUREMENTS.length,
skipped: 0,
});
const back = (await carried(user.id)).measurements;
const original = (await carried(ownerId)).measurements;
expect(back).toHaveLength(original.length);
back.forEach((m, i) => {
const o = original[i]!;
// mmol/L at one decimal: the stored mg/dL value comes back within it.
const tolerance = m.type === "BLOOD_GLUCOSE" ? 0.05 * 18.0156 : 1e-9;
expect({ ...m, value: 0 }).toEqual({ ...o, value: 0 });
expect(Math.abs(m.value - o.value)).toBeLessThanOrEqual(tolerance);
});
});
});