fix(import): take the app's own full backup and JSON export back

The JSON import refused the app's own files: the full backup writes an
empty note, tag list or glucose context as null and the JSON export as
an empty string, and the import schema read either as a value of the
wrong type. The file is now read in the exporter's terms before it is
validated: the export's envelope is opened, every null or empty field
is treated as absent, a backup row carrying a deletion is left out and
counted as skipped, and a unit the app converts on every other way in
is converted to the stored one.

The round trip also showed the portable backup dropping what a reading
itself says: glucose context, sleep stage, range bounds, rhythm and
device. They ride the portable file now, and a portable restore puts
them back. Sync bookkeeping stays out.

An integration test exports an account three ways (full backup, JSON
export, measurement CSV), imports each into a fresh account and compares
what the import carries.
This commit is contained in:
Marc-André Bombeck committed 2026-10-23 17:33:58 +02:00
1 parent 8a11b7bfc6
commit 3d26cee566
7 files changed
+510 -12

No files matched your search

+7 -3
View File
@@ -1,7 +1,8 @@
/**
* `POST /api/import` — import user data (JSON format matching the export
* structure). The route authenticates, meters and parses the file; the
* writers live in `src/lib/import/json/`.
* writers live in `src/lib/import/json/`. The app's own full backup and JSON
* export are read in their own terms first (`normalizeJsonImportFile`).
*/
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { annotate } from "@/lib/logging/context";
@@ -28,6 +29,7 @@ import {
} from "@/lib/validations/entry-instant";
import { isoDateTime } from "@/lib/validations/datetime";
import { fallbackMoodDateStart } from "@/lib/import/json/mood";
import { normalizeJsonImportFile } from "@/lib/import/json/normalize";
import { runJsonImport } from "@/lib/import/json/run";
// Derived from canonical enum so round-trip export → import covers every
@@ -120,7 +122,8 @@ export const POST = apiHandler(async (request: NextRequest) => {
});
if (jsonError) return jsonError;
const parsed = importSchema.safeParse(body);
const normalized = normalizeJsonImportFile(body);
const parsed = importSchema.safeParse(normalized.file);
if (!parsed.success) {
return apiValidationError(
parsed.error.issues[0]?.message ?? "Invalid import payload",
@@ -129,12 +132,13 @@ export const POST = apiHandler(async (request: NextRequest) => {
);
}
const { stats, writeFailed } = await runJsonImport({
const { stats: written, writeFailed } = await runJsonImport({
userId: user.id,
measurements: parsed.data.measurements,
moodEntries: parsed.data.moodEntries,
ipAddress: getClientIp(request),
});
const stats = { ...written, skipped: written.skipped + normalized.skipped };
if (writeFailed) {
return apiError("Import write failed", 503, {
@@ -1002,10 +1002,14 @@ describe("buildFullBackupPayload — aggregation provenance", () => {
);
const rows = payload.measurements as Array<Record<string, unknown>>;
// The portable arm carries the human-readable subset only — no
// externalSourceVersion, no glucoseContext, and no provenance either.
// The portable arm carries the human-readable subset only: none of the
// sync bookkeeping (source ids, their version, the provenance), but every
// attribute of the reading itself, its glucose context among them.
expect(rows[0]).not.toHaveProperty("aggregationProvenance");
expect(rows[0]).not.toHaveProperty("externalSourceVersion");
expect(rows[0]).not.toHaveProperty("externalId");
expect(rows[0]).toHaveProperty("glucoseContext");
expect(rows[0]).toHaveProperty("sleepStage");
});
});
+13 -1
View File
@@ -91,14 +91,26 @@ function serialiseMeasurement(
createdAt: measurement.createdAt.toISOString(),
updatedAt: measurement.updatedAt.toISOString(),
}
: {
: // The portable file is the human-readable subset: no tombstones, no
// ciphertext and none of the sync bookkeeping (the source's ids, its
// version, the aggregate's provenance). What the reading itself says
// stays: a glucose value without its fasting context, a sleep row
// without its stage or a range without its bounds reads as a
// different measurement once it is imported or restored elsewhere.
{
id: measurement.id,
type: measurement.type,
value: measurement.value,
valueMin: measurement.valueMin,
valueMax: measurement.valueMax,
unit: measurement.unit,
measuredAt: measurement.measuredAt.toISOString(),
source: measurement.source,
notes: readNote(measurement.notesEncrypted, measurement.notes),
glucoseContext: measurement.glucoseContext,
sleepStage: measurement.sleepStage,
rhythmClassification: measurement.rhythmClassification,
deviceType: measurement.deviceType,
deletedAt: measurement.deletedAt?.toISOString() ?? null,
};
}
+108
View File
@@ -0,0 +1,108 @@
/**
* The JSON import reads what the app itself writes.
*
* Two of the app's files carry the import's two arrays: the full backup
* (`/api/export/full-backup`), at the top level, and the JSON export
* (`/api/export?format=json`), inside the response envelope's `data`. Neither
* was written for the import schema, and both spell an absent value the way
* their own readers expect: the backup writes `null`, the export writes an
* empty string. The import schema reads an absent value as a missing key, so
* a note, tag list, glucose context or external id the file left empty used
* to refuse the whole file.
*
* So the file is read in the exporter's terms before it is validated:
* - the envelope is opened when the arrays sit inside `data`;
* - every `null` or empty-string field of an entry is dropped, whatever the
* field is called, so a nullable column added to an export later cannot
* reopen the refusal;
* - a backup row that carries a `deletedAt` is the record of a deletion,
* not a reading, and is left out (and counted as skipped);
* - a measurement in a unit the app converts on every other way in (lb,
* °F, in, mmol/L, as the CSV import does) is converted to the stored
* unit, so a file kept in display units does not land as stored values.
*
* Validation stays where it was: the schema in the route still bounds every
* value and instant.
*/
import { resolveToCanonicalUnit } from "@/lib/measurements/unit-aliases";
export interface NormalizedJsonImportFile {
/** The file in the import schema's terms, ready for `safeParse`. */
file: unknown;
/** Entries left out before validation: deletion records from a backup. */
skipped: number;
}
type Entry = Record<string, unknown>;
function isRecord(value: unknown): value is Entry {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function withoutAbsentFields(entry: Entry): Entry {
const out: Entry = {};
for (const [key, value] of Object.entries(entry)) {
if (value === null || value === "") continue;
out[key] = value;
}
return out;
}
function inStoredUnit(entry: Entry): Entry {
const { type, value, unit } = entry;
if (
typeof type !== "string" ||
typeof value !== "number" ||
typeof unit !== "string"
) {
return entry;
}
const canonical = resolveToCanonicalUnit(type, value, unit);
if (!canonical || canonical.unit === unit) return entry;
return { ...entry, value: canonical.value, unit: canonical.unit };
}
function readEntries(
value: unknown,
shape: (entry: Entry) => Entry,
): { entries: unknown; skipped: number } {
if (!Array.isArray(value)) return { entries: value, skipped: 0 };
let skipped = 0;
const entries: unknown[] = [];
for (const raw of value) {
if (!isRecord(raw)) {
entries.push(raw);
continue;
}
if (typeof raw.deletedAt === "string" && raw.deletedAt !== "") {
skipped += 1;
continue;
}
entries.push(shape(withoutAbsentFields(raw)));
}
return { entries, skipped };
}
export function normalizeJsonImportFile(
body: unknown,
): NormalizedJsonImportFile {
if (!isRecord(body)) return { file: body, skipped: 0 };
const source =
!("measurements" in body) && !("moodEntries" in body) && isRecord(body.data)
? body.data
: body;
const file: Entry = {};
let skipped = 0;
if ("measurements" in source) {
const read = readEntries(source.measurements, inStoredUnit);
file.measurements = read.entries;
skipped += read.skipped;
}
if ("moodEntries" in source) {
const read = readEntries(source.moodEntries, (entry) => entry);
file.moodEntries = read.entries;
skipped += read.skipped;
}
return { file, skipped };
}