From a840e30fac767a62a01ba9bc3a218e7d20f9c102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 23 Oct 2026 21:33:06 +0200 Subject: [PATCH] feat(import): carry every field the app's own files write The JSON import read six measurement fields and six mood fields while the full backup and the JSON export carry twice that. A sleep night came back as stageless rows, a range reading as a single value, an ECG alert without its verdict, and a mood entry without its note, its level-A values, its zone, its tag and factor links or its day context. The import now reads valueMin/valueMax, sleepStage, rhythmClassification and deviceType on a reading, and note, a1 to a5, tz, factors, structuredTags and context on a mood entry. Each mood entry is written with its links and context in one transaction, the way the bulk mood ingest writes one. A link to a tag the importing account does not have is dropped and named in droppedTagKeys / droppedFactorKeys, and the import card says which. The schema moves to src/lib/import/json/schema.ts so the guards can read it. json-import-field-coverage.test.ts runs the JSON export and both backup variants on an account with every column filled in: every field they write is either read by the import or named with the reason it is not (row ids, source, deletion records, ciphertext, sync bookkeeping), and a reason for a field no exporter writes fails as stale. The round trip imports each file into a fresh account and compares the re-export field for field. The published contract is held to the runtime schema. --- docs/api/openapi.yaml | 291 +++++++++++- docs/integrations/data-import.md | 70 ++- messages/de.json | 1 + messages/en.json | 1 + messages/es.json | 1 + messages/fr.json | 1 + messages/it.json | 1 + messages/ko.json | 1 + messages/nl.json | 1 + messages/pl.json | 1 + messages/pt-BR.json | 1 + .../baselines/source-function-lines.json | 5 +- .../baselines/version-stamp-comments.json | 5 +- .../json-import-openapi-parity.test.ts | 75 +++ ...asurement-write-plausibility-guard.test.ts | 2 +- src/app/api/import/__tests__/route.test.ts | 13 +- src/app/api/import/route.ts | 110 +---- .../__tests__/json-import-result.test.tsx | 54 +++ .../import-panel/json-import-card.tsx | 118 +++-- src/lib/import/json/measurements.ts | 63 ++- src/lib/import/json/mood-instant.ts | 23 + src/lib/import/json/mood.ts | 197 ++++---- src/lib/import/json/run.ts | 18 +- src/lib/import/json/schema.ts | 184 ++++++++ src/lib/import/json/tally.ts | 9 + src/lib/openapi/routes/import.ts | 116 ++++- .../json-import-field-coverage.test.ts | 262 +++++++++++ .../json-import-own-exports.test.ts | 429 ++++++++++-------- tests/integration/json-import-seed.ts | 267 +++++++++++ .../import-json.01-first-file.json.snap | 1 + .../import-json.02-same-file.json.snap | 1 + 31 files changed, 1814 insertions(+), 508 deletions(-) create mode 100644 src/__tests__/json-import-openapi-parity.test.ts create mode 100644 src/components/settings/import-panel/__tests__/json-import-result.test.tsx create mode 100644 src/lib/import/json/mood-instant.ts create mode 100644 src/lib/import/json/schema.ts create mode 100644 tests/integration/json-import-field-coverage.test.ts create mode 100644 tests/integration/json-import-seed.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 563d48eb1..ea03883bf 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -16162,16 +16162,19 @@ paths: tags: - Import summary: Import measurements and mood entries from JSON - description: "Restores a JSON export (the structure `GET /api/export/json` emits): up to 10 000 measurements plus 10 000 - mood entries per call, body ≤ 16 MB. Rows carrying an `externalId` upsert idempotently (re-import updates in - place); rows without one are first-write-wins and dedup against the existing unique keys, so an exact duplicate - is `skipped` rather than doubled. A row with an unstable `externalId` is skipped while the rest of the file - still lands. Every imported measurement stores `source = IMPORT`; every imported mood entry re-derives - pleasantness from its label. Writes are per-row, not transactional: on a mid-file write failure the response is - 503 (`meta.errorCode` = `import.write_failed`, `meta.retryable` = true) with `meta.stats` carrying what landed — - rows already written stay written, and a retry is only safe against duplication for rows that carry an - `externalId`. Validation is fail-fast (the 422 carries the first issue's message, not the multi-issue envelope). - Rate-limited 5/hour per user (shared `import:` bucket with the CSV importer)." + description: "Imports the app's own files back: the JSON export (`GET /api/export?format=json`, read inside its `data` + envelope) or the portable full backup (`GET /api/export/full-backup`). Every field either file writes for a + reading or a mood entry is carried, apart from the row ids, the source and the sync bookkeeping; a backup's + deletion records are skipped and counted. Up to 10 000 measurements plus 10 000 mood entries per call, body ≤ 16 + MB. Rows carrying an `externalId` upsert idempotently (re-import updates in place); rows without one are + first-write-wins and dedup against the existing unique keys, so an exact duplicate is `skipped` rather than + doubled. A row with an unstable `externalId` is skipped while the rest of the file still lands. Every imported + row stores `source = IMPORT`; a mood entry without `a1` derives pleasantness from its label. Writes are per-row, + not transactional: on a mid-file write failure the response is 503 (`meta.errorCode` = `import.write_failed`, + `meta.retryable` = true) with `meta.stats` carrying what landed — rows already written stay written, and a retry + is only safe against duplication for rows that carry an `externalId`. Validation is fail-fast (the 422 carries + the first issue's message, not the multi-issue envelope). Rate-limited 5/hour per user (shared `import:` bucket + with the CSV importer)." requestBody: required: true content: @@ -33206,8 +33209,8 @@ components: type: array items: $ref: "#/components/schemas/JsonImportMoodEntry" - description: JSON import payload, matching the structure `GET /api/export/json` emits. Both sections optional; up to 10 - 000 rows each. + description: "JSON import payload: the measurements and mood entries of the JSON export or the portable full backup. + Both sections optional; up to 10 000 rows each." JsonImportMeasurement: type: object properties: @@ -33217,6 +33220,12 @@ components: type: number description: Checked against the type's server-side plausibility range; an out-of-range value fails the whole request with 422. + valueMin: + description: Lower bound of a range reading (an hourly heart-rate statistic). Not above `valueMax`. + type: number + valueMax: + description: Upper bound of a range reading. + type: number unit: type: string measuredAt: @@ -33232,9 +33241,31 @@ components: - POSTPRANDIAL - RANDOM - BEDTIME - source: - description: "Accepted for export-format compatibility and ignored: every imported row stores `source = IMPORT`." + sleepStage: + description: A sleep row's stage. type: string + enum: + - IN_BED + - ASLEEP + - AWAKE + - CORE + - DEEP + - REM + rhythmClassification: + description: The device's own verdict on an ECG, a rhythm alert or a steadiness reading. + type: string + enum: + - IRREGULAR + - NOT_DETECTED + - INCONCLUSIVE + - LOW + - VERY_LOW + - FIRED + deviceType: + description: The writer of the reading (watch, phone). + type: string + minLength: 1 + maxLength: 32 notes: description: Encrypted at rest. type: string @@ -33271,17 +33302,231 @@ components: type: integer minimum: 1 maximum: 5 - description: Stored as sent — the export format's own field. Pleasantness (`a1`) is re-derived from `mood` on every - imported row, never taken from this. + description: Stored as sent — the export format's own field. Pleasantness (`a1`) is never taken from this. + a1: + description: Pleasantness, 0–10. Absent → derived from `mood`. + type: integer + minimum: 0 + maximum: 10 + a2: + description: Stress, 0–10. + type: integer + minimum: 0 + maximum: 10 + a3: + description: Energy, 0–10. + type: integer + minimum: 0 + maximum: 10 + a4: + description: Connection, 0–10. + type: integer + minimum: 0 + maximum: 10 + a5: + description: Stability, 0–10. `a2` to `a5` absent leave a row the import updates as it was. + type: integer + minimum: 0 + maximum: 10 tags: description: JSON-encoded array of tag strings, stored verbatim. type: string + note: + description: The entry's note. Encrypted at rest. + type: string + maxLength: 500 loggedAt: description: Bounded like `measuredAt`. Absent → the server derives a stable per-row instant within the `date` day, so a re-import of the same file dedups instead of duplicating. type: string format: date-time pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + tz: + description: IANA zone the `date` is anchored to. Absent → the legacy reading. + type: string + maxLength: 64 + factors: + description: Rated factors by catalogue key; replaces the entry's rated links. A key the account cannot resolve is + dropped and named in `droppedFactorKeys`; a rating outside the factor's own scale skips the entry. + maxItems: 30 + type: array + items: + type: object + properties: + key: + type: string + maxLength: 60 + rating: + type: integer + minimum: 1 + maximum: 5 + required: + - key + - rating + structuredTags: + description: Ticked tags by key (the catalogue or the account's own); replaces the entry's tag links. An unresolvable + key is dropped and named in `droppedTagKeys`. + maxItems: 30 + type: array + items: + type: string + maxLength: 60 + context: + description: The day context; replaces the entry's context. `contactCircles` and `leisureCategories` are also read as + the backup's JSON-encoded string. + type: object + properties: + workStatus: + description: Shape of the working day. + anyOf: + - type: string + enum: + - off + - partial + - regular + - overtime + - type: "null" + workMinutes: + anyOf: + - type: integer + minimum: 0 + maximum: 1440 + - type: "null" + overtimeMinutes: + anyOf: + - type: integer + minimum: 0 + maximum: 1440 + - type: "null" + workLoad: + description: "0-10, INVERSE-oriented: a higher value is a heavier day, not a better one." + anyOf: + - type: integer + minimum: 0 + maximum: 10 + - type: "null" + workSatisfaction: + anyOf: + - type: integer + minimum: 0 + maximum: 10 + - type: "null" + contactCircles: + description: "Who the day's contact was with. Multi-select. Never scored: the count of people is not a good or a bad + number." + anyOf: + - type: array + items: + type: string + enum: + - partner + - child + - family + - friends + - colleagues + - acquaintances + - professional + - other + - type: "null" + contactForm: + anyOf: + - type: string + enum: + - inPerson + - phone + - video + - written + - type: "null" + contactExtent: + anyOf: + - type: string + enum: + - brief + - medium + - extended + - type: "null" + contactQuality: + anyOf: + - type: integer + minimum: 0 + maximum: 10 + - type: "null" + contactSupport: + anyOf: + - type: integer + minimum: 0 + maximum: 10 + - type: "null" + leisureCategories: + anyOf: + - type: array + items: + type: string + enum: + - film + - reading + - gaming + - music + - creative + - outing + - garden + - cooking + - timeWithChild + - relaxation + - other + - type: "null" + leisureMinutes: + anyOf: + - type: integer + minimum: 0 + maximum: 1440 + - type: "null" + leisureJoy: + anyOf: + - type: integer + minimum: 0 + maximum: 10 + - type: "null" + leisureRecovery: + anyOf: + - type: integer + minimum: 0 + maximum: 10 + - type: "null" + eventType: + description: One notable event. One per entry, not a list. + anyOf: + - type: string + enum: + - goodNews + - badNews + - success + - disappointment + - conflict + - appointment + - familyEvent + - change + - unexpected + - type: "null" + eventValence: + description: -5 (clearly bad) to +5 (clearly good). + anyOf: + - type: integer + minimum: -5 + maximum: 5 + - type: "null" + eventAt: + anyOf: + - type: string + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + - type: "null" + note: + description: One free-text note for the whole context. AES-256-GCM at rest; the ciphertext never rides the wire. + anyOf: + - type: string + maxLength: 500 + - type: "null" externalId: description: "Source-stable id (e.g. a Daylio row id) making re-import idempotent (upsert on `(userId, source=IMPORT, externalId)`). Same stability rule as the measurement `externalId`: an unstable id skips the row." @@ -73827,7 +74072,19 @@ components: type: integer minimum: 0 maximum: 9007199254740991 - description: "Rows not written: duplicates of existing rows, and rows carrying an unstable `externalId`." + description: "Rows not written: duplicates of existing rows, rows carrying an unstable `externalId`, mood entries with a + factor rating outside its own scale, and a backup's deletion records." + droppedTagKeys: + description: Tag keys of imported mood entries this account cannot resolve (another account's custom tag, a retired + key). The entries landed without them. Present only when non-empty. + type: array + items: + type: string + droppedFactorKeys: + description: Rated-factor keys this account cannot resolve, as `droppedTagKeys`. + type: array + items: + type: string required: - measurements - moodEntries diff --git a/docs/integrations/data-import.md b/docs/integrations/data-import.md index 5e3d43edd..f53fc3e33 100644 --- a/docs/integrations/data-import.md +++ b/docs/integrations/data-import.md @@ -57,23 +57,43 @@ arrays: } ``` -Each array holds at most **10 000** entries. The endpoint is rate-limited -to **5 imports per hour per account**. Existing rows (matched on the -type + timestamp uniqueness, or on `externalId` for mood entries) are -skipped and counted under `skipped` in the response. +Each array holds at most **10 000** entries and the file at most 16 MB. +The endpoint is rate-limited to **5 imports per hour per account**. +Existing rows (matched on the type + timestamp uniqueness, or on +`externalId` for mood entries) are skipped and counted under `skipped` in +the response. + +The app's own files go straight in: the JSON export (**Export → JSON**, +read inside its `data` envelope) and the portable full backup. Every field +either file writes for a reading or a mood entry comes back, apart from +the row ids, the source (every imported row is stored as `IMPORT`) and the +sync bookkeeping of the account the file came from. An empty field, `null` +in the backup and `""` in the export, reads as absent, a backup's deletion +records are skipped and counted, and a reading in a display unit (lb, °F, +mmol/L) is converted to the stored one. + +A mood entry's tags and rated factors are linked by key. A key the +importing account has no tag for, typically a tag another account made +for itself, is dropped from that entry, and the response names it in +`droppedTagKeys` or `droppedFactorKeys`. The rest of the entry lands. ### Measurement entry -| Field | Required | Type | Notes | -| ---------------- | -------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `type` | yes | enum | One of the `MeasurementType` values in the table below. | -| `value` | yes | number | Plausibility-checked per type; out-of-range values are rejected. | -| `unit` | yes | string | The canonical unit for the type (see the table). | -| `measuredAt` | yes | string | ISO-8601 datetime, e.g. `2026-05-01T08:00:00.000Z`. | -| `source` | no | string | Free-text origin label. Imported rows are tagged `IMPORT` server-side regardless. | -| `notes` | no | string | Optional free text. | -| `glucoseContext` | no | enum | Only for `BLOOD_GLUCOSE`. See [Glucose context](#glucose-context). | -| `externalId` | no | string | A source-stable id (1–120 chars). When present, a re-import upserts on `(user, type, source, externalId)` so an edit upstream is reflected rather than skipped. | +| Field | Required | Type | Notes | +| ---------------------- | -------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `type` | yes | enum | One of the `MeasurementType` values in the table below. | +| `value` | yes | number | Plausibility-checked per type; out-of-range values are rejected. | +| `unit` | yes | string | The canonical unit for the type (see the table). | +| `measuredAt` | yes | string | ISO-8601 datetime, e.g. `2026-05-01T08:00:00.000Z`. | +| `source` | no | string | Free-text origin label, not stored. Imported rows are tagged `IMPORT` server-side regardless. | +| `notes` | no | string | Optional free text. | +| `glucoseContext` | no | enum | Only for `BLOOD_GLUCOSE`. See [Glucose context](#glucose-context). | +| `valueMin` | no | number | Lower bound of a range reading, such as an hourly heart-rate statistic. Not above `valueMax`. | +| `valueMax` | no | number | Upper bound of a range reading. | +| `sleepStage` | no | enum | A sleep row's stage: `IN_BED`, `ASLEEP`, `AWAKE`, `CORE`, `DEEP` or `REM`. | +| `rhythmClassification` | no | enum | The device's verdict on an ECG, a rhythm alert or a steadiness reading. | +| `deviceType` | no | string | The writer of the reading (watch, phone), 1–32 chars. | +| `externalId` | no | string | A source-stable id (1–120 chars). When present, a re-import upserts on `(user, type, source, externalId)` so an edit upstream is reflected rather than skipped. | ### Glucose context @@ -98,14 +118,20 @@ moment you know it. ### Mood entry -| Field | Required | Type | Notes | -| ------------ | -------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `date` | yes | string | `YYYY-MM-DD`. | -| `mood` | yes | enum | One of `SUPER_GUT`, `GUT`, `OKAY`, `SCHLECHT`, `LAUSIG`. | -| `score` | yes | integer | `1`–`5`. | -| `tags` | no | string | Comma-separated tags. | -| `loggedAt` | no | string | ISO-8601 datetime; defaults to the start of the given `date` in the default timezone. | -| `externalId` | no | string | A source-stable id (1–120 chars). When present, a re-import upserts on `(user, source, externalId)` so an edit upstream is reflected rather than skipped. | +| Field | Required | Type | Notes | +| ---------------- | -------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `date` | yes | string | `YYYY-MM-DD`. | +| `mood` | yes | enum | One of `SUPER_GUT`, `GUT`, `OKAY`, `SCHLECHT`, `LAUSIG`. | +| `score` | yes | integer | `1`–`5`. | +| `tags` | no | string | Comma-separated tags. | +| `note` | no | string | The entry's note, up to 500 chars. | +| `a1`–`a5` | no | integer | The five level-A values, `0`–`10`. Without `a1`, pleasantness is derived from `mood`. | +| `loggedAt` | no | string | ISO-8601 datetime; defaults to the start of the given `date` in the default timezone. | +| `tz` | no | string | The IANA zone `date` is anchored to. | +| `factors` | no | array | Rated factors, `[{ "key": "…", "rating": 1–5 }]`; replace the entry's rated links. | +| `structuredTags` | no | array | Ticked tags by key; replace the entry's tag links. | +| `context` | no | object | The day context (work, contacts, leisure, an event, a note), as the backup writes it; replaces the entry's context. | +| `externalId` | no | string | A source-stable id (1–120 chars). When present, a re-import upserts on `(user, source, externalId)` so an edit upstream is reflected rather than skipped. | ### `MeasurementType` values and canonical units diff --git a/messages/de.json b/messages/de.json index 2a3d40ac2..77d4a8f41 100644 --- a/messages/de.json +++ b/messages/de.json @@ -5794,6 +5794,7 @@ "readFailed": "Diese Datei konnte nicht gelesen werden.", "resultSummary": "{measurements} Messwerte und {moods} Stimmungseinträge importiert, {skipped} übersprungen.", "resultSuccess": "{measurements} Messwerte und {moods} Stimmungseinträge importiert", + "droppedLinks": "Ohne diese Tags importiert, weil dieses Konto keine mit dem Namen hat: {keys}", "resultNothing": "Nichts importiert. Alle {skipped} Einträge wurden übersprungen.", "resultEmpty": "Diese Datei enthielt weder Messwerte noch Stimmungseinträge." }, diff --git a/messages/en.json b/messages/en.json index 97234ceaf..4353cf62c 100644 --- a/messages/en.json +++ b/messages/en.json @@ -5794,6 +5794,7 @@ "readFailed": "Couldn't read that file.", "resultSummary": "Imported {measurements} measurements and {moods} mood entries, {skipped} skipped.", "resultSuccess": "Imported {measurements} measurements and {moods} mood entries", + "droppedLinks": "Imported without these tags, because this account has none by that name: {keys}", "resultNothing": "Nothing was imported. All {skipped} entries were skipped.", "resultEmpty": "That file carried no measurements and no mood entries." }, diff --git a/messages/es.json b/messages/es.json index 3ca0cf9fa..0f92c4ca9 100644 --- a/messages/es.json +++ b/messages/es.json @@ -5794,6 +5794,7 @@ "readFailed": "No se pudo leer ese archivo.", "resultSummary": "{measurements} mediciones y {moods} entradas de ánimo importadas, {skipped} omitidas.", "resultSuccess": "{measurements} mediciones y {moods} entradas de ánimo importadas", + "droppedLinks": "Importado sin estas etiquetas, porque esta cuenta no tiene ninguna con ese nombre: {keys}", "resultNothing": "No se importó nada. Se omitieron las {skipped} entradas.", "resultEmpty": "Ese archivo no contenía mediciones ni entradas de ánimo." }, diff --git a/messages/fr.json b/messages/fr.json index 106d1d321..e293e2c47 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -5794,6 +5794,7 @@ "readFailed": "Impossible de lire ce fichier.", "resultSummary": "{measurements} mesures et {moods} entrées d'humeur importées, {skipped} ignorées.", "resultSuccess": "{measurements} mesures et {moods} entrées d'humeur importées", + "droppedLinks": "Importé sans ces étiquettes, car ce compte n’en a aucune de ce nom : {keys}", "resultNothing": "Rien n'a été importé. Les {skipped} entrées ont toutes été ignorées.", "resultEmpty": "Ce fichier ne contenait ni mesures ni entrées d'humeur." }, diff --git a/messages/it.json b/messages/it.json index ea4d45364..57c7a2ecd 100644 --- a/messages/it.json +++ b/messages/it.json @@ -5794,6 +5794,7 @@ "readFailed": "Impossibile leggere il file.", "resultSummary": "{measurements} misurazioni e {moods} voci dell'umore importate, {skipped} saltate.", "resultSuccess": "{measurements} misurazioni e {moods} voci dell'umore importate", + "droppedLinks": "Importato senza questi tag, perché questo account non ne ha nessuno con quel nome: {keys}", "resultNothing": "Non è stato importato nulla. Tutte le {skipped} voci sono state saltate.", "resultEmpty": "Questo file non conteneva né misurazioni né voci dell'umore." }, diff --git a/messages/ko.json b/messages/ko.json index 6ec545b8e..49f7f9551 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -5794,6 +5794,7 @@ "readFailed": "파일을 읽지 못했어요.", "resultSummary": "측정 {measurements}개와 기분 기록 {moods}개를 불러왔고, {skipped}개는 건너뛰었어요.", "resultSuccess": "측정 {measurements}개와 기분 기록 {moods}개를 불러왔어요", + "droppedLinks": "이 계정에 같은 이름의 태그가 없어 다음 태그 없이 가져왔습니다: {keys}", "resultNothing": "불러온 게 없어요. {skipped}개 기록을 모두 건너뛰었어요.", "resultEmpty": "그 파일에는 측정도 기분 기록도 없었어요." }, diff --git a/messages/nl.json b/messages/nl.json index 6415f4489..92c3bc0ba 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -5794,6 +5794,7 @@ "readFailed": "Dat bestand kon niet worden gelezen.", "resultSummary": "{measurements} metingen en {moods} stemmingsregistraties geïmporteerd, {skipped} overgeslagen.", "resultSuccess": "{measurements} metingen en {moods} stemmingsregistraties geïmporteerd", + "droppedLinks": "Geïmporteerd zonder deze tags, omdat dit account er geen met die naam heeft: {keys}", "resultNothing": "Er is niets geïmporteerd. Alle {skipped} registraties zijn overgeslagen.", "resultEmpty": "Dat bestand bevatte geen metingen en geen stemmingsregistraties." }, diff --git a/messages/pl.json b/messages/pl.json index cb5dc32b7..81ef5b1d2 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -5794,6 +5794,7 @@ "readFailed": "Nie można odczytać tego pliku.", "resultSummary": "Zaimportowano {measurements} pomiarów i {moods} wpisów nastroju, pominięto {skipped}.", "resultSuccess": "Zaimportowano {measurements} pomiarów i {moods} wpisów nastroju", + "droppedLinks": "Zaimportowano bez tych tagów, bo to konto nie ma żadnego o tej nazwie: {keys}", "resultNothing": "Nic nie zaimportowano. Pominięto wszystkie {skipped} wpisów.", "resultEmpty": "Ten plik nie zawierał pomiarów ani wpisów nastroju." }, diff --git a/messages/pt-BR.json b/messages/pt-BR.json index aa102f900..e20851594 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -5794,6 +5794,7 @@ "readFailed": "Não foi possível ler esse arquivo.", "resultSummary": "Importadas {measurements} medições e {moods} registros de humor, {skipped} pulados.", "resultSuccess": "Importadas {measurements} medições e {moods} registros de humor", + "droppedLinks": "Importado sem estas tags, porque esta conta não tem nenhuma com esse nome: {keys}", "resultNothing": "Nada foi importado. Todas as {skipped} entradas foram puladas.", "resultEmpty": "Esse arquivo não continha medições nem registros de humor." }, diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index 41a832005..965c9159f 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 355, - "total": 114640, + "count": 354, + "total": 114417, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -208,7 +208,6 @@ "components/settings/import-panel/csv-import-card.tsx :: CsvImportCard": 212, "components/settings/import-panel/dose-history-import-card.tsx :: DoseHistoryImportCard": 225, "components/settings/import-panel/health-connect-import-card.tsx :: HealthConnectImportCard": 293, - "components/settings/import-panel/json-import-card.tsx :: JsonImportCard": 223, "components/settings/import-panel/record-csv-import-card.tsx :: RecordCsvImportSheet": 460, "components/settings/insights-pill-order-section.tsx :: InsightsPillOrderSection": 255, "components/settings/integrations/connections-panel.tsx :: ConnectionsPanel": 230, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 2bb3a084e..de6f472fb 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", - "count": 2898, - "total": 9294, + "count": 2897, + "total": 9290, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -284,7 +284,6 @@ "app/api/illness/episodes/route.ts": 1, "app/api/import/__tests__/route.test.ts": 3, "app/api/import/csv/__tests__/route.test.ts": 2, - "app/api/import/route.ts": 4, "app/api/ingest/medication/__tests__/route.test.ts": 1, "app/api/ingest/medication/route.ts": 7, "app/api/insights/biomarker-assessment/route.ts": 1, diff --git a/src/__tests__/json-import-openapi-parity.test.ts b/src/__tests__/json-import-openapi-parity.test.ts new file mode 100644 index 000000000..7fe77d326 --- /dev/null +++ b/src/__tests__/json-import-openapi-parity.test.ts @@ -0,0 +1,75 @@ +/** + * The published JSON import contract names the fields the import reads. + * + * `src/lib/openapi/routes/import.ts` restates `jsonImportSchema` by hand, + * because the runtime schema carries transforms the document describes in + * prose. A restatement drifts: a field the import learns to read stays + * undocumented, and one it stops reading stays promised. So the two field + * sets are compared here, per row kind, both ways. + */ +import { describe, expect, it } from "vitest"; + +import { + jsonImportMeasurementSchema, + jsonImportMoodContextSchema, + jsonImportMoodEntrySchema, +} from "@/lib/import/json/schema"; +import { buildOpenApiDocument } from "@/lib/openapi/registry"; + +type SchemaObject = { + properties?: Record; + $ref?: string; + allOf?: SchemaObject[]; +}; + +const doc = buildOpenApiDocument() as { + components: { schemas: Record }; +}; + +function resolve(schema: SchemaObject): SchemaObject { + if (schema.$ref) { + const name = schema.$ref.split("/").pop()!; + return resolve(doc.components.schemas[name]!); + } + if (schema.allOf) { + return { + properties: Object.assign( + {}, + ...schema.allOf.map((part) => resolve(part).properties ?? {}), + ), + }; + } + return schema; +} + +function documented(name: string): string[] { + const schema = doc.components.schemas[name]; + expect(schema, `${name} is published`).toBeDefined(); + return Object.keys(resolve(schema!).properties ?? {}).sort(); +} + +describe("the published JSON import contract", () => { + it("names every measurement field the import reads, and no other", () => { + const fields = documented("JsonImportMeasurement"); + expect(fields.length).toBeGreaterThan(5); + expect(fields).toEqual( + Object.keys(jsonImportMeasurementSchema.shape).sort(), + ); + }); + + it("names every mood-entry field the import reads, and no other", () => { + const fields = documented("JsonImportMoodEntry"); + expect(fields.length).toBeGreaterThan(5); + expect(fields).toEqual(Object.keys(jsonImportMoodEntrySchema.shape).sort()); + }); + + it("names every day-context field the import reads, and no other", () => { + const entry = resolve(doc.components.schemas.JsonImportMoodEntry!); + const context = resolve(entry.properties!.context!); + const fields = Object.keys(context.properties ?? {}).sort(); + expect(fields.length).toBeGreaterThan(10); + expect(fields).toEqual( + Object.keys(jsonImportMoodContextSchema.shape).sort(), + ); + }); +}); diff --git a/src/__tests__/measurement-write-plausibility-guard.test.ts b/src/__tests__/measurement-write-plausibility-guard.test.ts index 0de1ac9ce..7b8af5a77 100644 --- a/src/__tests__/measurement-write-plausibility-guard.test.ts +++ b/src/__tests__/measurement-write-plausibility-guard.test.ts @@ -75,7 +75,7 @@ const WRITERS: Record = { }, "lib/import/json/measurements.ts": { kind: "upstream", - validatedIn: "app/api/import/route.ts", + validatedIn: "lib/import/json/schema.ts", }, // The batch ingest's set-based insert writes rows that already passed // `validateMeasurementRange` per entry in its first pass; everything else diff --git a/src/app/api/import/__tests__/route.test.ts b/src/app/api/import/__tests__/route.test.ts index 0c334a71e..64cea8978 100644 --- a/src/app/api/import/__tests__/route.test.ts +++ b/src/app/api/import/__tests__/route.test.ts @@ -8,8 +8,8 @@ vi.mock("@/lib/api-handler", () => ({ })), })); -vi.mock("@/lib/db", () => ({ - prisma: { +vi.mock("@/lib/db", () => { + const prisma = { measurement: { create: vi.fn(), createManyAndReturn: vi.fn(), @@ -18,8 +18,13 @@ vi.mock("@/lib/db", () => ({ upsert: vi.fn(), }, moodEntry: { create: vi.fn(), findUnique: vi.fn(), upsert: vi.fn() }, - }, -})); + // A mood entry is written in a transaction with its links and context; + // the callback runs against this same client. Given as the mock's own + // implementation, so `resetAllMocks` keeps it. + $transaction: vi.fn(async (fn: (tx: unknown) => unknown) => fn(prisma)), + }; + return { prisma }; +}); vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: vi.fn(), diff --git a/src/app/api/import/route.ts b/src/app/api/import/route.ts index aedab2190..ccf6fe068 100644 --- a/src/app/api/import/route.ts +++ b/src/app/api/import/route.ts @@ -1,7 +1,7 @@ /** * `POST /api/import` — import user data (JSON format matching the export * structure). The route authenticates, meters and parses the file; the - * writers live in `src/lib/import/json/`. The app's own full backup and JSON + * schema and the writers live in `src/lib/import/json/`. The app's own full backup and JSON * export are read in their own terms first (`normalizeJsonImportFile`). */ import { apiHandler, requireAuth } from "@/lib/api-handler"; @@ -16,94 +16,14 @@ import { } from "@/lib/api-response"; import { checkRateLimit } from "@/lib/rate-limit"; import { NextRequest } from "next/server"; -import { z } from "zod/v4"; -import { - validateMeasurementRange, - measurementTypeEnum, - glucoseContextEnum, -} from "@/lib/validations/measurement"; -import { - isPlausibleEntryInstant, - validateEntryInstant, -} from "@/lib/validations/entry-instant"; -import { isoDateTime } from "@/lib/validations/datetime"; -import { fallbackMoodDateStart } from "@/lib/import/json/mood"; +import { droppedLinkKeysForWire } from "@/lib/mood/tag-links"; import { normalizeJsonImportFile } from "@/lib/import/json/normalize"; import { runJsonImport } from "@/lib/import/json/run"; - -// Derived from canonical enum so round-trip export → import covers every -// type. Previous hardcoded subset silently dropped 4 of 11 types -// (V3 audit: enum drift cousins). -const measurementSchema = z - .object({ - type: measurementTypeEnum, - value: z.number(), - unit: z.string(), - // v1.17.1 — share the canonical entry-instant bound used by the single - // POST + edit paths (v1.17 W1b). Previously a bare datetime check - // with no ceiling, so a crafted JSON import could forward-date a reading. - // `validateEntryInstant` rejects any future instant beyond a 5-min skew - // tolerance and any instant before 1900; the field transforms to a `Date`. - measuredAt: validateEntryInstant( - isoDateTime().transform((s) => new Date(s)), - ), - glucoseContext: glucoseContextEnum.optional(), - source: z.string().optional(), - notes: z.string().optional(), - // v1.17.1 — optional source-stable id. When present, the import upserts on - // `(userId, type, source=IMPORT, externalId)` so a re-import of the same - // export is idempotent rather than minting a duplicate set — mirrors the - // mood path. Absent → first-write-wins create with a NULL externalId - // (distinct in the unique key), which DOES duplicate on re-import. - // Stability is enforced per entry in the import loop, not at the - // field: a schema-level refusal would fail the whole file on one bad - // row. - // @external-id-checked-per-entry: src/lib/import/json/measurements.ts - externalId: z.string().min(1).max(120).optional(), - }) - .refine( - (data) => validateMeasurementRange(data.type, data.value) === null, - "Wert ausserhalb des plausiblen Bereichs", - ); - -const moodEntrySchema = z - .object({ - date: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), - mood: z.enum(["SUPER_GUT", "GUT", "OKAY", "SCHLECHT", "LAUSIG"]), - score: z.number().int().min(1).max(5), - tags: z.string().optional(), - // v1.17.1 — same entry-instant bound as measurements; an imported mood - // log cannot be forward-dated. Optional + transforms to a `Date` when set. - loggedAt: validateEntryInstant( - isoDateTime().transform((s) => new Date(s)), - ).optional(), - // v1.12.1 — optional source-stable id (e.g. a Daylio row id). When - // present, the import upserts on `(userId, source, externalId)` so a - // re-import of the same export is idempotent rather than minting a - // duplicate; absent → the legacy first-write-wins create path. - // Stability is enforced per entry in the import loop, not at the - // field: a schema-level refusal would fail the whole file on one bad - // row. - // @external-id-checked-per-entry: src/lib/import/json/mood.ts - externalId: z.string().min(1).max(120).optional(), - }) - .superRefine((entry, ctx) => { - if (entry.loggedAt) return; - const dateStart = fallbackMoodDateStart(entry.date); - if (!dateStart || !isPlausibleEntryInstant(dateStart)) { - ctx.addIssue({ - code: "custom", - path: ["date"], - message: "Mood date must be between 1900 and today", - }); - } - }); - -const importSchema = z.object({ - measurements: z.array(measurementSchema).max(10000).optional(), - moodEntries: z.array(moodEntrySchema).max(10000).optional(), -}); +import { + JSON_IMPORT_MAX_BYTES, + jsonImportSchema, +} from "@/lib/import/json/schema"; export const POST = apiHandler(async (request: NextRequest) => { const { user } = await requireAuth(); @@ -118,12 +38,12 @@ export const POST = apiHandler(async (request: NextRequest) => { } const { data: body, error: jsonError } = await safeJson(request, { - maxBytes: 16 * 1024 * 1024, + maxBytes: JSON_IMPORT_MAX_BYTES, }); if (jsonError) return jsonError; const normalized = normalizeJsonImportFile(body); - const parsed = importSchema.safeParse(normalized.file); + const parsed = jsonImportSchema.safeParse(normalized.file); if (!parsed.success) { return apiValidationError( parsed.error.issues[0]?.message ?? "Invalid import payload", @@ -132,13 +52,23 @@ export const POST = apiHandler(async (request: NextRequest) => { ); } - const { stats: written, writeFailed } = await runJsonImport({ + const { + stats: written, + writeFailed, + droppedLinkKeys, + } = await runJsonImport({ userId: user.id, measurements: parsed.data.measurements, moodEntries: parsed.data.moodEntries, ipAddress: getClientIp(request), }); - const stats = { ...written, skipped: written.skipped + normalized.skipped }; + const stats = { + ...written, + skipped: written.skipped + normalized.skipped, + // The mood links the account could not resolve, by key, only when there + // are any: "imported" alone told the person their tags had landed. + ...droppedLinkKeysForWire(droppedLinkKeys), + }; if (writeFailed) { return apiError("Import write failed", 503, { diff --git a/src/components/settings/import-panel/__tests__/json-import-result.test.tsx b/src/components/settings/import-panel/__tests__/json-import-result.test.tsx new file mode 100644 index 000000000..64825e985 --- /dev/null +++ b/src/components/settings/import-panel/__tests__/json-import-result.test.tsx @@ -0,0 +1,54 @@ +/** + * What the JSON import card says once a file is in. + * + * SSR-only, like the sibling card tests: the markup each result renders. The + * claim worth pinning is the one the counts cannot make. A mood entry whose + * tag the account does not have lands without that link, and the card names + * the tag rather than reporting a clean import. + */ +import { describe, expect, it } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { I18nProvider } from "@/lib/i18n/context"; + +import { + JsonImportResultView, + readJsonImportResult, +} from "../json-import-card"; + +function render(data: Parameters[0]) { + return renderToStaticMarkup( + + + , + ); +} + +describe("", () => { + it("names the tags an import could not link", () => { + const html = render({ + measurements: 3, + moodEntries: 2, + skipped: 0, + droppedTagKeys: ["custom:long_walk"], + droppedFactorKeys: ["retired_factor"], + }); + expect(html).toContain('data-testid="import-json-dropped-links"'); + expect(html).toContain("custom:long_walk, retired_factor"); + expect(html).toContain("Imported 3 measurements and 2 mood entries"); + }); + + it("says nothing about links when every one landed", () => { + const html = render({ measurements: 3, moodEntries: 2, skipped: 0 }); + expect(html).not.toContain("import-json-dropped-links"); + }); + + it("reads a missing count as none", () => { + expect(readJsonImportResult(null)).toEqual({ + measurements: 0, + moodEntries: 0, + skipped: 0, + droppedLinkKeys: [], + }); + }); +}); diff --git a/src/components/settings/import-panel/json-import-card.tsx b/src/components/settings/import-panel/json-import-card.tsx index ca19663cd..f64e6b7bc 100644 --- a/src/components/settings/import-panel/json-import-card.tsx +++ b/src/components/settings/import-panel/json-import-card.tsx @@ -18,45 +18,90 @@ import { import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line"; import { classifyWrittenOutcome } from "@/lib/outcome/written-outcome"; -interface JsonImportResult { +export interface JsonImportResult { measurements: number; moodEntries: number; skipped: number; + /** + * Tag and factor keys of imported mood entries this account has no tag + * for (another account's own tags, mostly). The entries landed without + * those links, which the count alone would not say. + */ + droppedLinkKeys: string[]; } -export function JsonImportCard() { +/** The route's `data`, read field by field; a missing count reads as none. */ +export function readJsonImportResult( + data: { + measurements?: number; + moodEntries?: number; + skipped?: number; + droppedTagKeys?: string[]; + droppedFactorKeys?: string[]; + } | null, +): JsonImportResult { + return { + measurements: data?.measurements ?? 0, + moodEntries: data?.moodEntries ?? 0, + skipped: data?.skipped ?? 0, + droppedLinkKeys: [ + ...(data?.droppedTagKeys ?? []), + ...(data?.droppedFactorKeys ?? []), + ], + }; +} + +/** What an import wrote, and the links it could not make. */ +export function JsonImportResultView({ result }: { result: JsonImportResult }) { const { t } = useTranslations(); // Same reasoning as the CSV card: the JSON route answers 200 with a per- // concept count envelope, so a payload where every entry was refused used // to render the success tick. The counts decide the tone. - const jsonOutcome = (r: JsonImportResult) => - classifyWrittenOutcome({ - written: r.measurements + r.moodEntries, - skipped: r.skipped, - }); - const jsonMessage = (r: JsonImportResult) => { - const outcome = jsonOutcome(r); - if (outcome === "empty") { - return t("settings.sections.export.import.json.resultEmpty"); - } - if (outcome === "failed") { - return t("settings.sections.export.import.json.resultNothing", { - skipped: r.skipped, - }); - } - if (outcome === "partial") { - return t("settings.sections.export.import.json.resultSummary", { - measurements: r.measurements, - moods: r.moodEntries, - skipped: r.skipped, - }); - } - return t("settings.sections.export.import.json.resultSuccess", { - measurements: r.measurements, - moods: r.moodEntries, - }); - }; + const outcome = classifyWrittenOutcome({ + written: result.measurements + result.moodEntries, + skipped: result.skipped, + }); + const message = + outcome === "empty" + ? t("settings.sections.export.import.json.resultEmpty") + : outcome === "failed" + ? t("settings.sections.export.import.json.resultNothing", { + skipped: result.skipped, + }) + : outcome === "partial" + ? t("settings.sections.export.import.json.resultSummary", { + measurements: result.measurements, + moods: result.moodEntries, + skipped: result.skipped, + }) + : t("settings.sections.export.import.json.resultSuccess", { + measurements: result.measurements, + moods: result.moodEntries, + }); + return ( + <> + + {result.droppedLinkKeys.length > 0 && ( +

+ {t("settings.sections.export.import.json.droppedLinks", { + keys: result.droppedLinkKeys.join(", "), + })} +

+ )} + + ); +} + +export function JsonImportCard() { + const { t } = useTranslations(); const fileInputRef = useRef(null); const [text, setText] = useState(""); @@ -112,12 +157,7 @@ export function JsonImportCard() { setError(t("settings.sections.export.import.json.failed")); return; } - const data = (await res.json()).data as JsonImportResult; - setResult({ - measurements: data?.measurements ?? 0, - moodEntries: data?.moodEntries ?? 0, - skipped: data?.skipped ?? 0, - }); + setResult(readJsonImportResult((await res.json()).data ?? null)); } catch { setError(t("settings.sections.export.import.json.failed")); } finally { @@ -180,13 +220,7 @@ export function JsonImportCard() {

- {result && ( - - )} + {result && } {error && (

; - value: number; - unit: string; - measuredAt: Date; - glucoseContext?: z.infer; - source?: string; - notes?: string; - externalId?: string; +export type { JsonImportMeasurement } from "./schema"; + +/** + * A reading's own columns beyond its value: what the reading says, carried + * whole so a range keeps its bounds, a sleep row its stage and an ECG its + * verdict. Every arm of the write states them, so a re-import of the same + * file puts back what an edit in between changed. + */ +function readingColumns(m: JsonImportMeasurement) { + return { + value: m.value, + valueMin: m.valueMin ?? null, + valueMax: m.valueMax ?? null, + unit: m.unit, + measuredAt: m.measuredAt, + glucoseContext: m.glucoseContext ?? null, + sleepStage: m.sleepStage ?? null, + rhythmClassification: m.rhythmClassification ?? null, + deviceType: m.deviceType ?? null, + notes: null, + notesEncrypted: encryptNote(m.notes || null), + }; } export interface ImportedMeasurements { @@ -72,8 +80,8 @@ export async function importJsonMeasurements( try { // `measuredAt` is already a `Date` (validateEntryInstant transform). const measuredAt = m.measuredAt; + const columns = readingColumns(m); if (m.externalId) { - const notesEncrypted = encryptNote(m.notes || null); // INSERT ... RETURNING is the only insertion proof. A concurrent // importer can win this key after any existence probe, so attempt // the insert first and update only when the statement returns no row. @@ -82,14 +90,9 @@ export async function importJsonMeasurements( { userId, type: m.type, - value: m.value, - unit: m.unit, source: "IMPORT", externalId: m.externalId, - measuredAt, - notes: null, - notesEncrypted, - glucoseContext: m.glucoseContext ?? null, + ...columns, }, ], skipDuplicates: true, @@ -112,14 +115,7 @@ export async function importJsonMeasurements( externalId: m.externalId, }, }, - data: { - value: m.value, - unit: m.unit, - measuredAt, - notes: null, - notesEncrypted, - glucoseContext: m.glucoseContext ?? null, - }, + data: columns, }); } catch (err) { if (isP2025(err)) { @@ -134,13 +130,8 @@ export async function importJsonMeasurements( data: { userId, type: m.type, - value: m.value, - unit: m.unit, source: "IMPORT", - measuredAt, - notes: null, - notesEncrypted: encryptNote(m.notes || null), - glucoseContext: m.glucoseContext ?? null, + ...columns, }, select: { id: true, type: true, measuredAt: true }, }); diff --git a/src/lib/import/json/mood-instant.ts b/src/lib/import/json/mood-instant.ts new file mode 100644 index 000000000..e07d29e57 --- /dev/null +++ b/src/lib/import/json/mood-instant.ts @@ -0,0 +1,23 @@ +/** + * The day bound of a JSON-imported mood entry without a logged instant, on + * its own so the import schema can bound such a row without reaching the + * writer and the database client behind it. + */ +import { moodDateKey } from "@/lib/mood/date-key"; +import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; +import { zonedWallClockToUtc } from "@/lib/tz/wall-clock"; + +/** + * The start of a `YYYY-MM-DD` day in the default zone, or null when the + * string does not name a real day. The route's schema uses it to bound a row + * without a logged instant. + */ +export function fallbackMoodDateStart(date: string): Date | null { + const [year, month, day] = date.split("-").map(Number); + if (!year || !month || !day) return null; + const instant = zonedWallClockToUtc( + { year, month, day, hour: 0, minute: 0 }, + DEFAULT_TIMEZONE, + ); + return moodDateKey(instant, DEFAULT_TIMEZONE) === date ? instant : null; +} diff --git a/src/lib/import/json/mood.ts b/src/lib/import/json/mood.ts index 675292a43..e706b6b62 100644 --- a/src/lib/import/json/mood.ts +++ b/src/lib/import/json/mood.ts @@ -6,43 +6,34 @@ * derived from the row's content and its occurrence in the file, so the same * file lands on the same instants every time and twin rows land on distinct * ones. + * + * Each entry is written with its note, its level-A values, its zone, its day + * context and its tag and factor links, in one transaction, the way the bulk + * mood ingest writes one: a link that throws (a rating outside its factor's + * own scale) takes the entry back with it, so a skipped row left nothing + * behind. */ import { createHash } from "node:crypto"; +import type { Prisma } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { encryptNote } from "@/lib/crypto/note-cipher"; import { ENTRY_INSTANT_CLOCK_SKEW_MS } from "@/lib/validations/entry-instant"; import { classifyExternalId } from "@/lib/validations/external-id"; import { isP2002 } from "@/lib/prisma-errors"; -import { moodDateKey } from "@/lib/mood/date-key"; -import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; -import { deriveA1 } from "@/lib/mood/level-a"; -import { zonedWallClockToUtc } from "@/lib/tz/wall-clock"; +import { levelAForWrite } from "@/lib/mood/level-a"; +import { persistMoodContext } from "@/lib/mood/context"; +import { + RatedFactorOutOfRangeError, + replaceRatedFactorLinks, + replaceTagLinks, +} from "@/lib/mood/tag-links"; +import { fallbackMoodDateStart } from "./mood-instant"; +import type { JsonImportMoodEntry } from "./schema"; import type { JsonImportTally } from "./tally"; -export interface JsonImportMoodEntry { - date: string; - mood: "SUPER_GUT" | "GUT" | "OKAY" | "SCHLECHT" | "LAUSIG"; - score: number; - tags?: string; - loggedAt?: Date; - externalId?: string; -} - -/** - * The start of a `YYYY-MM-DD` day in the default zone, or null when the - * string does not name a real day. The route's schema uses it to bound a row - * without a logged instant. - */ -export function fallbackMoodDateStart(date: string): Date | null { - const [year, month, day] = date.split("-").map(Number); - if (!year || !month || !day) return null; - const instant = zonedWallClockToUtc( - { year, month, day, hour: 0, minute: 0 }, - DEFAULT_TIMEZONE, - ); - return moodDateKey(instant, DEFAULT_TIMEZONE) === date ? instant : null; -} +export type { JsonImportMoodEntry } from "./schema"; function fallbackMoodIdentity(entry: JsonImportMoodEntry): string { return JSON.stringify([ @@ -75,65 +66,109 @@ interface FallbackState { reserved: Set; } +/** + * The entry's own columns, as both arms of the write state them. The level-A + * values follow `levelAForWrite`: the file's `a1` wins over the one derived + * from the label, and an `a2` to `a5` the file leaves out is left alone on a + * row the import updates rather than blanked. + */ +function moodEntryColumns(e: JsonImportMoodEntry, loggedAt: Date) { + return { + date: e.date, + mood: e.mood, + score: e.score, + ...levelAForWrite(e.mood, { + a1: e.a1, + a2: e.a2, + a3: e.a3, + a4: e.a4, + a5: e.a5, + }), + tags: e.tags || null, + note: null, + noteEncrypted: encryptNote(e.note ?? null), + ...(e.tz === undefined ? {} : { tz: e.tz }), + moodLoggedAt: loggedAt, + }; +} + +/** Link keys an entry's write could not resolve. */ +interface DroppedKeys { + tags: Set; + factors: Set; +} + +/** The entry's links and day context, when the file states them. */ +async function writeMoodEntryParts( + tx: Prisma.TransactionClient, + userId: string, + moodEntryId: string, + e: JsonImportMoodEntry, + dropped: DroppedKeys, +): Promise { + await persistMoodContext(tx, moodEntryId, userId, e.context); + if (e.factors !== undefined) { + const keys = await replaceRatedFactorLinks( + moodEntryId, + userId, + e.factors, + tx, + ); + for (const key of keys) dropped.factors.add(key); + } + if (e.structuredTags !== undefined) { + const keys = await replaceTagLinks( + moodEntryId, + userId, + e.structuredTags, + tx, + ); + for (const key of keys) dropped.tags.add(key); + } +} + async function writeMoodEntry( userId: string, e: JsonImportMoodEntry, loggedAt: Date, + tally: JsonImportTally, ): Promise { - if (e.externalId) { + const columns = moodEntryColumns(e, loggedAt); + // Links resolve against what the importing account can see. Collected + // here and handed to the tally only once the transaction committed, so a + // rolled-back entry reports nothing. + const dropped: DroppedKeys = { tags: new Set(), factors: new Set() }; + await prisma.$transaction(async (tx) => { // Idempotent re-import keyed on the source-stable id. A second import // of the same export updates the row in place rather than skipping it as // a duplicate, so an upstream edit (re-scored mood, added tag) is // reflected. - await prisma.moodEntry.upsert({ - where: { - userId_source_externalId: { - userId, - source: "IMPORT", - externalId: e.externalId, - }, - }, - update: { - date: e.date, - mood: e.mood, - score: e.score, - // Pleasantness derived from the row's own label, not from the file's - // `score`. This is the one writer whose score comes out of the file - // rather than out of the label, so the two can disagree; the column - // that records a self-assessment has to agree with the label the - // user actually picked. - moodA1: deriveA1(e.mood), - tags: e.tags || null, - moodLoggedAt: loggedAt, - }, - create: { - userId, - date: e.date, - mood: e.mood, - score: e.score, - moodA1: deriveA1(e.mood), - tags: e.tags || null, - source: "IMPORT", - externalId: e.externalId, - moodLoggedAt: loggedAt, - }, - }); - } else { - await prisma.moodEntry.create({ - data: { - userId, - date: e.date, - mood: e.mood, - score: e.score, - // See the upsert arm above: derived from the label, because an - // imported file's `score` is its own field. - moodA1: deriveA1(e.mood), - tags: e.tags || null, - source: "IMPORT", - moodLoggedAt: loggedAt, - }, - }); - } + const row = e.externalId + ? await tx.moodEntry.upsert({ + where: { + userId_source_externalId: { + userId, + source: "IMPORT", + externalId: e.externalId, + }, + }, + update: columns, + create: { + userId, + ...columns, + source: "IMPORT", + externalId: e.externalId, + }, + select: { id: true }, + }) + : await tx.moodEntry.create({ + data: { userId, ...columns, source: "IMPORT" }, + select: { id: true }, + }); + await writeMoodEntryParts(tx, userId, row.id, e, dropped); + }); + for (const key of dropped.tags) tally.droppedTagKeys.add(key); + for (const key of dropped.factors) tally.droppedFactorKeys.add(key); } /** @@ -199,13 +234,19 @@ async function importMoodEntry( continue; } try { - await writeMoodEntry(userId, e, loggedAt); + await writeMoodEntry(userId, e, loggedAt, tally); if (!e.loggedAt) { fallback.reserved.add(loggedAt.getTime()); } tally.stats.moodEntries++; return; } catch (err) { + // A rating outside its factor's own scale refuses that one entry, the + // way the bulk ingest refuses it; the transaction took the row back. + if (err instanceof RatedFactorOutOfRangeError) { + tally.stats.skipped++; + return; + } if (!isP2002(err)) { tally.writeFailed = true; return; diff --git a/src/lib/import/json/run.ts b/src/lib/import/json/run.ts index 1f965c4a8..572b21cca 100644 --- a/src/lib/import/json/run.ts +++ b/src/lib/import/json/run.ts @@ -11,6 +11,7 @@ import { recomputeUserMoodRollups } from "@/lib/rollups/mood-rollups"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { maybeEnqueueMorningRefresh } from "@/lib/daily/morning-refresh-trigger"; +import type { DroppedLinkKeys } from "@/lib/mood/tag-links"; import { importJsonMeasurements, @@ -24,7 +25,11 @@ export async function runJsonImport(input: { measurements: JsonImportMeasurement[] | undefined; moodEntries: JsonImportMoodEntry[] | undefined; ipAddress: string | null; -}): Promise> { +}): Promise< + Pick & { + droppedLinkKeys: DroppedLinkKeys; + } +> { const { userId } = input; const tally = newJsonImportTally(); @@ -80,6 +85,8 @@ export async function runJsonImport(input: { annotate({ meta: { import_stats: tally.stats, + import_dropped_link_keys: + tally.droppedTagKeys.size + tally.droppedFactorKeys.size, ...(tally.unstableShapes.length > 0 ? unstableExternalIdMeta("import.json", tally.unstableShapes) : {}), @@ -97,5 +104,12 @@ export async function runJsonImport(input: { annotate({ meta: { import_audit_failed: true } }); } - return { stats: tally.stats, writeFailed: tally.writeFailed }; + return { + stats: tally.stats, + writeFailed: tally.writeFailed, + droppedLinkKeys: { + droppedTagKeys: [...tally.droppedTagKeys].sort(), + droppedFactorKeys: [...tally.droppedFactorKeys].sort(), + }, + }; } diff --git a/src/lib/import/json/schema.ts b/src/lib/import/json/schema.ts new file mode 100644 index 000000000..aea967780 --- /dev/null +++ b/src/lib/import/json/schema.ts @@ -0,0 +1,184 @@ +/** + * The schema of `POST /api/import`, one file of the app's own JSON back in. + * + * Two of the app's files feed it: the full backup's portable half and the + * JSON export (both read in their own terms first, `normalize.ts`). What a + * reading or a mood entry says in either file has a field here, and + * `json-import-field-coverage.test.ts` holds the two exporters to it: a field + * either exporter writes is either read below or named there with the reason + * the import leaves it out. + * + * Lives here rather than in the route because the route module may export + * nothing but its handlers, and the guard needs the field set. + */ +import { z } from "zod/v4"; + +import { RhythmClassification } from "@/generated/prisma/enums"; +import { decodeKeyList } from "@/lib/mood/context-vocabulary"; +import { isValidTimezone } from "@/lib/tz/format"; +import { isoDateTime } from "@/lib/validations/datetime"; +import { + isPlausibleEntryInstant, + validateEntryInstant, +} from "@/lib/validations/entry-instant"; +import { + glucoseContextEnum, + measurementTypeEnum, + validateMeasurementRange, +} from "@/lib/validations/measurement"; +import { moodContextSchema, moodLevelEnum } from "@/lib/validations/mood"; +import { SLEEP_STAGE_NAMES } from "@/lib/validations/sleep-stage-entry"; + +import { fallbackMoodDateStart } from "./mood-instant"; + +/** Rows per section, and the file as a whole, as the route has always capped them. */ +export const JSON_IMPORT_MAX_ROWS = 10_000; +export const JSON_IMPORT_MAX_BYTES = 16 * 1024 * 1024; + +// The type is the canonical enum, so every type the exporters write comes +// back; a hand-kept subset once dropped four of eleven. +export const jsonImportMeasurementSchema = z + .object({ + type: measurementTypeEnum, + value: z.number(), + // A range reading's bounds (an hourly heart-rate statistic's min and + // max). Without them the reading comes back as a single value. + valueMin: z.number().finite().optional(), + valueMax: z.number().finite().optional(), + unit: z.string(), + // The canonical entry-instant bound the single POST and edit paths use, + // so a crafted file cannot forward-date a reading. `validateEntryInstant` + // rejects any future instant beyond a 5-min skew tolerance and any + // instant before 1900; the field transforms to a `Date`. + measuredAt: validateEntryInstant( + isoDateTime().transform((s) => new Date(s)), + ), + glucoseContext: glucoseContextEnum.optional(), + // A sleep row's stage. Without it a night of stages comes back as that + // many undifferentiated sleep readings. + sleepStage: z.enum(SLEEP_STAGE_NAMES).optional(), + // A device's own verdict on an ECG, a rhythm alert or a steadiness + // score: the reading's meaning, not a detail of it. + rhythmClassification: z.enum(RhythmClassification).optional(), + // Which device wrote the reading (watch, phone). The per-night sleep + // collapse picks between writers by it. + deviceType: z.string().min(1).max(32).optional(), + notes: z.string().optional(), + // Optional source-stable id. When present, the import upserts on + // `(userId, type, source=IMPORT, externalId)` so a re-import of the same + // export is idempotent rather than minting a duplicate set — mirrors the + // mood path. Absent → first-write-wins create with a NULL externalId + // (distinct in the unique key), which DOES duplicate on re-import. + // Stability is enforced per entry in the import loop, not at the + // field: a schema-level refusal would fail the whole file on one bad + // row. + // @external-id-checked-per-entry: src/lib/import/json/measurements.ts + externalId: z.string().min(1).max(120).optional(), + }) + .refine( + (data) => validateMeasurementRange(data.type, data.value) === null, + "Wert ausserhalb des plausiblen Bereichs", + ) + .refine( + (data) => + data.valueMin === undefined || + data.valueMax === undefined || + data.valueMin <= data.valueMax, + { message: "valueMin must be <= valueMax", path: ["valueMin"] }, + ); + +/** + * A multi-select of the day context. The backup writes it as the stored JSON + * string, unparsed; every other writer sends the list. Both read as the list. + */ +function keyList(list: T) { + return z.preprocess( + (value) => (typeof value === "string" ? decodeKeyList(value) : value), + list, + ); +} + +export const jsonImportMoodContextSchema = moodContextSchema.extend({ + contactCircles: keyList(moodContextSchema.shape.contactCircles), + leisureCategories: keyList(moodContextSchema.shape.leisureCategories), +}); + +const levelADimension = z.number().int().min(0).max(10); + +export const jsonImportMoodEntrySchema = z + .object({ + date: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), + mood: moodLevelEnum, + score: z.number().int().min(1).max(5), + // The five level-A values. `a1` is derived from `mood` when the file + // leaves it out; the other four are only ever what the person answered. + a1: levelADimension.optional(), + a2: levelADimension.optional(), + a3: levelADimension.optional(), + a4: levelADimension.optional(), + a5: levelADimension.optional(), + tags: z.string().optional(), + // Capped like every other mood note. + note: z.string().max(500).optional(), + // The same entry-instant bound as measurements; an imported mood log + // cannot be forward-dated. Optional, and a `Date` when set. + loggedAt: validateEntryInstant( + isoDateTime().transform((s) => new Date(s)), + ).optional(), + // The zone the `date` string is anchored to. Absent, the entry keeps the + // legacy reading, which is what an entry without one always meant. + tz: z + .string() + .max(64) + .refine(isValidTimezone, { message: "Unknown time zone" }) + .optional(), + // The entry's rated factors and ticked tags, by catalogue key. A key the + // importing account cannot resolve costs that one link, and is counted. + factors: z + .array( + z.object({ + key: z.string().max(60), + rating: z.number().int().min(1).max(5), + }), + ) + .max(30) + .optional(), + structuredTags: z.array(z.string().max(60)).max(30).optional(), + // The day context. Absent says nothing about it; a file that carries one + // replaces whatever the entry held. + context: jsonImportMoodContextSchema.optional(), + // Optional source-stable id (e.g. a Daylio row id). When + // present, the import upserts on `(userId, source, externalId)` so a + // re-import of the same export is idempotent rather than minting a + // duplicate; absent → the legacy first-write-wins create path. + // Stability is enforced per entry in the import loop, not at the + // field: a schema-level refusal would fail the whole file on one bad + // row. + // @external-id-checked-per-entry: src/lib/import/json/mood.ts + externalId: z.string().min(1).max(120).optional(), + }) + .superRefine((entry, ctx) => { + if (entry.loggedAt) return; + const dateStart = fallbackMoodDateStart(entry.date); + if (!dateStart || !isPlausibleEntryInstant(dateStart)) { + ctx.addIssue({ + code: "custom", + path: ["date"], + message: "Mood date must be between 1900 and today", + }); + } + }); + +export const jsonImportSchema = z.object({ + measurements: z + .array(jsonImportMeasurementSchema) + .max(JSON_IMPORT_MAX_ROWS) + .optional(), + moodEntries: z + .array(jsonImportMoodEntrySchema) + .max(JSON_IMPORT_MAX_ROWS) + .optional(), +}); + +export type JsonImportMeasurement = z.infer; +export type JsonImportMoodEntry = z.infer; diff --git a/src/lib/import/json/tally.ts b/src/lib/import/json/tally.ts index 243150f06..9110417a7 100644 --- a/src/lib/import/json/tally.ts +++ b/src/lib/import/json/tally.ts @@ -4,6 +4,13 @@ import type { UnstableExternalIdShape } from "@/lib/validations/external-id"; export interface JsonImportTally { stats: { measurements: number; moodEntries: number; skipped: number }; unstableShapes: UnstableExternalIdShape[]; + /** + * Link keys of imported mood entries that the importing account cannot + * resolve (a custom tag of the account the file came from, a retired + * catalogue key). The entry lands without that link; the key is named. + */ + droppedTagKeys: Set; + droppedFactorKeys: Set; /** A write failed for a reason other than a duplicate; the import stops. */ writeFailed: boolean; } @@ -12,6 +19,8 @@ export function newJsonImportTally(): JsonImportTally { return { stats: { measurements: 0, moodEntries: 0, skipped: 0 }, unstableShapes: [], + droppedTagKeys: new Set(), + droppedFactorKeys: new Set(), writeFailed: false, }; } diff --git a/src/lib/openapi/routes/import.ts b/src/lib/openapi/routes/import.ts index a9fc17f68..871fa2d04 100644 --- a/src/lib/openapi/routes/import.ts +++ b/src/lib/openapi/routes/import.ts @@ -16,6 +16,9 @@ import { z } from "zod/v4"; import { glucoseContextEnum } from "@/lib/validations/measurement"; import { moodLevelEnum } from "@/lib/validations/mood"; +import { SLEEP_STAGE_NAMES } from "@/lib/validations/sleep-stage-entry"; +import { RhythmClassification } from "@/generated/prisma/enums"; +import { moodContextWire } from "./mood"; import { dataEnvelope, errorEnvelope, @@ -94,10 +97,10 @@ const csvImportResponse = z "Per-row import outcome. Mirrors the batch-route per-entry envelope. In `dryRun` mode no write runs and every valid row reports its projected `inserted` status.", }); -// ── JSON import — mirrors the route-local `importSchema` -// (`src/app/api/import/route.ts`), which is deliberately not exported: the -// runtime side carries transforms + plausibility refinements the document -// restates in prose. +// ── JSON import — mirrors `jsonImportSchema` +// (`src/lib/import/json/schema.ts`). The runtime side carries transforms and +// plausibility refinements the document restates in prose; its field set is +// held to this one by `json-import-field-coverage.test.ts`. const jsonImportMeasurement = z .object({ @@ -107,6 +110,13 @@ const jsonImportMeasurement = z .describe( "Checked against the type's server-side plausibility range; an out-of-range value fails the whole request with 422.", ), + valueMin: z + .number() + .optional() + .describe( + "Lower bound of a range reading (an hourly heart-rate statistic). Not above `valueMax`.", + ), + valueMax: z.number().optional().describe("Upper bound of a range reading."), unit: z.string(), measuredAt: isoDateTime().describe( "Bounded: no future instants beyond a 5-min clock-skew tolerance, nothing before 1900.", @@ -116,12 +126,22 @@ const jsonImportMeasurement = z .describe( "Optional even on a BLOOD_GLUCOSE row — a sensor export classifies nothing per sample.", ), - source: z - .string() + sleepStage: z + .enum(SLEEP_STAGE_NAMES) + .optional() + .describe("A sleep row's stage."), + rhythmClassification: z + .enum(RhythmClassification) .optional() .describe( - "Accepted for export-format compatibility and ignored: every imported row stores `source = IMPORT`.", + "The device's own verdict on an ECG, a rhythm alert or a steadiness reading.", ), + deviceType: z + .string() + .min(1) + .max(32) + .optional() + .describe("The writer of the reading (watch, phone)."), notes: z.string().optional().describe("Encrypted at rest."), externalId: z .string() @@ -150,17 +170,79 @@ const jsonImportMoodEntry = z .min(1) .max(5) .describe( - "Stored as sent — the export format's own field. Pleasantness (`a1`) is re-derived from `mood` on every imported row, never taken from this.", + "Stored as sent — the export format's own field. Pleasantness (`a1`) is never taken from this.", + ), + a1: z + .number() + .int() + .min(0) + .max(10) + .optional() + .describe("Pleasantness, 0–10. Absent → derived from `mood`."), + a2: z.number().int().min(0).max(10).optional().describe("Stress, 0–10."), + a3: z.number().int().min(0).max(10).optional().describe("Energy, 0–10."), + a4: z + .number() + .int() + .min(0) + .max(10) + .optional() + .describe("Connection, 0–10."), + a5: z + .number() + .int() + .min(0) + .max(10) + .optional() + .describe( + "Stability, 0–10. `a2` to `a5` absent leave a row the import updates as it was.", ), tags: z .string() .optional() .describe("JSON-encoded array of tag strings, stored verbatim."), + note: z + .string() + .max(500) + .optional() + .describe("The entry's note. Encrypted at rest."), loggedAt: isoDateTime() .optional() .describe( "Bounded like `measuredAt`. Absent → the server derives a stable per-row instant within the `date` day, so a re-import of the same file dedups instead of duplicating.", ), + tz: z + .string() + .max(64) + .optional() + .describe( + "IANA zone the `date` is anchored to. Absent → the legacy reading.", + ), + factors: z + .array( + z.object({ + key: z.string().max(60), + rating: z.number().int().min(1).max(5), + }), + ) + .max(30) + .optional() + .describe( + "Rated factors by catalogue key; replaces the entry's rated links. A key the account cannot resolve is dropped and named in `droppedFactorKeys`; a rating outside the factor's own scale skips the entry.", + ), + structuredTags: z + .array(z.string().max(60)) + .max(30) + .optional() + .describe( + "Ticked tags by key (the catalogue or the account's own); replaces the entry's tag links. An unresolvable key is dropped and named in `droppedTagKeys`.", + ), + context: moodContextWire + .partial() + .optional() + .describe( + "The day context; replaces the entry's context. `contactCircles` and `leisureCategories` are also read as the backup's JSON-encoded string.", + ), externalId: z .string() .min(1) @@ -183,7 +265,7 @@ const jsonImportRequest = z .meta({ id: "JsonImportRequest", description: - "JSON import payload, matching the structure `GET /api/export/json` emits. Both sections optional; up to 10 000 rows each.", + "JSON import payload: the measurements and mood entries of the JSON export or the portable full backup. Both sections optional; up to 10 000 rows each.", }); const jsonImportStats = z @@ -203,7 +285,19 @@ const jsonImportStats = z .int() .nonnegative() .describe( - "Rows not written: duplicates of existing rows, and rows carrying an unstable `externalId`.", + "Rows not written: duplicates of existing rows, rows carrying an unstable `externalId`, mood entries with a factor rating outside its own scale, and a backup's deletion records.", + ), + droppedTagKeys: z + .array(z.string()) + .optional() + .describe( + "Tag keys of imported mood entries this account cannot resolve (another account's custom tag, a retired key). The entries landed without them. Present only when non-empty.", + ), + droppedFactorKeys: z + .array(z.string()) + .optional() + .describe( + "Rated-factor keys this account cannot resolve, as `droppedTagKeys`.", ), }) .meta({ @@ -425,7 +519,7 @@ export const importPaths: NonNullable = { tags: ["Import"], summary: "Import measurements and mood entries from JSON", description: - "Restores a JSON export (the structure `GET /api/export/json` emits): up to 10 000 measurements plus 10 000 mood entries per call, body ≤ 16 MB. Rows carrying an `externalId` upsert idempotently (re-import updates in place); rows without one are first-write-wins and dedup against the existing unique keys, so an exact duplicate is `skipped` rather than doubled. A row with an unstable `externalId` is skipped while the rest of the file still lands. Every imported measurement stores `source = IMPORT`; every imported mood entry re-derives pleasantness from its label. Writes are per-row, not transactional: on a mid-file write failure the response is 503 (`meta.errorCode` = `import.write_failed`, `meta.retryable` = true) with `meta.stats` carrying what landed — rows already written stay written, and a retry is only safe against duplication for rows that carry an `externalId`. Validation is fail-fast (the 422 carries the first issue's message, not the multi-issue envelope). Rate-limited 5/hour per user (shared `import:` bucket with the CSV importer).", + "Imports the app's own files back: the JSON export (`GET /api/export?format=json`, read inside its `data` envelope) or the portable full backup (`GET /api/export/full-backup`). Every field either file writes for a reading or a mood entry is carried, apart from the row ids, the source and the sync bookkeeping; a backup's deletion records are skipped and counted. Up to 10 000 measurements plus 10 000 mood entries per call, body ≤ 16 MB. Rows carrying an `externalId` upsert idempotently (re-import updates in place); rows without one are first-write-wins and dedup against the existing unique keys, so an exact duplicate is `skipped` rather than doubled. A row with an unstable `externalId` is skipped while the rest of the file still lands. Every imported row stores `source = IMPORT`; a mood entry without `a1` derives pleasantness from its label. Writes are per-row, not transactional: on a mid-file write failure the response is 503 (`meta.errorCode` = `import.write_failed`, `meta.retryable` = true) with `meta.stats` carrying what landed — rows already written stay written, and a retry is only safe against duplication for rows that carry an `externalId`. Validation is fail-fast (the 422 carries the first issue's message, not the multi-issue envelope). Rate-limited 5/hour per user (shared `import:` bucket with the CSV importer).", requestBody: { required: true, content: { "application/json": { schema: jsonImportRequest } }, diff --git a/tests/integration/json-import-field-coverage.test.ts b/tests/integration/json-import-field-coverage.test.ts new file mode 100644 index 000000000..69bcf01fb --- /dev/null +++ b/tests/integration/json-import-field-coverage.test.ts @@ -0,0 +1,262 @@ +/** + * Every field the app's exporters write for a reading or a mood entry is + * either read by the JSON import or named here with the reason it is not. + * + * The same kind of guard as `measurement-backup-completeness.test.ts` and + * `mood-backup-completeness.test.ts`, one step further down the pipe: those + * prove a column reaches the file, this one proves a field of the file reaches + * the import. The JSON import used to read six measurement fields and six + * mood fields while the files carried twice that, so a sleep night came back + * as stageless rows, a range as a single value, a mood entry without its note, + * its sliders, its links or its day. + * + * The field sets are read from what the exporters actually write, for an + * account with every column filled in (`json-import-seed.ts`), so a field an + * exporter learns to write appears here the release it does. Each file is + * walked in both of its forms: the JSON export, the portable full backup and + * the disaster-recovery backup. + * + * The reasons are checked the other way too: a reason for a field no exporter + * writes any more is stale and fails, so the list cannot grow into a list of + * things that were once true. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { GET as exportJson } from "@/app/api/export/route"; +import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; +import { + jsonImportMeasurementSchema, + jsonImportMoodContextSchema, + jsonImportMoodEntrySchema, +} from "@/lib/import/json/schema"; + +import { callRoute } from "../kit/call-route"; +import { makeSession } from "../kit/factories"; +import { + seedJsonImportCatalogue, + seedJsonImportOwner, +} from "./json-import-seed"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +const prisma = getPrismaClient(); + +/** Why a field of an exported row is not a field of the imported one. */ +const ROW_ID = + "The import writes new rows into the importing account; the file's row id names a row of the account it came from."; +const SOURCE = + "Every imported row is stamped IMPORT. Source priority and the sync paths key on the source, and a row claiming an integration would be taken for that integration's own."; +const DELETED_AT = + "A deletion record, not a reading: the import skips the row and counts it (`normalize.ts`)."; +const CIPHERTEXT = + "Ciphertext under the exporting instance's key (disaster-recovery file only). The portable file carries the same note as text, and that is what the import reads."; +const BOOKKEEPING = + "Sync bookkeeping of the account the file came from (disaster-recovery file only). It describes that account's history with its sources, not the reading."; +const ROW_STAMP = + "The row's own write stamp (disaster-recovery file only); an imported row is written now."; + +const MEASUREMENT_NOT_IMPORTED: Record = { + id: ROW_ID, + source: SOURCE, + deletedAt: DELETED_AT, + notesEncrypted: CIPHERTEXT, + externalSourceVersion: BOOKKEEPING, + aggregationProvenance: BOOKKEEPING, + ingestTokenId: + "The API token that wrote the reading on the exporting account; no token of that account exists in the importing one.", + syncVersion: BOOKKEEPING, + createdAt: ROW_STAMP, + updatedAt: ROW_STAMP, +}; + +const MOOD_NOT_IMPORTED: Record = { + id: ROW_ID, + source: SOURCE, + deletedAt: DELETED_AT, + noteEncrypted: CIPHERTEXT, + syncedAt: BOOKKEEPING, + syncVersion: BOOKKEEPING, + createdAt: ROW_STAMP, + updatedAt: ROW_STAMP, + linkCreatedAt: ROW_STAMP, +}; + +const CONTEXT_NOT_IMPORTED: Record = { + notesEncrypted: CIPHERTEXT, + createdAt: ROW_STAMP, + updatedAt: ROW_STAMP, +}; + +/** + * The files' other sections. The JSON import carries readings and mood + * entries into an account that has its own record already; an account's + * whole record (medications, labs, documents, settings, …) comes back through + * the backup restore, which replaces it. The mood vocabulary is named apart + * because the entries' links point into it: a link to a tag the importing + * account does not have is dropped and named in the response. + */ +const IMPORTED_SECTIONS = new Set(["measurements", "moodEntries"]); +const VOCABULARY_SECTIONS: Record = { + customMoodTags: + "Account vocabulary, restored by the backup restore. An imported link to one of these tags resolves only when the importing account has a tag of that key, and is otherwise dropped and named (`droppedTagKeys`).", + customMoodTagCategories: + "Account vocabulary, restored by the backup restore with the tags it groups.", + hiddenMoodTags: + "An account setting (which tags its editor hides), restored by the backup restore.", +}; + +type Row = Record; + +function keysOf(rows: readonly Row[]): Set { + const keys = new Set(); + for (const row of rows) for (const key of Object.keys(row)) keys.add(key); + return keys; +} + +/** Fields of `written` the import neither reads nor names. */ +function unaccounted( + written: Set, + read: Set, + named: Record, +): string[] { + return [...written].filter((key) => !read.has(key) && !(key in named)); +} + +interface Files { + jsonExport: { top: Row; measurements: Row[]; moodEntries: Row[] }; + portable: Row & { measurements: Row[]; moodEntries: Row[] }; + disasterRecovery: Row & { measurements: Row[]; moodEntries: Row[] }; +} + +let files: Files; + +beforeAll(async () => { + await truncateAllTables(prisma); + await seedJsonImportCatalogue(prisma); + const { ownerId } = await seedJsonImportOwner(prisma); + const owner = await prisma.user.findUniqueOrThrow({ where: { id: ownerId } }); + const exported = await callRoute(exportJson, { + method: "GET", + route: "/api/export", + query: { format: "json", type: "all" }, + auth: await makeSession(owner), + }); + expect(exported.status).toBe(200); + const data = (exported.body as { data: Row }).data; + const portable = await buildFullBackupPayload(prisma, ownerId); + const disasterRecovery = await buildFullBackupPayload(prisma, ownerId, { + purpose: "disaster-recovery", + }); + files = { + jsonExport: { + top: data, + measurements: data.measurements as Row[], + moodEntries: data.moodEntries as Row[], + }, + portable: portable.payload as Files["portable"], + disasterRecovery: disasterRecovery.payload as Files["disasterRecovery"], + }; +}); + +afterAll(async () => { + await truncateAllTables(prisma); +}); + +const MEASUREMENT_READ = new Set( + Object.keys(jsonImportMeasurementSchema.shape), +); +const MOOD_READ = new Set(Object.keys(jsonImportMoodEntrySchema.shape)); +const CONTEXT_READ = new Set(Object.keys(jsonImportMoodContextSchema.shape)); + +const FILE_NAMES = ["jsonExport", "portable", "disasterRecovery"] as const; + +function measurementsOf(name: (typeof FILE_NAMES)[number]): Row[] { + return files[name].measurements; +} +function moodOf(name: (typeof FILE_NAMES)[number]): Row[] { + return files[name].moodEntries; +} +function contextsOf(name: (typeof FILE_NAMES)[number]): Row[] { + return moodOf(name) + .map((entry) => entry.context) + .filter((context): context is Row => !!context); +} + +describe("the JSON import reads what the exporters write", () => { + it.each(FILE_NAMES)("every measurement field of the %s", (name) => { + const rows = measurementsOf(name); + expect(rows.length).toBeGreaterThan(3); + expect( + unaccounted(keysOf(rows), MEASUREMENT_READ, MEASUREMENT_NOT_IMPORTED), + ).toEqual([]); + }); + + it.each(FILE_NAMES)("every mood-entry field of the %s", (name) => { + const rows = moodOf(name); + expect(rows.length).toBeGreaterThan(0); + expect(unaccounted(keysOf(rows), MOOD_READ, MOOD_NOT_IMPORTED)).toEqual([]); + }); + + it.each(["portable", "disasterRecovery"] as const)( + "every day-context field of the %s", + (name) => { + const rows = contextsOf(name); + expect(rows.length).toBeGreaterThan(0); + expect( + unaccounted(keysOf(rows), CONTEXT_READ, CONTEXT_NOT_IMPORTED), + ).toEqual([]); + }, + ); + + it("names the mood vocabulary the entries' links point into", () => { + for (const name of ["portable", "disasterRecovery"] as const) { + const sections = Object.keys(files[name]); + expect(sections).toEqual(expect.arrayContaining([...IMPORTED_SECTIONS])); + for (const section of Object.keys(VOCABULARY_SECTIONS)) { + expect(sections, `${name} carries ${section}`).toContain(section); + } + } + // The JSON export's other two sections are medications, restored the + // same way as every other section of an account's record. + expect(Object.keys(files.jsonExport.top).sort()).toEqual([ + "intakeEvents", + "measurements", + "medications", + "moodEntries", + ]); + }); + + it("gives no reason for a field no exporter writes", () => { + const writtenMeasurement = new Set( + FILE_NAMES.flatMap((name) => [...keysOf(measurementsOf(name))]), + ); + const writtenMood = new Set( + FILE_NAMES.flatMap((name) => [...keysOf(moodOf(name))]), + ); + const writtenContext = new Set( + FILE_NAMES.flatMap((name) => [...keysOf(contextsOf(name))]), + ); + const stale = [ + ...Object.keys(MEASUREMENT_NOT_IMPORTED) + .filter((key) => !writtenMeasurement.has(key)) + .map((key) => `measurement.${key}`), + ...Object.keys(MOOD_NOT_IMPORTED) + .filter((key) => !writtenMood.has(key)) + .map((key) => `moodEntry.${key}`), + ...Object.keys(CONTEXT_NOT_IMPORTED) + .filter((key) => !writtenContext.has(key)) + .map((key) => `context.${key}`), + ]; + expect(stale).toEqual([]); + }); + + it("reads no field it also names as left out", () => { + const both = [ + ...Object.keys(MEASUREMENT_NOT_IMPORTED).filter((k) => + MEASUREMENT_READ.has(k), + ), + ...Object.keys(MOOD_NOT_IMPORTED).filter((k) => MOOD_READ.has(k)), + ...Object.keys(CONTEXT_NOT_IMPORTED).filter((k) => CONTEXT_READ.has(k)), + ]; + expect(both).toEqual([]); + }); +}); diff --git a/tests/integration/json-import-own-exports.test.ts b/tests/integration/json-import-own-exports.test.ts index 4457c29e4..50487557f 100644 --- a/tests/integration/json-import-own-exports.test.ts +++ b/tests/integration/json-import-own-exports.test.ts @@ -1,18 +1,24 @@ /** - * The JSON import takes the app's own files back. + * The JSON import takes the app's own files back, whole. * - * An account's full backup (`/api/export/full-backup`, the portable export) - * and its JSON export (`/api/export?format=json`) are written, as they are - * served, and imported into a fresh account. What the import carries (a - * measurement's type, value, unit, instant, glucose context and note; a mood - * entry's day, label, score, tags and instant) must come back as it was. + * An account with every column filled in (`json-import-seed.ts`) is written + * out by the full backup (the portable export, `/api/export/full-backup`) and + * by the JSON export (`/api/export?format=json`), each file is imported into + * a fresh account, and the fresh account is written out again by the same + * exporter. The two files must agree on everything the import carries; what + * it deliberately leaves out is named in `json-import-field-coverage.test.ts` + * and is the only thing stripped before the comparison. + * + * Reading the result back through the exporter rather than through a + * hand-written projection of the tables is the point: a field the import + * drops shows up as a difference in the file a person would get back, which + * is the loss that matters, and a field an exporter learns to write is + * compared from the release it starts. * * Both files spell an empty field their own way, `null` in the backup and an * empty string in the export. Before the import read files in the exporter's * terms, the first empty note, tag list or glucose context refused the whole - * file with a 422. The portable backup also left a reading's own attributes - * out (a glucose value's fasting context among them), so the backup half - * pins those too. + * file with a 422. */ import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -21,163 +27,60 @@ import { POST as importCsv } from "@/app/api/import/csv/route"; import { GET as exportMeasurementsCsv } from "@/app/api/export/measurements/route"; import { POST as importJson } from "@/app/api/import/route"; import { encryptNote, readNote } from "@/lib/crypto/note-cipher"; +import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; import { streamFullBackupJson } from "@/lib/export/full-backup-stream"; import { callRoute } from "../kit/call-route"; import { makeSession, makeUser } from "../kit/factories"; +import { + JI_OWN_TAG_KEY, + seedJsonImportCatalogue, + seedJsonImportOwner, +} from "./json-import-seed"; import { getPrismaClient, truncateAllTables } from "./setup"; const prisma = getPrismaClient(); -const MEASUREMENTS = [ - { - type: "WEIGHT", - value: 81.4, - unit: "kg", - at: "2026-09-10T06:30:00.000Z", - note: "after the run", - }, - { - type: "WEIGHT", - value: 81.9, - unit: "kg", - at: "2026-09-11T06:30:00.000Z", - note: null, - }, - { - type: "PULSE", - value: 62, - unit: "bpm", - at: "2026-09-11T07:00:00.000Z", - note: null, - }, - { - type: "BLOOD_GLUCOSE", - value: 104, - unit: "mg/dL", - at: "2026-09-12T08:00:00.000Z", - note: null, - glucoseContext: "FASTING" as const, - }, - { - type: "BLOOD_GLUCOSE", - value: 141, - unit: "mg/dL", - at: "2026-09-12T13:00:00.000Z", - note: "after lunch", - glucoseContext: "POSTPRANDIAL" as const, - }, -] as const; +type Row = Record; -const MOOD = [ - { - date: "2026-09-10", - mood: "GUT", - score: 4, - tags: null, - at: "2026-09-10T19:00:00.000Z", - }, - { - date: "2026-09-11", - mood: "OKAY", - score: 3, - tags: JSON.stringify(["work", "sleep"]), - at: "2026-09-11T20:30:00.000Z", - }, - { - date: "2026-09-12", - mood: "SUPER_GUT", - score: 5, - tags: null, - at: "2026-09-12T21:15:00.000Z", - }, -] as const; +interface ImportStats { + measurements: number; + moodEntries: number; + skipped: number; + droppedTagKeys?: string[]; + droppedFactorKeys?: string[]; +} let ownerId: string; -async function seedOwner(): Promise { - const owner = await makeUser({ - timezone: "Europe/Berlin", - glucoseUnit: "mmol/L" /* the JSON export stays in the stored unit */, +async function fullBackupOf(userId: string): Promise<{ + measurements: Row[]; + moodEntries: Row[]; +}> { + let text = ""; + await streamFullBackupJson(prisma, userId, (chunk) => { + text += chunk; }); - ownerId = owner.id; - for (const m of MEASUREMENTS) { - await prisma.measurement.create({ - data: { - userId: ownerId, - type: m.type, - value: m.value, - unit: m.unit, - measuredAt: new Date(m.at), - source: "MANUAL", - notes: null, - notesEncrypted: encryptNote(m.note), - glucoseContext: "glucoseContext" in m ? m.glucoseContext : null, - }, - }); - } - // A reading deleted before the backup: its row is the record of a - // deletion, and the import leaves it out. - await prisma.measurement.create({ - data: { - userId: ownerId, - type: "WEIGHT", - value: 90, - unit: "kg", - measuredAt: new Date("2026-09-09T06:30:00.000Z"), - source: "MANUAL", - deletedAt: new Date("2026-09-09T07:00:00.000Z"), - }, - }); - for (const e of MOOD) { - await prisma.moodEntry.create({ - data: { - userId: ownerId, - date: e.date, - mood: e.mood, - score: e.score, - tags: e.tags, - source: "MANUAL", - moodLoggedAt: new Date(e.at), - }, - }); - } + return JSON.parse(text); } -/** What an import carries, read back from an account. */ -async function carried(userId: string) { - const measurements = await prisma.measurement.findMany({ - where: { userId, deletedAt: null }, - orderBy: [{ measuredAt: "asc" }, { type: "asc" }], +async function jsonExportOf(userId: string): Promise<{ + data: { measurements: Row[]; moodEntries: Row[] }; +}> { + const user = await prisma.user.findUniqueOrThrow({ where: { id: userId } }); + const res = await callRoute(exportJson, { + method: "GET", + route: "/api/export", + query: { format: "json", type: "all" }, + auth: await makeSession(user), }); - const mood = await prisma.moodEntry.findMany({ - where: { userId, deletedAt: null }, - orderBy: { moodLoggedAt: "asc" }, - }); - return { - measurements: measurements.map((m) => ({ - type: m.type, - value: m.value, - unit: m.unit, - measuredAt: m.measuredAt.toISOString(), - glucoseContext: m.glucoseContext, - note: readNote(m.notesEncrypted, m.notes), - })), - mood: mood.map((e) => ({ - date: e.date, - mood: e.mood, - score: e.score, - tags: e.tags, - loggedAt: e.moodLoggedAt.toISOString(), - })), - }; + expect(res.status).toBe(200); + return res.body as { data: { measurements: Row[]; moodEntries: Row[] } }; } async function importInto(file: unknown) { const user = await makeUser({ timezone: "Europe/Berlin" }); - const res = await callRoute<{ - data: { measurements: number; moodEntries: number; skipped: number }; - }>(importJson, { + const res = await callRoute(importJson, { method: "POST", route: "/api/import", body: file, @@ -186,77 +89,194 @@ async function importInto(file: unknown) { return { userId: user.id, res }; } +/** A row without the fields the import deliberately does not carry. */ +function comparable(row: Row): Row { + const { id: _id, source: _source, ...rest } = row; + return rest; +} + +function byInstant(at: string) { + return (a: Row, b: Row) => + String(a[at]).localeCompare(String(b[at])) || + String(a.type ?? "").localeCompare(String(b.type ?? "")); +} + beforeAll(async () => { await truncateAllTables(prisma); - await seedOwner(); + await seedJsonImportCatalogue(prisma); + ({ ownerId } = await seedJsonImportOwner(prisma)); }); afterAll(async () => { await truncateAllTables(prisma); }); -describe("the JSON import reads the app's own files", () => { - it("takes a full backup back, nulls and deletion records included", async () => { - let text = ""; - await streamFullBackupJson(prisma, ownerId, (chunk) => { - text += chunk; - }); - const backup = JSON.parse(text) as { - measurements: Array>; - moodEntries: Array>; - }; - // The file carries the shapes the import used to refuse. - expect(backup.measurements.some((m) => m.notes === null)).toBe(true); - expect(backup.moodEntries.some((e) => e.tags === null)).toBe(true); +describe("the JSON import takes the app's own files back", () => { + it("takes a full backup back field for field", async () => { + const original = await fullBackupOf(ownerId); + // The file carries the shapes the import used to refuse or drop. + expect(original.measurements.some((m) => m.notes === null)).toBe(true); + expect(original.measurements.some((m) => m.sleepStage === "DEEP")).toBe( + true, + ); + expect(original.moodEntries.some((e) => e.context !== null)).toBe(true); - const { userId, res } = await importInto(backup); + const { userId, res } = await importInto(original); expect(res.status).toBe(200); - const deletedInFile = backup.measurements.filter( - (m) => typeof m.deletedAt === "string", - ).length; expect(res.data).toEqual({ - measurements: MEASUREMENTS.length, - moodEntries: MOOD.length, - skipped: deletedInFile, + measurements: original.measurements.length, + moodEntries: original.moodEntries.length, + skipped: 0, + // The account's own tag is not this account's: the entry lands without + // that one link, and the response says which. + droppedTagKeys: [JI_OWN_TAG_KEY], }); - expect(await carried(userId)).toEqual(await carried(ownerId)); + const back = await fullBackupOf(userId); + const expectedMeasurements = original.measurements + .map(comparable) + .sort(byInstant("measuredAt")); + expect( + back.measurements.map(comparable).sort(byInstant("measuredAt")), + ).toEqual(expectedMeasurements); + const expectedMood = original.moodEntries + .map((entry) => ({ + ...comparable(entry), + structuredTags: (entry.structuredTags as string[]).filter( + (key) => key !== JI_OWN_TAG_KEY, + ), + })) + .sort(byInstant("loggedAt")); + expect( + back.moodEntries.map(comparable).sort(byInstant("loggedAt")), + ).toEqual(expectedMood); }); it("takes the JSON export back, empty strings and its envelope included", async () => { - const owner = await prisma.user.findUniqueOrThrow({ - where: { id: ownerId }, - }); - const exported = await callRoute(exportJson, { - method: "GET", - route: "/api/export", - query: { format: "json", type: "all" }, - auth: await makeSession(owner), - }); - expect(exported.status).toBe(200); - const file = exported.body as { - data: { - measurements: Array>; - moodEntries: Array>; - }; - }; - expect(file.data.measurements.some((m) => m.notes === "")).toBe(true); - expect(file.data.moodEntries.some((e) => e.tags === "")).toBe(true); + const original = await jsonExportOf(ownerId); + expect(original.data.measurements.some((m) => m.notes === "")).toBe(true); + expect(original.data.moodEntries.some((e) => e.note === "")).toBe(true); - const { userId, res } = await importInto(file); + const { userId, res } = await importInto(original); expect(res.status).toBe(200); expect(res.data).toMatchObject({ - measurements: MEASUREMENTS.length, - moodEntries: MOOD.length, + measurements: original.data.measurements.length, + moodEntries: original.data.moodEntries.length, + skipped: 0, }); - expect(await carried(userId)).toEqual(await carried(ownerId)); + // A sleep night is one summary record in this file, and its `notes` is + // the stage breakdown the exporter writes from the stored stages. The + // import stores the night and that text as its note; a re-export + // rebuilds the column from stages the summary does not have. So the + // night's note is compared against what was stored, and the rest of the + // record through the file. + const nights = original.data.measurements.filter( + (m) => m.type === "SLEEP_DURATION", + ); + expect(nights).toHaveLength(1); + const storedNight = await prisma.measurement.findFirstOrThrow({ + where: { userId, type: "SLEEP_DURATION" }, + }); + expect(readNote(storedNight.notesEncrypted, storedNight.notes)).toBe( + nights[0]!.notes, + ); + const withoutNightNote = (m: Row) => + m.type === "SLEEP_DURATION" + ? { ...comparable(m), notes: null } + : comparable(m); + + const back = await jsonExportOf(userId); + expect( + back.data.measurements + .map(withoutNightNote) + .sort(byInstant("measuredAt")), + ).toEqual( + original.data.measurements + .map(withoutNightNote) + .sort(byInstant("measuredAt")), + ); + expect( + back.data.moodEntries.map(comparable).sort(byInstant("loggedAt")), + ).toEqual( + original.data.moodEntries.map(comparable).sort(byInstant("loggedAt")), + ); }); - it("takes the measurement CSV back into the CSV import, in display units too", async () => { - const owner = await prisma.user.findUniqueOrThrow({ - where: { id: ownerId }, + it("leaves out a disaster-recovery file's deletion records, and counts them", async () => { + const { payload } = await buildFullBackupPayload(prisma, ownerId, { + purpose: "disaster-recovery", }); + const rows = (payload as { measurements: Row[] }).measurements; + const deleted = rows.filter((m) => typeof m.deletedAt === "string"); + expect(deleted).toHaveLength(1); + + const { userId, res } = await importInto(payload); + expect(res.status).toBe(200); + expect(res.data).toMatchObject({ + measurements: rows.length - deleted.length, + skipped: deleted.length, + }); + expect( + await prisma.measurement.count({ where: { userId, value: 90 } }), + ).toBe(0); + }); + + it("takes the same file twice without changing what the first import wrote", async () => { + const original = await fullBackupOf(ownerId); + const { userId } = await importInto(original); + const first = await fullBackupOf(userId); + const user = await prisma.user.findUniqueOrThrow({ where: { id: userId } }); + const again = await callRoute(importJson, { + method: "POST", + route: "/api/import", + body: original, + auth: await makeSession(user), + }); + expect(again.status).toBe(200); + // The mood entries carry their instant, so the second pass finds them; + // the readings carry no source id in a portable file and are written + // again, as the import has always documented for a file without one. + expect(again.data.moodEntries).toBe(0); + const second = await fullBackupOf(userId); + expect( + second.moodEntries.map(comparable).sort(byInstant("loggedAt")), + ).toEqual(first.moodEntries.map(comparable).sort(byInstant("loggedAt"))); + }); +}); + +describe("the CSV import reads the measurement CSV export", () => { + const READINGS = [ + { type: "WEIGHT", value: 81.4, unit: "kg", at: "2026-09-10T06:30:00.000Z" }, + { type: "PULSE", value: 62, unit: "bpm", at: "2026-09-11T07:00:00.000Z" }, + { + type: "BLOOD_GLUCOSE", + value: 104, + unit: "mg/dL", + at: "2026-09-12T08:00:00.000Z", + glucoseContext: "FASTING" as const, + }, + ] as const; + + it("takes it back, in display units too", async () => { + const owner = await makeUser({ + timezone: "Europe/Berlin", + glucoseUnit: "mmol/L", + }); + for (const m of READINGS) { + await prisma.measurement.create({ + data: { + userId: owner.id, + type: m.type, + value: m.value, + unit: m.unit, + measuredAt: new Date(m.at), + source: "MANUAL", + notesEncrypted: encryptNote(null), + glucoseContext: "glucoseContext" in m ? m.glucoseContext : null, + }, + }); + } const exported = await callRoute(exportMeasurementsCsv, { method: "GET", route: "/api/export/measurements", @@ -279,13 +299,24 @@ describe("the JSON import reads the app's own files", () => { }, ); expect(res.status).toBe(200); - expect(res.data).toMatchObject({ - inserted: MEASUREMENTS.length, - skipped: 0, - }); + expect(res.data).toMatchObject({ inserted: READINGS.length, skipped: 0 }); - const back = (await carried(user.id)).measurements; - const original = (await carried(ownerId)).measurements; + const read = async (userId: string) => + ( + await prisma.measurement.findMany({ + where: { userId, deletedAt: null }, + orderBy: [{ measuredAt: "asc" }, { type: "asc" }], + }) + ).map((m) => ({ + type: m.type, + value: m.value, + unit: m.unit, + measuredAt: m.measuredAt.toISOString(), + glucoseContext: m.glucoseContext, + note: readNote(m.notesEncrypted, m.notes), + })); + const back = await read(user.id); + const original = await read(owner.id); expect(back).toHaveLength(original.length); back.forEach((m, i) => { const o = original[i]!; diff --git a/tests/integration/json-import-seed.ts b/tests/integration/json-import-seed.ts new file mode 100644 index 000000000..5191d1b0c --- /dev/null +++ b/tests/integration/json-import-seed.ts @@ -0,0 +1,267 @@ +/** + * One account with every column the exporters read filled in, for the JSON + * import's round trip and its field-coverage guard. + * + * Every reading carries what makes it that reading: a range keeps its bounds, + * a sleep night its stages and its writer, a rhythm alert its verdict, a + * glucose value its meal-time context, a note its text. The mood entry + * carries a note, all five level-A values, its zone, a rated factor, a + * catalogue tag, a tag of the account's own and a full day context. A field + * the seed leaves empty is a field neither test can see arrive, so nothing + * here is left at its default. + */ +import type { PrismaClient } from "@/generated/prisma/client"; +import { encryptNote } from "@/lib/crypto/note-cipher"; +import { encodeKeyList } from "@/lib/mood/context-vocabulary"; + +import { makeUser } from "../kit/factories"; + +export const JI_FACTOR_KEY = "ji_sleep_quality"; +export const JI_TAG_KEY = "ji_headache"; +/** The account's own tag: another account importing the file cannot resolve it. */ +export const JI_OWN_TAG_KEY = "custom:ji_long_walk"; + +export const JI_MEASUREMENTS = [ + { + type: "WEIGHT", + value: 81.4, + unit: "kg", + at: "2026-09-10T06:30:00.000Z", + note: "after the run", + }, + { + type: "PULSE", + value: 64, + valueMin: 52, + valueMax: 91, + unit: "bpm", + at: "2026-09-11T07:00:00.000Z", + deviceType: "watch", + }, + { + type: "BLOOD_GLUCOSE", + value: 104, + unit: "mg/dL", + at: "2026-09-12T08:00:00.000Z", + glucoseContext: "FASTING", + }, + { + type: "BLOOD_GLUCOSE", + value: 141, + unit: "mg/dL", + at: "2026-09-12T13:00:00.000Z", + note: "after lunch", + glucoseContext: "POSTPRANDIAL", + }, + { + type: "IRREGULAR_RHYTHM_NOTIFICATION", + value: 1, + unit: "count", + at: "2026-09-12T15:00:00.000Z", + rhythmClassification: "IRREGULAR", + deviceType: "watch", + }, + // One night, as a watch writes it: a stage per row. + { + type: "SLEEP_DURATION", + value: 70, + unit: "minutes", + at: "2026-09-12T23:40:00.000Z", + sleepStage: "DEEP", + deviceType: "watch", + }, + { + type: "SLEEP_DURATION", + value: 240, + unit: "minutes", + at: "2026-09-13T03:40:00.000Z", + sleepStage: "CORE", + deviceType: "watch", + }, + { + type: "SLEEP_DURATION", + value: 95, + unit: "minutes", + at: "2026-09-13T05:15:00.000Z", + sleepStage: "REM", + deviceType: "watch", + }, +] as const; + +export const JI_MOOD = { + date: "2026-09-12", + mood: "GUT", + score: 4, + a1: 8, + a2: 3, + a3: 6, + a4: 7, + a5: 5, + tags: JSON.stringify(["work", "sleep"]), + note: "long day, good evening", + tz: "America/New_York", + at: "2026-09-13T01:30:00.000Z", +} as const; + +/** A second entry with nothing but the label: what a one-tap check-in writes. */ +export const JI_MOOD_BARE = { + date: "2026-09-11", + mood: "OKAY", + score: 3, + at: "2026-09-11T20:30:00.000Z", +} as const; + +export interface SeededOwner { + ownerId: string; +} + +/** The catalogue the links resolve against: seeded rows, owned by no account. */ +export async function seedJsonImportCatalogue( + prisma: PrismaClient, +): Promise { + const category = await prisma.moodTagCategory.create({ + data: { key: "ji_category", labelKey: "mood.tagCategory.ji" }, + }); + await prisma.moodTag.create({ + data: { + categoryId: category.id, + key: JI_TAG_KEY, + labelKey: "mood.tag.jiHeadache", + kind: "BINARY", + }, + }); + await prisma.moodTag.create({ + data: { + categoryId: category.id, + key: JI_FACTOR_KEY, + labelKey: "mood.tag.jiSleepQuality", + kind: "RATED", + scaleMin: 1, + scaleMax: 5, + }, + }); +} + +export async function seedJsonImportOwner( + prisma: PrismaClient, +): Promise { + const owner = await makeUser({ + timezone: "Europe/Berlin", + glucoseUnit: "mmol/L" /* the JSON export stays in the stored unit */, + }); + const ownerId = owner.id; + for (const m of JI_MEASUREMENTS) { + await prisma.measurement.create({ + data: { + userId: ownerId, + type: m.type, + value: m.value, + valueMin: "valueMin" in m ? m.valueMin : null, + valueMax: "valueMax" in m ? m.valueMax : null, + unit: m.unit, + measuredAt: new Date(m.at), + source: "APPLE_HEALTH", + notes: null, + notesEncrypted: encryptNote("note" in m ? m.note : null), + glucoseContext: "glucoseContext" in m ? m.glucoseContext : null, + sleepStage: "sleepStage" in m ? m.sleepStage : null, + rhythmClassification: + "rhythmClassification" in m ? m.rhythmClassification : null, + deviceType: "deviceType" in m ? m.deviceType : null, + }, + }); + } + // A reading deleted before the backup: its row is the record of a + // deletion, and the import leaves it out. + await prisma.measurement.create({ + data: { + userId: ownerId, + type: "WEIGHT", + value: 90, + unit: "kg", + measuredAt: new Date("2026-09-09T06:30:00.000Z"), + source: "MANUAL", + deletedAt: new Date("2026-09-09T07:00:00.000Z"), + }, + }); + + const category = await prisma.moodTagCategory.findUniqueOrThrow({ + where: { key: "ji_category" }, + }); + const ownTag = await prisma.moodTag.create({ + data: { + userId: ownerId, + categoryId: category.id, + key: JI_OWN_TAG_KEY, + labelKey: "mood.tag.custom", + kind: "BINARY", + }, + }); + const [tag, factor] = await Promise.all([ + prisma.moodTag.findUniqueOrThrow({ where: { key: JI_TAG_KEY } }), + prisma.moodTag.findUniqueOrThrow({ where: { key: JI_FACTOR_KEY } }), + ]); + + const entry = await prisma.moodEntry.create({ + data: { + userId: ownerId, + date: JI_MOOD.date, + mood: JI_MOOD.mood, + score: JI_MOOD.score, + moodA1: JI_MOOD.a1, + stressA2: JI_MOOD.a2, + energyA3: JI_MOOD.a3, + connectionA4: JI_MOOD.a4, + stabilityA5: JI_MOOD.a5, + tags: JI_MOOD.tags, + note: null, + noteEncrypted: encryptNote(JI_MOOD.note), + tz: JI_MOOD.tz, + source: "MANUAL", + moodLoggedAt: new Date(JI_MOOD.at), + }, + }); + await prisma.moodEntryTagLink.createMany({ + data: [ + { moodEntryId: entry.id, moodTagId: tag.id }, + { moodEntryId: entry.id, moodTagId: ownTag.id }, + { moodEntryId: entry.id, moodTagId: factor.id, rating: 4 }, + ], + }); + await prisma.moodContext.create({ + data: { + moodEntryId: entry.id, + userId: ownerId, + workStatus: "regular", + workMinutes: 540, + overtimeMinutes: 60, + workLoad: 7, + workSatisfaction: 6, + contactCircles: encodeKeyList(["partner", "family"]), + contactForm: "inPerson", + contactExtent: "extended", + contactQuality: 8, + contactSupport: 7, + leisureCategories: encodeKeyList(["reading"]), + leisureMinutes: 90, + leisureJoy: 8, + leisureRecovery: 7, + eventType: "goodNews", + eventValence: 3, + eventAt: new Date("2026-09-12T18:00:00.000Z"), + notesEncrypted: encryptNote("the call went well"), + }, + }); + await prisma.moodEntry.create({ + data: { + userId: ownerId, + date: JI_MOOD_BARE.date, + mood: JI_MOOD_BARE.mood, + score: JI_MOOD_BARE.score, + moodA1: 5, + source: "MANUAL", + moodLoggedAt: new Date(JI_MOOD_BARE.at), + }, + }); + return { ownerId }; +} diff --git a/tests/integration/routes/parity/__snapshots__/import-json.01-first-file.json.snap b/tests/integration/routes/parity/__snapshots__/import-json.01-first-file.json.snap index 8662cf5e3..385d893fa 100644 --- a/tests/integration/routes/parity/__snapshots__/import-json.01-first-file.json.snap +++ b/tests/integration/routes/parity/__snapshots__/import-json.01-first-file.json.snap @@ -20,6 +20,7 @@ "moodEntries": 4, "skipped": 2 }, + "import_dropped_link_keys": 0, "external_id_rejected": 2, "external_id_shapes": "object_description", "external_id_surface": "import.json" diff --git a/tests/integration/routes/parity/__snapshots__/import-json.02-same-file.json.snap b/tests/integration/routes/parity/__snapshots__/import-json.02-same-file.json.snap index 845d804ef..782d83261 100644 --- a/tests/integration/routes/parity/__snapshots__/import-json.02-same-file.json.snap +++ b/tests/integration/routes/parity/__snapshots__/import-json.02-same-file.json.snap @@ -20,6 +20,7 @@ "moodEntries": 1, "skipped": 6 }, + "import_dropped_link_keys": 0, "external_id_rejected": 2, "external_id_shapes": "object_description", "external_id_surface": "import.json"