diff --git a/src/__tests__/backup-plan-classification.test.ts b/src/__tests__/backup-plan-classification.test.ts index ffe4fda3a..e41ddd97c 100644 --- a/src/__tests__/backup-plan-classification.test.ts +++ b/src/__tests__/backup-plan-classification.test.ts @@ -322,7 +322,7 @@ describe("every backed-up model travels both ways, or is named as debt", () => { it("does not count a nested relation read as a restore write", () => { expect( touches( - ["src/lib/export/measurements-backup.ts"], + ["src/lib/export/measurements-restore.ts"], "User", WRITE_OPS, WRITE_RELATION_OPS, @@ -565,7 +565,7 @@ describe("tables outside the schema have a backup verdict too", () => { it("carries the medication category through both ends", () => { const writer = readFileSync( - join(SRC, "lib/export/full-backup-payload.ts"), + join(SRC, "lib/export/medications-backup.ts"), "utf8", ); const restore = readFileSync( diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index 98c5891a9..21e023bf0 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 128, - "total": 153023, + "count": 126, + "total": 150668, "entries": { "app/api/dashboard/summary/route.ts": 1032, "app/api/documents/inbound/route.ts": 811, @@ -68,11 +68,9 @@ "lib/dashboard/snapshot.ts": 1549, "lib/day/records.ts": 1175, "lib/doctor-report/collect.ts": 803, - "lib/export/backup-plan.ts": 854, + "lib/export/backup-plan.ts": 853, "lib/export/coach-backup.ts": 1070, - "lib/export/full-backup-payload.ts": 1520, "lib/export/profile-backup.ts": 1019, - "lib/export/records-backup.ts": 834, "lib/fitbit/client.ts": 1529, "lib/fitbit/sync-core.ts": 968, "lib/gamification/achievements-result.ts": 1094, diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index 321c976e3..65e66b646 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 413, - "total": 149647, + "count": 410, + "total": 148267, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -319,11 +319,8 @@ "lib/documents/fenced-chat.ts :: streamFencedReply": 309, "lib/documents/ingest.ts :: ingestDocument": 272, "lib/export/backup-json-scan.ts :: scanBackupJson": 225, - "lib/export/full-backup-payload.ts :: buildFullBackupPayload": 804, - "lib/export/full-backup-payload.ts :: buildFullBackupPayload > medications.map(arg) #3": 268, "lib/export/profile-backup.ts :: buildProfileBackupSection": 270, "lib/export/profile-backup.ts :: restoreProfileData": 291, - "lib/export/records-backup.ts :: buildRecordsBackupSection": 308, "lib/export/vaccinations-backup.ts :: restoreVaccinationsData": 216, "lib/export/visits-backup.ts :: restoreVisitsData": 261, "lib/fhir/build-bundle.ts :: buildFhirDocumentBundle": 363, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 6271a5df6..deed4fffb 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", - "count": 2961, - "total": 10571, + "count": 2959, + "total": 10559, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -2044,11 +2044,9 @@ "lib/export/account-settings-backup.ts": 2, "lib/export/backup-plan.ts": 9, "lib/export/coach-backup.ts": 13, - "lib/export/full-backup-payload.ts": 8, "lib/export/intraday-profile-backup.ts": 1, "lib/export/passphrase-archive.ts": 1, "lib/export/profile-backup.ts": 10, - "lib/export/records-backup.ts": 4, "lib/export/reminders-backup.ts": 4, "lib/export/visits-backup.ts": 5, "lib/feature-flags/__tests__/index.test.ts": 2, diff --git a/src/__tests__/measurement-backup-completeness.test.ts b/src/__tests__/measurement-backup-completeness.test.ts index fb0c36f1f..693fb0bc4 100644 --- a/src/__tests__/measurement-backup-completeness.test.ts +++ b/src/__tests__/measurement-backup-completeness.test.ts @@ -19,24 +19,27 @@ * backup-relevant; converting every model at once buys exclusion-list churn * rather than safety. * + * Both backup ends of the measurements live in the measurements domain + * (`src/lib/export/measurements-backup.ts` out, `measurements-restore.ts` + * back), so the guard checks the restore half too: every carried column is + * written back by `toRestoredMeasurementData`. + * * Mutation check: comment out `sleepStage: true` in `MEASUREMENT_BACKUP_SELECT` * and this goes red naming that column; add a scalar to the model without - * adding it to the select and it goes red naming the new one. + * adding it to the select and it goes red naming the new one; drop a key from + * `toRestoredMeasurementData` and the restore case goes red naming it. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, it, expect } from "vitest"; +import { readSource } from "./helpers/source-files"; + const ROOT = process.cwd(); const SCHEMA = join(ROOT, "prisma", "schema.prisma"); -const BACKUP_BUILDER = join( - ROOT, - "src", - "lib", - "export", - "full-backup-payload.ts", -); +const BACKUP_MODULE = "src/lib/export/measurements-backup.ts"; +const RESTORE_MODULE = "src/lib/export/measurements-restore.ts"; const PROFILE_BACKUP_BUILDER = join( ROOT, "src", @@ -105,18 +108,50 @@ function selectFields(source: string): string[] { ); } +/** The text of one top-level function, from its declaration to its end. */ +function functionBody(source: string, declaration: string): string { + const start = source.indexOf(declaration); + expect(start, `${declaration} not found`).toBeGreaterThan(-1); + const end = source.indexOf("\n}\n", start); + expect(end).toBeGreaterThan(start); + return source.slice(start, end); +} + /** * Fields either serialization arm actually emits. The two arms share one * restorable wire format, so this is a union and therefore a weaker claim than * the select comparison: it catches a column that is selected and then written - * to neither payload, not a column carried by only one arm. + * to neither payload, not a column carried by only one arm. Read from the + * serialiser alone, because the restore beside it spells its rows the same + * way and would answer for a column the export dropped. */ function serialisedFields(source: string): Set { + const body = functionBody(source, "function serialiseMeasurement("); return new Set( - [...source.matchAll(/([a-zA-Z_$][\w$]*): measurement\./g)].map((m) => m[1]), + [...body.matchAll(/([a-zA-Z_$][\w$]*): measurement\./g)].map((m) => m[1]), ); } +/** Keys the restore writes for each row, the same file's other half. */ +function restoredFields(source: string): Set { + const body = functionBody(source, "function toRestoredMeasurementData("); + return new Set( + [ + ...body.matchAll( + /^ {4}(?:\.\.\.\(measurement\.([a-zA-Z_$][\w$]*)|([a-zA-Z_$][\w$]*):)/gm, + ), + ].map((m) => m[1] ?? m[2]), + ); +} + +/** + * Selected columns the per-row restore data does not write, with the reason. + * `id` is written beside it, from the file's id, by the batch insert. + */ +const RESTORED_ELSEWHERE: Record = { + id: "written beside the row data by the batch insert, from the file's own id", +}; + function interfaceFields(source: string, name: string): Set { const start = source.indexOf(`export interface ${name} {`); expect(start, `interface ${name} not found`).toBeGreaterThan(-1); @@ -139,7 +174,13 @@ function emittedModelFields(source: string, variable: string): Set { describe("measurement backup completeness", () => { const schema = readFileSync(SCHEMA, "utf8"); - const source = readFileSync(BACKUP_BUILDER, "utf8"); + const source = readSource(BACKUP_MODULE, [ + "const MEASUREMENT_BACKUP_SELECT = {", + "function serialiseMeasurement(", + ]); + const restoreSource = readSource(RESTORE_MODULE, [ + "function toRestoredMeasurementData(", + ]); const models = declaredModels(schema); const columns = scalarFields(modelBlock(schema, "Measurement"), models); const selected = selectFields(source); @@ -179,6 +220,18 @@ describe("measurement backup completeness", () => { ).toHaveLength(0); }); + it("the restore writes every carried column back", () => { + const restored = restoredFields(restoreSource); + expect(restored.size).toBeGreaterThan(15); + const unwritten = selected.filter( + (c) => !restored.has(c) && !(c in RESTORED_ELSEWHERE), + ); + expect( + unwritten, + `carried in the file but never written back: ${unwritten}`, + ).toEqual([]); + }); + it("the exclusion list carries no stale entry", () => { for (const [column, reason] of Object.entries(EXCLUDED_COLUMNS)) { expect( diff --git a/src/__tests__/measurement-write-plausibility-guard.test.ts b/src/__tests__/measurement-write-plausibility-guard.test.ts index 1600b753c..386c58703 100644 --- a/src/__tests__/measurement-write-plausibility-guard.test.ts +++ b/src/__tests__/measurement-write-plausibility-guard.test.ts @@ -123,7 +123,7 @@ const WRITERS: Record = { }, // ── Restore ─────────────────────────────────────────────────────────── - "lib/export/measurements-backup.ts": { + "lib/export/measurements-restore.ts": { kind: "restore", why: "replays the record's own export; dropping rows would break the restore contract", }, diff --git a/src/__tests__/measurement-write-tail-guard.test.ts b/src/__tests__/measurement-write-tail-guard.test.ts index 30934f85d..e93ebd836 100644 --- a/src/__tests__/measurement-write-tail-guard.test.ts +++ b/src/__tests__/measurement-write-tail-guard.test.ts @@ -81,7 +81,7 @@ const WRITE_SITES: Record = { "the import worker refolds the import's span and re-warms every imported type", "lib/import/health-connect/import.ts": "the Health Connect import worker refolds the import's span and re-warms every imported type", - "lib/export/measurements-backup.ts": + "lib/export/measurements-restore.ts": "a restore replaces the whole record: it refolds the whole window and drops every per-user cache", // Value-preserving reshapes: the same readings folded into daily or hourly // rows. The rollups are refolded; the figures an assessment reads do not diff --git a/src/__tests__/medication-ingredient-backup-completeness.test.ts b/src/__tests__/medication-ingredient-backup-completeness.test.ts index 65b92c9c8..eeddc5845 100644 --- a/src/__tests__/medication-ingredient-backup-completeness.test.ts +++ b/src/__tests__/medication-ingredient-backup-completeness.test.ts @@ -1,7 +1,7 @@ /** * v1.43 — pair guard binding the `MedicationIngredient` model to both ends of * the backup: the writer's `ingredients: m.ingredients.map(...)` block in - * `full-backup-payload.ts` and the nested create in `medications-restore.ts`. + * `medications-backup.ts` and the nested create in `medications-restore.ts`. * * The failure it catches is the one the measurement guard beside it was * written for: a column added to the composition in a later release, the @@ -79,7 +79,7 @@ describe("medication ingredient backup completeness", () => { it("the writer carries every column", () => { const keys = blockKeys( - read("src", "lib", "export", "full-backup-payload.ts"), + read("src", "lib", "export", "medications-backup.ts"), "ingredients: m.ingredients.map((i) => ({", ); expect(carried.filter((c) => !keys.has(c))).toEqual([]); diff --git a/src/__tests__/medication-intake-tracking-guard.test.ts b/src/__tests__/medication-intake-tracking-guard.test.ts index 2ad178de8..5638ee82f 100644 --- a/src/__tests__/medication-intake-tracking-guard.test.ts +++ b/src/__tests__/medication-intake-tracking-guard.test.ts @@ -61,7 +61,7 @@ const INTAKE_MAY_SEE_EVERY_ROW: Record = { "engagement streak of days with any logged action, no rate", "app/api/export/medications/route.ts": "export of the recorded history", "app/api/export/route.ts": "export of the recorded history", - "lib/export/full-backup-payload.ts": "backup of the recorded history", + "lib/export/medications-backup.ts": "backup of the recorded history", "app/api/sync/state/route.ts": "sync watermarks, no rate", "lib/jobs/measurement-tombstone-cleanup.ts": "tombstone purge", "lib/ai/coach/tools/availability.ts": @@ -94,7 +94,7 @@ const INTAKE_MAY_SEE_EVERY_ROW: Record = { const MAY_READ_EVERY_ROW: Record = { // Record-keeping: a backup, a restore and an export carry the medication // as stored, schedule and switch included. - "lib/export/full-backup-payload.ts": "backup carries the stored rows", + "lib/export/medications-backup.ts": "backup carries the stored rows", "lib/export/medications-restore.ts": "restore writes the stored rows back", // v1.44 — the CSV import writes new medications and reads only their names // and doses to skip duplicates; it derives nothing due. diff --git a/src/__tests__/mood-backup-completeness.test.ts b/src/__tests__/mood-backup-completeness.test.ts index 20d786091..050953e4f 100644 --- a/src/__tests__/mood-backup-completeness.test.ts +++ b/src/__tests__/mood-backup-completeness.test.ts @@ -36,15 +36,12 @@ import { join } from "node:path"; import { describe, it, expect } from "vitest"; +import { readSource } from "./helpers/source-files"; + const ROOT = process.cwd(); const SCHEMA = join(ROOT, "prisma", "schema.prisma"); -const BACKUP_BUILDER = join( - ROOT, - "src", - "lib", - "export", - "full-backup-payload.ts", -); +/** The mood domain's export half, where the selects and serialisers live. */ +const BACKUP_MODULE = "src/lib/export/mood-backup.ts"; /** * Columns the backup is correct to omit, each with the reason. The reasons are @@ -175,7 +172,15 @@ function serialisedFields(source: string, variable: string): Set { describe("mood backup completeness", () => { const schema = readFileSync(SCHEMA, "utf8"); - const source = stripCommentLines(readFileSync(BACKUP_BUILDER, "utf8")); + const source = stripCommentLines( + readSource(BACKUP_MODULE, [ + "const MOOD_ENTRY_BACKUP_SELECT = {", + "const MOOD_CONTEXT_BACKUP_SELECT = {", + "const MOOD_ENTRY_TAG_LINK_SELECT = {", + "function serialiseMoodEntry(", + "function serialiseMoodContext(", + ]), + ); const models = declaredModels(schema); const entryColumns = scalarFields(modelBlock(schema, "MoodEntry"), models); diff --git a/src/lib/export/allergies-backup.ts b/src/lib/export/allergies-backup.ts index 7030d969a..002998548 100644 --- a/src/lib/export/allergies-backup.ts +++ b/src/lib/export/allergies-backup.ts @@ -1,12 +1,59 @@ /** * The allergy record, with both backup ends in one file. */ +import { Buffer } from "node:buffer"; + +import type { PrismaClient } from "@/generated/prisma/client"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; +import { toAllergyDTO, type AllergyDTO } from "@/lib/records/dto"; import { decodeEncryptedBytes, type RestoreContext, } from "@/lib/export/restore-context"; +// ── Export ─────────────────────────────────────────────────────────────────── + +export type CanonicalAllergy = AllergyDTO & { + reactionEncrypted?: string | null; + notesEncrypted?: string | null; + deletedAt?: string | null; +}; + +/** The allergy slice of the records section. */ +export async function buildAllergiesBackupSlice( + prisma: Pick, + userId: string, + disasterRecovery: boolean, +): Promise<{ allergies: CanonicalAllergy[] }> { + const allergyRows = await prisma.allergy.findMany({ + where: disasterRecovery ? { userId } : { userId, deletedAt: null }, + orderBy: { createdAt: "desc" }, + }); + const allergies: CanonicalAllergy[] = disasterRecovery + ? allergyRows.map((row) => ({ + id: row.id, + substance: row.substance, + category: row.category, + type: row.type, + severity: row.severity, + status: row.status, + onsetAt: row.onsetAt?.toISOString() ?? null, + reaction: null, + note: null, + reactionEncrypted: row.reactionEncrypted + ? Buffer.from(row.reactionEncrypted).toString("base64") + : null, + notesEncrypted: row.notesEncrypted + ? Buffer.from(row.notesEncrypted).toString("base64") + : null, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + deletedAt: row.deletedAt?.toISOString() ?? null, + })) + : allergyRows.map(toAllergyDTO); + return { allergies }; +} + // ── Restore ────────────────────────────────────────────────────────────────── export async function clearAllergies(ctx: RestoreContext): Promise { diff --git a/src/lib/export/app-settings-backup.ts b/src/lib/export/app-settings-backup.ts index 0eeb318f9..a5502c410 100644 --- a/src/lib/export/app-settings-backup.ts +++ b/src/lib/export/app-settings-backup.ts @@ -14,9 +14,43 @@ * (`restoreInstanceSettings`), not one the account restore assumes. */ import { DEFAULT_REASONING_MAX_EFFORT } from "@/lib/ai/reasoning/levels"; +import type { + BackupExportContext, + BackupExportPart, +} from "@/lib/export/export-context"; import type { RestoreContext } from "@/lib/export/restore-context"; import { foldLegacyCoachAvailability } from "@/lib/modules/operator-availability"; +// ── Export ─────────────────────────────────────────────────────────────────── + +/** + * The instance settings slice: the singleton row, in a disaster-recovery + * payload only, with its timestamps and its quota made JSON-safe. + */ +export async function buildAppSettingsBackupPart( + ctx: BackupExportContext, +): Promise { + const appSettings = ctx.disasterRecovery + ? await ctx.prisma.appSettings.findUnique({ where: { id: "singleton" } }) + : null; + return { + section: { + appSettings: + ctx.disasterRecovery && appSettings + ? { + ...appSettings, + adminCodexTokenExpiresAt: + appSettings.adminCodexTokenExpiresAt?.toISOString() ?? null, + adminCodexConnectedAt: + appSettings.adminCodexConnectedAt?.toISOString() ?? null, + documentQuotaBytes: appSettings.documentQuotaBytes.toString(), + } + : null, + }, + counts: {}, + }; +} + // ── Restore ────────────────────────────────────────────────────────────────── export async function restoreAppSettings(ctx: RestoreContext): Promise { diff --git a/src/lib/export/backup-domains.ts b/src/lib/export/backup-domains.ts index 7f4767abb..f013237e5 100644 --- a/src/lib/export/backup-domains.ts +++ b/src/lib/export/backup-domains.ts @@ -4,7 +4,9 @@ * Each domain keeps its schema slice, its export and its restore together in * its own module (`-backup.ts`, with `-restore.ts` and * `-backup-schema.ts` beside it where those are large). This file only says - * in which order the restore calls them. The order is the whole content: + * in which order the restore calls them (the export's order, which is the + * file's key order, is `backup-export-sections.ts`). The order is the whole + * content: * * - The account is emptied first, child before parent wherever a count * is reported, because a parent's delete cascades into its children and @@ -53,7 +55,7 @@ import { clearMeasurementRollups, clearMeasurements, restoreMeasurements, -} from "@/lib/export/measurements-backup"; +} from "@/lib/export/measurements-restore"; import { clearIntakeEvents, clearMedications, diff --git a/src/lib/export/backup-export-sections.ts b/src/lib/export/backup-export-sections.ts new file mode 100644 index 000000000..01a4d1ab0 --- /dev/null +++ b/src/lib/export/backup-export-sections.ts @@ -0,0 +1,429 @@ +/** + * The slices of a backup file, in the order their keys appear in it. + * + * Each domain builds its own slice in its own module; this file lays them + * out. The order is every backup file's key order so far, and the restore's + * dependency order lives beside it in `backup-domains.ts`. + */ +import { buildCycleBackupSection } from "@/lib/cycle/backup"; +import { buildAccountSettingsBackupSection } from "@/lib/export/account-settings-backup"; +import { buildAppSettingsBackupPart } from "@/lib/export/app-settings-backup"; +import { + buildAwardsBackupSection, + countAwardsBackupSection, +} from "@/lib/export/awards-backup"; +import { + buildCoachBackupSection, + buildCoachMemoryBackupSection, + countCoachBackupSection, + countCoachMemoryBackupSection, +} from "@/lib/export/coach-backup"; +import { deferredRows } from "@/lib/export/deferred-rows"; +import { + buildDocumentFilingBackupSection, + countDocumentFilingBackupSection, +} from "@/lib/export/document-filing-backup"; +import { iterateBackupDocuments } from "@/lib/export/documents-backup"; +import { + buildEcgBackupSection, + countEcgBackupSection, + iterateBackupEcgRecordings, +} from "@/lib/export/ecg-backup"; +import { + buildEnvironmentBackupSection, + countEnvironmentBackupSection, +} from "@/lib/export/environment-backup"; +import type { + BackupExportContext, + BackupExportPart, +} from "@/lib/export/export-context"; +import { + buildHealthScoreBackupSection, + countHealthScoreBackupSection, +} from "@/lib/export/health-score-backup"; +import { + buildIntradayProfileBackupSection, + countIntradayProfileBackupSection, +} from "@/lib/export/intraday-profile-backup"; +import { + buildLifeEventsBackupSection, + countLifeEventsBackupSection, +} from "@/lib/export/life-events-backup"; +import { buildMeasurementsBackupPart } from "@/lib/export/measurements-backup"; +import { buildMedicationsBackupPart } from "@/lib/export/medications-backup"; +import { buildMoodBackupPart } from "@/lib/export/mood-backup"; +import { buildNutrientsBackupPart } from "@/lib/export/nutrients-backup"; +import { + buildOnboardingBackupSection, + countOnboardingBackupSection, +} from "@/lib/export/onboarding-backup"; +import { + buildProfileBackupSection, + countProfileBackupSection, +} from "@/lib/export/profile-backup"; +import { + buildRecordsBackupSection, + countRecordsBackupSection, +} from "@/lib/export/records-backup"; +import { + buildRelationLinksBackupSection, + countRelationLinksBackupSection, +} from "@/lib/export/relation-links-backup"; +import { + buildRemindersBackupSection, + countRemindersBackupSection, +} from "@/lib/export/reminders-backup"; +import { + buildSensitiveBackupSection, + countSensitiveBackupSection, +} from "@/lib/export/sensitive-backup"; +import { + buildSymptomsBackupSection, + countSymptomsBackupSection, +} from "@/lib/export/symptoms-backup"; +import { + buildVaccinationsBackupSection, + countVaccinationsBackupSection, +} from "@/lib/export/vaccinations-backup"; +import { + buildVisitsBackupSection, + countVisitsBackupSection, +} from "@/lib/export/visits-backup"; + +/** One domain's slice of the file. */ +export interface BackupExportSection { + /** The slice's name in `BACKUP_EXPORT_COUNT_ORDER`. */ + readonly name: string; + build(ctx: BackupExportContext): Promise; +} + +/** A section module's slice and counts, as a part. */ +function part(section: object, counts: object = {}): BackupExportPart { + return { + section: { ...section } as Record, + counts: { ...counts } as Record, + }; +} + +/** + * The slices of the file, in the order their keys appear in it. Every section + * is SPREAD, not hand-picked key by key: hand-picking is what lost the custom + * cycle symptoms, whose builder returned them while the list of keys copied + * out of it was never extended, so every backup carried `customSymptoms: []`. + * Spreading makes each section's own interface the declaration of what rides + * the wire. + * + * A key two slices both write is merged where it first appears: `manifest`, + * which the records slice opens and the sensitive slice adds its disclosures + * to. + */ +export const BACKUP_EXPORT_SECTIONS: readonly BackupExportSection[] = [ + // The host's own settings, in a disaster-recovery payload only. + { name: "appSettings", build: buildAppSettingsBackupPart }, + { + // The account's own settings: units, language, modules, thresholds, the + // AI choices without the keys. + name: "accountSettings", + build: async (ctx) => + part( + await buildAccountSettingsBackupSection(ctx.prisma, ctx.userId, { + purpose: ctx.purpose, + }), + ), + }, + { name: "measurements", build: buildMeasurementsBackupPart }, + { name: "medications", build: buildMedicationsBackupPart }, + { name: "mood", build: buildMoodBackupPart }, + { + name: "cycle", + build: async (ctx) => { + const cycle = await buildCycleBackupSection(ctx.prisma, ctx.userId, { + purpose: ctx.purpose, + }); + return part(cycle, { + cycles: cycle.cycles.length, + cycleDayLogs: cycle.cycleDayLogs.length, + }); + }, + }, + { + // Labs, illness, allergies, family history, workouts and documents, with + // the manifest that discloses what the file leaves out. Deferred, the + // documents keep their position (a key that already exists keeps its + // place when it is assigned again) and are read a document at a time by + // the streaming writer (#1239). + name: "records", + build: async (ctx) => { + const records = await buildRecordsBackupSection(ctx.prisma, ctx.userId, { + purpose: ctx.purpose, + deferDocuments: ctx.deferBulk, + }); + return part( + { + ...records, + ...(ctx.deferBulk + ? { + documents: deferredRows(() => + iterateBackupDocuments(ctx.prisma, ctx.userId, { + purpose: ctx.purpose, + }), + ), + } + : {}), + }, + countRecordsBackupSection(records), + ); + }, + }, + { + // Durable self-context, user-defined metrics, and persisted pattern + // decisions. + name: "profile", + build: async (ctx) => { + const profile = await buildProfileBackupSection(ctx.prisma, ctx.userId, { + purpose: ctx.purpose, + }); + return part(profile, countProfileBackupSection(profile)); + }, + }, + { + // The hourly shape of a cumulative day. + name: "intradayProfiles", + build: async (ctx) => { + const intraday = await buildIntradayProfileBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(intraday, countIntradayProfileBackupSection(intraday)); + }, + }, + { + // The score as it was SHOWN on each local day. + name: "healthScoreRecords", + build: async (ctx) => { + const score = await buildHealthScoreBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(score, countHealthScoreBackupSection(score)); + }, + }, + { + // The needs-based setup answers. + name: "onboardingRecord", + build: async (ctx) => { + const onboarding = await buildOnboardingBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(onboarding, countOnboardingBackupSection(onboarding)); + }, + }, + { + // Visits, the address book behind them, and the three link tables. + name: "visits", + build: async (ctx) => { + const visits = await buildVisitsBackupSection(ctx.prisma, ctx.userId, { + purpose: ctx.purpose, + }); + return part(visits, countVisitsBackupSection(visits)); + }, + }, + { + // The immunization log and the pages it was transcribed from. + name: "vaccinations", + build: async (ctx) => { + const vaccinations = await buildVaccinationsBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(vaccinations, countVaccinationsBackupSection(vaccinations)); + }, + }, + { + // The Vorsorge reminders and their completion ledger. + name: "reminders", + build: async (ctx) => { + const reminders = await buildRemindersBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(reminders, countRemindersBackupSection(reminders)); + }, + }, + { + // The Coach's conversations, turns and document links. + name: "coach", + build: async (ctx) => { + const coach = await buildCoachBackupSection(ctx.prisma, ctx.userId, { + purpose: ctx.purpose, + }); + return part(coach, countCoachBackupSection(coach)); + }, + }, + { + // What the Coach keeps BETWEEN threads. + name: "coachMemory", + build: async (ctx) => { + const memory = await buildCoachMemoryBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(memory, countCoachMemoryBackupSection(memory)); + }, + }, + { + // The screener history and the consent record, disaster-recovery only; + // its manifest entries join the records slice's manifest. + name: "sensitive", + build: async (ctx) => { + const sensitive = await buildSensitiveBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part( + { + mentalHealthAssessments: sensitive.mentalHealthAssessments, + consentReceipts: sensitive.consentReceipts, + manifest: sensitive.manifest, + }, + countSensitiveBackupSection(sensitive), + ); + }, + }, + { + // Which conditions a document was filed against, and what the + // extraction pass read out of it. + name: "documentFiling", + build: async (ctx) => { + const filing = await buildDocumentFilingBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(filing, countDocumentFilingBackupSection(filing)); + }, + }, + { + // The bests and the badges. No `purpose`: neither model tombstones and + // neither holds ciphertext, so both purposes carry the same bytes. + name: "awards", + build: async (ctx) => { + const awards = await buildAwardsBackupSection(ctx.prisma, ctx.userId); + return part(awards, countAwardsBackupSection(awards)); + }, + }, + { + // The per-day readings and the location periods that explain them. + name: "environment", + build: async (ctx) => { + const environment = await buildEnvironmentBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(environment, countEnvironmentBackupSection(environment)); + }, + }, + { + // The ECG strips, their rhythm verdicts and their traces; deferred, read + // a page of strips at a time by the streaming writer. + name: "ecg", + build: async (ctx) => { + const ecg = await buildEcgBackupSection(ctx.prisma, ctx.userId, { + purpose: ctx.purpose, + defer: ctx.deferBulk, + }); + return part( + { + ...ecg, + ...(ctx.deferBulk + ? { + ecgRecordings: deferredRows(() => + iterateBackupEcgRecordings(ctx.prisma, ctx.userId, { + purpose: ctx.purpose, + }), + ), + } + : {}), + }, + countEcgBackupSection(ecg), + ); + }, + }, + { + // The person's own symptoms with their occurrences nested inside. + name: "symptoms", + build: async (ctx) => { + const symptoms = await buildSymptomsBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(symptoms, countSymptomsBackupSection(symptoms)); + }, + }, + { + // The person's life events. + name: "lifeEvents", + build: async (ctx) => { + const lifeEvents = await buildLifeEventsBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(lifeEvents, countLifeEventsBackupSection(lifeEvents)); + }, + }, + { + // The five relation links that close the relation matrix. + name: "relationLinks", + build: async (ctx) => { + const links = await buildRelationLinksBackupSection( + ctx.prisma, + ctx.userId, + { purpose: ctx.purpose }, + ); + return part(links, countRelationLinksBackupSection(links)); + }, + }, + { name: "nutrients", build: buildNutrientsBackupPart }, +]; + +/** + * The order the audit trail lists the slices' row counts in. It is not the + * file's order: the counts were laid out before most sections existed, and + * every backup job's audit row since reads this way. + */ +export const BACKUP_EXPORT_COUNT_ORDER: readonly string[] = [ + "measurements", + "medications", + "mood", + "cycle", + "nutrients", + "records", + "profile", + "intradayProfiles", + "healthScoreRecords", + "onboardingRecord", + "visits", + "vaccinations", + "reminders", + "coach", + "coachMemory", + "sensitive", + "documentFiling", + "awards", + "environment", + "ecg", + "symptoms", + "lifeEvents", + "relationLinks", +]; diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index ac0408e93..f0786f13f 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -238,12 +238,19 @@ export const BACKED_UP_MODELS = [ * live, and the guard reads those files instead of guessing which ones matter. */ export const BACKUP_WRITER_FILES: readonly string[] = [ - "src/lib/export/full-backup-payload.ts", - // Measurements are read through this pager, not directly. The guard caught - // its absence from this list on the first run — the same "the caller - // delegates" mistake the list exists to prevent, made while writing the list. + "src/lib/export/measurements-backup.ts", + "src/lib/export/labs-backup.ts", + "src/lib/export/illness-backup.ts", + "src/lib/export/allergies-backup.ts", + "src/lib/export/family-history-backup.ts", + "src/lib/export/workouts-backup.ts", + "src/lib/export/nutrients-backup.ts", + "src/lib/export/documents-backup.ts", + "src/lib/export/medications-backup.ts", + "src/lib/export/mood-backup.ts", + // Measurements are read through this pager, not directly — the "caller + // delegates" mistake this list exists to prevent, caught on its first run. "src/lib/export/paged-measurements.ts", - "src/lib/export/records-backup.ts", // The illness day-log's symptom include is a shared constant living here, // not a literal in the records writer. Same "the caller delegates" shape as // the measurement pager above, found the same way: by a check that stopped @@ -273,7 +280,7 @@ export const BACKUP_WRITER_FILES: readonly string[] = [ // reasons for that live in the module itself. "src/lib/export/sensitive-backup.ts", // What a document was filed against and what was read out of it. The - // documents themselves are read by `records-backup.ts`; these two read + // documents themselves are read by `documents-backup.ts`; these two read // through their OWN delegates here, for the reason the visits comment gives. "src/lib/export/document-filing-backup.ts", // The bests and the badges, and the environmental history with the location @@ -300,30 +307,22 @@ export const BACKUP_WRITER_FILES: readonly string[] = [ // ends beside each other, each through its OWN delegate. "src/lib/export/relation-links-backup.ts", "src/lib/cycle/backup.ts", - // The medication category is read through the one helper that normalises - // it; the payload writer calls the helper rather than the delegate. The - // same "the caller delegates" shape as the measurement pager. + // The medication category, read and written through its one helper. "src/lib/medication-category.ts", ]; const WRITER_FILES_WITHOUT_RESTORE = new Set([ - "src/lib/export/full-backup-payload.ts", + "src/lib/export/measurements-backup.ts", + "src/lib/export/medications-backup.ts", + "src/lib/export/mood-backup.ts", "src/lib/export/paged-measurements.ts", - "src/lib/export/records-backup.ts", "src/lib/illness/dto.ts", ]); export const BACKUP_RESTORE_FILES: readonly string[] = [ - "src/lib/export/measurements-backup.ts", + "src/lib/export/measurements-restore.ts", "src/lib/export/medications-restore.ts", "src/lib/export/mood-restore.ts", - "src/lib/export/labs-backup.ts", - "src/lib/export/illness-backup.ts", - "src/lib/export/allergies-backup.ts", - "src/lib/export/family-history-backup.ts", - "src/lib/export/workouts-backup.ts", - "src/lib/export/nutrients-backup.ts", - "src/lib/export/documents-backup.ts", ...BACKUP_WRITER_FILES.filter( (file) => !WRITER_FILES_WITHOUT_RESTORE.has(file), ), diff --git a/src/lib/export/deferred-rows.ts b/src/lib/export/deferred-rows.ts new file mode 100644 index 000000000..0d2f8dd29 --- /dev/null +++ b/src/lib/export/deferred-rows.ts @@ -0,0 +1,54 @@ +/** + * A bulk table the payload declares but has NOT read. + * + * `buildFullBackupPayload({ deferBulk: true })` puts one of these where the + * row array would sit. The streaming writer recognises it, opens the JSON + * array itself and pulls rows through page by page, so a table with half a + * million rows never exists as an array at all. Every other caller leaves + * `deferBulk` unset and gets the arrays, unchanged. + * + * The marker key is a symbol so it cannot collide with a payload field and so + * the deferred payload can never be handed to `JSON.stringify` by accident and + * come back looking merely empty — `streamFullBackupJson` is the only reader. + */ +const DEFERRED_ROWS = Symbol("healthlog.backup.deferredRows"); + +export interface DeferredRows { + readonly [DEFERRED_ROWS]: true; + /** + * Open the row source. Called exactly once, by the streaming writer. A row + * is a plain object, or a `StreamedField` (`streamed-field.ts`) for a row + * with one field too large to hold as a string. + */ + rows(): AsyncIterable; +} + +export function deferredRows(rows: () => AsyncIterable): DeferredRows { + return { [DEFERRED_ROWS]: true, rows }; +} + +/** True when a payload value is a deferred bulk table rather than an array. */ +export function isDeferredRows(value: unknown): value is DeferredRows { + return ( + typeof value === "object" && + value !== null && + (value as Record)[DEFERRED_ROWS] === true + ); +} + +/** + * How many rows one page of a bulk table carries. + * + * Smaller than the measurement pager's 10 000 on purpose: an intake event + * drags its medication's name along and a mood entry drags its links and its + * day context, so a page of either costs several times what a page of + * measurements costs. The number is a memory budget, not a throughput knob. + */ +export const BULK_PAGE_SIZE = 5_000; + +/** Drain an async row source into an array. The materialising arm's only use. */ +export async function collect(rows: AsyncIterable): Promise { + const out: T[] = []; + for await (const row of rows) out.push(row); + return out; +} diff --git a/src/lib/export/documents-backup.ts b/src/lib/export/documents-backup.ts index 0ae41386c..3e17610b6 100644 --- a/src/lib/export/documents-backup.ts +++ b/src/lib/export/documents-backup.ts @@ -4,6 +4,13 @@ * A disaster-recovery file carries each document's stored bytes verbatim; a * portable one carries metadata only and cannot be restored into documents. */ +import { Buffer } from "node:buffer"; + +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; +import { decryptDocumentSummary } from "@/lib/documents/store"; +import { getEvent } from "@/lib/logging/context"; +import { dateOnlyKey } from "@/lib/tz/date-only"; +import { streamedField, type StreamedField } from "@/lib/export/streamed-field"; import { decodeEncryptedBytes, type RestoreContext, @@ -11,6 +18,305 @@ import { import { normaliseSourceInstance } from "@/lib/validations/inbound-documents"; import type { BackupPayload } from "@/lib/validations/backup"; +// ── Export ─────────────────────────────────────────────────────────────────── + +export interface DocumentsBackupOptions { + purpose?: "portable-export" | "disaster-recovery"; +} + +export interface DocumentBackupEntry { + id: string; + kind: string; + title: string | null; + filename: string | null; + mimeType: string; + byteSize: number; + status: string; + reportDate: string | null; + documentDate: string | null; + /** Portable export only; canonical DR carries summaryEncrypted instead. */ + summary?: string | null; + createdAt: string; + updatedAt?: string; + /** Base64 of the already-encrypted BYTEA value; never plaintext. */ + contentEncrypted?: string; + contentSha256?: string | null; + contentCodec?: string; + providerType?: string | null; + errorReason?: string | null; + /** Base64 of the already-encrypted summary BYTEA value. */ + summaryEncrypted?: string | null; + summaryGeneratedAt?: string | null; + summaryState?: string; + /** Refs #776 — the index-attempt record, carried verbatim (canonical DR). */ + lastIndexAttemptAt?: string | null; + lastIndexOutcome?: string | null; + /** + * Where an imported document came from, both payload + * kinds. Optional so a file written before the fields existed still parses. + */ + sourceSystem?: string | null; + sourceId?: string | null; + /** The source instance (origin); absent in older files. */ + sourceInstance?: string | null; + /** Canonical DR only: the import's hold on automatic AI reading. */ + aiReadDeferred?: boolean; +} + +/** The document columns both purposes carry. */ +const DOCUMENT_PORTABLE_SELECT = { + id: true, + kind: true, + title: true, + filename: true, + mimeType: true, + byteSize: true, + status: true, + reportDate: true, + documentDate: true, + summaryEncrypted: true, + sourceSystem: true, + sourceId: true, + sourceInstance: true, + createdAt: true, +} as const satisfies Prisma.InboundDocumentSelect; + +/** + * A disaster-recovery document without its stored bytes. The paged reader + * reads this for every document at once and the bytes one slice at a time. + */ +const DOCUMENT_DR_METADATA_SELECT = { + ...DOCUMENT_PORTABLE_SELECT, + contentSha256: true, + contentCodec: true, + providerType: true, + errorReason: true, + summaryGeneratedAt: true, + summaryState: true, + lastIndexAttemptAt: true, + lastIndexOutcome: true, + aiReadDeferred: true, + updatedAt: true, +} as const satisfies Prisma.InboundDocumentSelect; + +const DOCUMENT_DR_SELECT = { + ...DOCUMENT_DR_METADATA_SELECT, + contentEncrypted: true, +} as const satisfies Prisma.InboundDocumentSelect; + +/** Total, so two exports of the same account list the documents alike. */ +const DOCUMENT_ORDER = [ + { createdAt: "desc" }, + { id: "desc" }, +] as const satisfies Prisma.InboundDocumentOrderByWithRelationInput[]; + +type PortableDocumentRow = Prisma.InboundDocumentGetPayload<{ + select: typeof DOCUMENT_PORTABLE_SELECT; +}>; +type DisasterRecoveryDocumentRow = Prisma.InboundDocumentGetPayload<{ + select: typeof DOCUMENT_DR_METADATA_SELECT; +}>; +type DocumentRow = + | PortableDocumentRow + | Prisma.InboundDocumentGetPayload<{ select: typeof DOCUMENT_DR_SELECT }>; + +/** + * One document as it rides the wire. `contentEncrypted` is the base64 of the + * stored bytes for a disaster-recovery entry and null for a portable one, + * which carries the readable summary instead. Shared by the materialising + * reader and the paged one, so the two cannot drift. + */ +function serialiseDocument( + d: PortableDocumentRow | DisasterRecoveryDocumentRow, + contentEncrypted: string | null, +): DocumentBackupEntry { + const metadata = { + id: d.id, + kind: d.kind, + title: d.title, + filename: d.filename, + mimeType: d.mimeType, + byteSize: d.byteSize, + status: d.status, + reportDate: d.reportDate ? dateOnlyKey(d.reportDate) : null, + documentDate: d.documentDate ? dateOnlyKey(d.documentDate) : null, + sourceSystem: d.sourceSystem, + sourceId: d.sourceId, + sourceInstance: d.sourceInstance, + createdAt: d.createdAt.toISOString(), + }; + + if (contentEncrypted !== null && "contentCodec" in d) { + const dr: DisasterRecoveryDocumentRow = d; + return { + ...metadata, + reportDate: dr.reportDate?.toISOString() ?? null, + documentDate: dr.documentDate?.toISOString() ?? null, + updatedAt: dr.updatedAt.toISOString(), + contentEncrypted, + contentSha256: dr.contentSha256, + contentCodec: dr.contentCodec, + providerType: dr.providerType, + errorReason: dr.errorReason, + summaryEncrypted: dr.summaryEncrypted + ? Buffer.from(dr.summaryEncrypted).toString("base64") + : null, + summaryGeneratedAt: dr.summaryGeneratedAt?.toISOString() ?? null, + summaryState: dr.summaryState, + lastIndexAttemptAt: dr.lastIndexAttemptAt?.toISOString() ?? null, + lastIndexOutcome: dr.lastIndexOutcome, + aiReadDeferred: dr.aiReadDeferred, + }; + } + + return { + ...metadata, + summary: decryptSummarySoft(d.summaryEncrypted), + }; +} + +/** + * How much of a document's stored bytes one read takes. A multiple of three, + * so the base64 of every slice joins into the base64 of the whole. + */ +const DOCUMENT_SLICE_BYTES = 1536 * 1024; + +/** + * How much of a document's base64 the streaming writer shows its observer + * (the restore preview). The key id sits in the first characters; the cap is + * above the longest value the preview keeps as a key sample, so a shortened + * value is never kept as one. A multiple of four, so it still reads as base64. + */ +const DOCUMENT_OBSERVED_CHARS = 128 * 1024; + +/** + * Every document of the account, one at a time, for the streaming backup + * writer (#1239). + * + * A document is up to a hundred megabytes, and the materialising read holds + * every document's bytes, their base64 and the section's JSON at once: about + * five times the size of the vault in heap, which a 524 MB process does not + * have for a vault of 100 MB. Here the metadata of all documents is read + * once, and each document's bytes are read a slice at a time and handed on as + * base64 pieces (`streamedField`), so what is held is one slice. + * + * A document that disappears between the metadata read and its first slice + * is left out, as a later read would not have seen it. One that disappears + * halfway fails the copy: half a document is not a document. + */ +export async function* iterateBackupDocuments( + prisma: Pick, + userId: string, + options: DocumentsBackupOptions = {}, +): AsyncGenerator { + if (options.purpose !== "disaster-recovery") { + const rows = await prisma.inboundDocument.findMany({ + where: { userId, deletedAt: null }, + orderBy: DOCUMENT_ORDER, + select: DOCUMENT_PORTABLE_SELECT, + }); + for (const row of rows) yield serialiseDocument(row, null); + return; + } + const rows = await prisma.inboundDocument.findMany({ + where: { userId, deletedAt: null }, + orderBy: DOCUMENT_ORDER, + select: DOCUMENT_DR_METADATA_SELECT, + }); + const slice = async (id: string, from: number): Promise => { + const [row] = await prisma.$queryRaw>` + SELECT substring(content_encrypted FROM ${from + 1}::int + FOR ${DOCUMENT_SLICE_BYTES}::int) AS part + FROM inbound_documents + WHERE id = ${id} AND user_id = ${userId} + `; + if (!row) return null; + const part = row.part ?? new Uint8Array(0); + // A view, not a copy: the slice is read once and dropped. + return Buffer.from(part.buffer, part.byteOffset, part.byteLength); + }; + for (const row of rows) { + const first = await slice(row.id, 0); + if (first === null) continue; + const head = first.toString("base64"); + const pieces = async function* (): AsyncGenerator { + yield head; + let last = first.byteLength; + for (let at = last; last === DOCUMENT_SLICE_BYTES; at += last) { + const next = await slice(row.id, at); + if (next === null) { + throw new Error( + `Document ${row.id} was removed while it was being backed up; ` + + "the next run copies the account again.", + ); + } + last = next.byteLength; + if (last > 0) yield next.toString("base64"); + } + }; + yield streamedField( + serialiseDocument(row, ""), + "contentEncrypted", + head.length > DOCUMENT_OBSERVED_CHARS + ? head.slice(0, DOCUMENT_OBSERVED_CHARS) + : head, + pieces, + ); + } +} + +/** Decrypt a document's optional AI summary fail-soft (null on any error). */ +function decryptSummarySoft(buf: Uint8Array | null): string | null { + if (!buf || buf.byteLength === 0) return null; + try { + return decryptDocumentSummary(buf); + } catch (err) { + getEvent()?.addWarning( + `document summary decrypt failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return null; + } +} + +/** + * The document slice of the records section. A portable entry carries the + * metadata and the readable summary; a disaster-recovery one the stored + * bytes verbatim, unless the caller reads them itself (`deferDocuments`). + */ +export async function buildDocumentsBackupSlice( + prisma: Pick, + userId: string, + disasterRecovery: boolean, + deferDocuments: boolean, +): Promise<{ documents: DocumentBackupEntry[] }> { + const documentRows: DocumentRow[] = deferDocuments + ? [] + : disasterRecovery + ? await prisma.inboundDocument.findMany({ + where: { userId, deletedAt: null }, + orderBy: DOCUMENT_ORDER, + select: DOCUMENT_DR_SELECT, + }) + : await prisma.inboundDocument.findMany({ + where: { userId, deletedAt: null }, + orderBy: DOCUMENT_ORDER, + select: DOCUMENT_PORTABLE_SELECT, + }); + + const documents: DocumentBackupEntry[] = documentRows.map((d) => + serialiseDocument( + d, + disasterRecovery && "contentEncrypted" in d + ? Buffer.from(d.contentEncrypted).toString("base64") + : null, + ), + ); + + return { documents }; +} + // ── Restore ────────────────────────────────────────────────────────────────── /** diff --git a/src/lib/export/export-context.ts b/src/lib/export/export-context.ts new file mode 100644 index 000000000..29d4864c0 --- /dev/null +++ b/src/lib/export/export-context.ts @@ -0,0 +1,30 @@ +/** + * What every domain's export is handed, and what it hands back. + * + * Each domain reads its own tables and returns its slice of the payload with + * its row counts. `buildFullBackupPayload` lays the slices out in the order + * `backup-domains.ts` lists them, which is the order the file has always had. + */ +import type { PrismaClient } from "@/generated/prisma/client"; + +export type BackupPurpose = "portable-export" | "disaster-recovery"; + +export interface BackupExportContext { + readonly prisma: PrismaClient; + readonly userId: string; + readonly purpose: BackupPurpose; + /** `purpose === "disaster-recovery"`, spelled out because every row asks. */ + readonly disasterRecovery: boolean; + /** + * Leave the bulk tables unread and put `DeferredRows` markers where their + * arrays sit (`deferred-rows.ts`); only the streaming writer asks for this. + */ + readonly deferBulk: boolean; +} + +export interface BackupExportPart { + /** The payload keys this domain writes, in the order they appear. */ + section: Record; + /** Row counts for the audit trail, in the order they appear. */ + counts: Record; +} diff --git a/src/lib/export/family-history-backup.ts b/src/lib/export/family-history-backup.ts index f678b7603..d24407eb6 100644 --- a/src/lib/export/family-history-backup.ts +++ b/src/lib/export/family-history-backup.ts @@ -1,9 +1,29 @@ /** * The family history, with both backup ends in one file. */ +import type { PrismaClient } from "@/generated/prisma/client"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; +import { + toFamilyHistoryEntryDTO, + type FamilyHistoryEntryDTO, +} from "@/lib/records/dto"; import type { RestoreContext } from "@/lib/export/restore-context"; +// ── Export ─────────────────────────────────────────────────────────────────── + +/** The family-history slice of the records section. */ +export async function buildFamilyHistoryBackupSlice( + prisma: Pick, + userId: string, +): Promise<{ familyHistory: FamilyHistoryEntryDTO[] }> { + const familyRows = await prisma.familyHistoryEntry.findMany({ + where: { userId, deletedAt: null }, + orderBy: { createdAt: "desc" }, + }); + const familyHistory = familyRows.map(toFamilyHistoryEntryDTO); + return { familyHistory }; +} + // ── Restore ────────────────────────────────────────────────────────────────── export async function clearFamilyHistory(ctx: RestoreContext): Promise { diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index 7702e7f96..5da9ef80d 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -1,5 +1,5 @@ /** - * v1.23 — shared builder for the user-scoped full-backup payload. + * The shared builder for the user-scoped full-backup payload. * * Extracted so both the plaintext `GET /api/export/full-backup` route and the * passphrase-encrypted `POST /api/export/encrypted` route emit the byte-for-byte @@ -14,141 +14,42 @@ * the whole payload, which is what the two export routes and every test want. * `streamFullBackupJson` (`full-backup-stream.ts`) asks for the same payload * with `deferBulk`, which leaves the unbounded tables (and the two whose rows - * carry stored bytes) as markers and pulls their rows through itself — the weekly job takes that path because - * materialising a large record is what used to exhaust the container's heap. - * Both go through the same per-row serialisers below, so a column that reaches - * one reaches the other; the integration suite pins the two outputs byte for + * carry stored bytes) as markers and pulls their rows through itself — the + * weekly job takes that path because materialising a large record is what + * used to exhaust the container's heap. Both go through the same per-row + * serialisers, each in its domain's module, so a column that reaches one + * reaches the other; the integration suite pins the two outputs byte for * byte rather than trusting that sentence. */ -import { decryptFromBytes } from "@/lib/ai/coach/bytes-codec"; -import { dayKeyOfDate } from "@/lib/medications/course-window"; -import { Buffer } from "node:buffer"; -import type { Prisma, PrismaClient } from "@/generated/prisma/client"; -import { readNote } from "@/lib/crypto/note-cipher"; -import { openSealedForExport } from "@/lib/export/sealed-text"; -import { iterateMeasurementPages } from "@/lib/export/paged-measurements"; -import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup-summary"; +import type { PrismaClient } from "@/generated/prisma/client"; +import type { AwardsBackupCounts } from "@/lib/export/awards-backup"; import { - buildCycleBackupSection, - type CycleBackupSection, -} from "@/lib/cycle/backup"; -import { - buildRecordsBackupSection, - countRecordsBackupSection, - iterateBackupDocuments, - type RecordsBackupCounts, - type RecordsBackupSection, -} from "@/lib/export/records-backup"; -import { - buildProfileBackupSection, - countProfileBackupSection, - type ProfileBackupCounts, - type ProfileBackupSection, -} from "@/lib/export/profile-backup"; -import { - buildHealthScoreBackupSection, - countHealthScoreBackupSection, - type HealthScoreBackupCounts, - type HealthScoreBackupSection, -} from "@/lib/export/health-score-backup"; -import { - buildVisitsBackupSection, - countVisitsBackupSection, - type VisitsBackupCounts, - type VisitsBackupSection, -} from "@/lib/export/visits-backup"; -import { - buildVaccinationsBackupSection, - countVaccinationsBackupSection, - type VaccinationsBackupCounts, - type VaccinationsBackupSection, -} from "@/lib/export/vaccinations-backup"; -import { - buildRemindersBackupSection, - countRemindersBackupSection, - type RemindersBackupCounts, - type RemindersBackupSection, -} from "@/lib/export/reminders-backup"; -import { - buildSensitiveBackupSection, - countSensitiveBackupSection, - type SensitiveBackupCounts, - type SensitiveBackupManifest, - type SensitiveBackupSection, -} from "@/lib/export/sensitive-backup"; -import { - buildCoachBackupSection, - buildCoachMemoryBackupSection, - countCoachBackupSection, - countCoachMemoryBackupSection, - type CoachBackupCounts, - type CoachBackupSection, - type CoachMemoryBackupCounts, - type CoachMemoryBackupSection, + BACKUP_EXPORT_COUNT_ORDER, + BACKUP_EXPORT_SECTIONS, +} from "@/lib/export/backup-export-sections"; +import type { + CoachBackupCounts, + CoachMemoryBackupCounts, } from "@/lib/export/coach-backup"; -import { - buildDocumentFilingBackupSection, - countDocumentFilingBackupSection, - type DocumentFilingBackupCounts, - type DocumentFilingBackupSection, -} from "@/lib/export/document-filing-backup"; -import { - buildRelationLinksBackupSection, - countRelationLinksBackupSection, - type RelationLinksBackupCounts, - type RelationLinksBackupSection, -} from "@/lib/export/relation-links-backup"; -import { - buildAwardsBackupSection, - countAwardsBackupSection, - type AwardsBackupCounts, - type AwardsBackupSection, -} from "@/lib/export/awards-backup"; -import { - buildEnvironmentBackupSection, - countEnvironmentBackupSection, - type EnvironmentBackupCounts, - type EnvironmentBackupSection, -} from "@/lib/export/environment-backup"; -import { - buildEcgBackupSection, - countEcgBackupSection, - iterateBackupEcgRecordings, - type EcgBackupCounts, - type EcgBackupSection, -} from "@/lib/export/ecg-backup"; -import { - buildSymptomsBackupSection, - countSymptomsBackupSection, - type SymptomsBackupCounts, - type SymptomsBackupSection, -} from "@/lib/export/symptoms-backup"; -import { - buildLifeEventsBackupSection, - countLifeEventsBackupSection, - type LifeEventsBackupCounts, - type LifeEventsBackupSection, -} from "@/lib/export/life-events-backup"; -import { - buildIntradayProfileBackupSection, - countIntradayProfileBackupSection, - type IntradayProfileBackupCounts, - type IntradayProfileBackupSection, -} from "@/lib/export/intraday-profile-backup"; -import { - buildOnboardingBackupSection, - countOnboardingBackupSection, - type OnboardingBackupCounts, - type OnboardingBackupSection, -} from "@/lib/export/onboarding-backup"; -import { - buildAccountSettingsBackupSection, - type AccountSettingsBackupSection, -} from "@/lib/export/account-settings-backup"; -import { - getMedicationCategories, - readMedicationCategoryLabelsForBackup, -} from "@/lib/medication-category"; +import type { DocumentFilingBackupCounts } from "@/lib/export/document-filing-backup"; +import type { EcgBackupCounts } from "@/lib/export/ecg-backup"; +import type { EnvironmentBackupCounts } from "@/lib/export/environment-backup"; +import type { BackupExportContext } from "@/lib/export/export-context"; +import type { HealthScoreBackupCounts } from "@/lib/export/health-score-backup"; +import type { IntradayProfileBackupCounts } from "@/lib/export/intraday-profile-backup"; +import type { LifeEventsBackupCounts } from "@/lib/export/life-events-backup"; +import type { OnboardingBackupCounts } from "@/lib/export/onboarding-backup"; +import type { ProfileBackupCounts } from "@/lib/export/profile-backup"; +import type { RecordsBackupCounts } from "@/lib/export/records-backup"; +import type { RelationLinksBackupCounts } from "@/lib/export/relation-links-backup"; +import type { RemindersBackupCounts } from "@/lib/export/reminders-backup"; +import type { SensitiveBackupCounts } from "@/lib/export/sensitive-backup"; +import type { SymptomsBackupCounts } from "@/lib/export/symptoms-backup"; +import type { VaccinationsBackupCounts } from "@/lib/export/vaccinations-backup"; +import type { VisitsBackupCounts } from "@/lib/export/visits-backup"; +import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup-summary"; + +export { isDeferredRows, type DeferredRows } from "@/lib/export/deferred-rows"; export interface FullBackupCounts extends @@ -207,512 +108,15 @@ export interface FullBackupOptions { */ deferBulk?: boolean; } -/** - * Every restorable `Measurement` scalar. `userId` is deliberately absent — the - * backup is user-scoped, so the owner is implicit in the query and restored - * from the target account rather than the payload. - * - * Named rather than inlined so `measurement-backup-completeness.test.ts` can - * compare it field-by-field against the model in `prisma/schema.prisma`. A - * column added to the model and forgotten here is how the aggregation - * provenance went missing from disaster-recovery backups for several releases. - */ -const MEASUREMENT_BACKUP_SELECT = { - id: true, - type: true, - value: true, - valueMin: true, - valueMax: true, - unit: true, - measuredAt: true, - source: true, - notes: true, - notesEncrypted: true, - externalId: true, - externalSourceVersion: true, - aggregationProvenance: true, - glucoseContext: true, - sleepStage: true, - rhythmClassification: true, - deviceType: true, - syncVersion: true, - deletedAt: true, - createdAt: true, - updatedAt: true, -} as const satisfies Prisma.MeasurementSelect; -/** - * Every restorable `MoodEntry` scalar. `userId` is absent for the same reason - * it is absent above: the backup is user-scoped and the owner is re-derived - * from the restoring account. - * - * Named rather than inlined so `mood-backup-completeness.test.ts` can compare - * it field-by-field against the model in `prisma/schema.prisma`, exactly as the - * measurement guard does for the select above. - */ -const MOOD_ENTRY_BACKUP_SELECT = { - id: true, - date: true, - mood: true, - score: true, - moodA1: true, - stressA2: true, - energyA3: true, - connectionA4: true, - stabilityA5: true, - tags: true, - note: true, - noteEncrypted: true, - source: true, - externalId: true, - moodLoggedAt: true, - tz: true, - syncedAt: true, - syncVersion: true, - deletedAt: true, - createdAt: true, - updatedAt: true, -} as const satisfies Prisma.MoodEntrySelect; - -/** - * A structured-tag link, as it rides the wire. The link is carried by the - * tag's KEY, not by its id: a restore resolves the key against the target - * instance's catalogue, where the source instance's row id may not exist at - * all. `kind` comes along so the mapper can split RATED factors from BINARY - * tags without a second query. - */ -const MOOD_ENTRY_TAG_LINK_SELECT = { - rating: true, - moodTag: { select: { key: true, kind: true } }, -} as const satisfies Prisma.MoodEntryTagLinkSelect; - -/** - * Every restorable `MoodContext` scalar. - * - * `moodEntryId` and `userId` are absent for the reasons the exclusion list in - * `mood-backup-completeness.test.ts` states: the context rides inside its - * entry and is re-bound to whatever id that entry got, and the owner is - * re-derived from the restoring account. The two row stamps are absent for the - * third reason there — they describe when the sub-row was written, not when - * the day happened, and the entry already carries the moment that matters. - * - * Named rather than inlined so the guard can compare it field-by-field against - * the model, exactly as it does for the two selects above. This is the whole - * point of the pair: a column added to `MoodContext` without a line here - * fails, and it fails in the release that added it. - */ -const MOOD_CONTEXT_BACKUP_SELECT = { - workStatus: true, - workMinutes: true, - overtimeMinutes: true, - workLoad: true, - workSatisfaction: true, - contactCircles: true, - contactForm: true, - contactExtent: true, - contactQuality: true, - contactSupport: true, - leisureCategories: true, - leisureMinutes: true, - leisureJoy: true, - leisureRecovery: true, - eventType: true, - eventValence: true, - eventAt: true, - notesEncrypted: true, -} as const satisfies Prisma.MoodContextSelect; - -type BackupMoodContext = Prisma.MoodContextGetPayload<{ - select: typeof MOOD_CONTEXT_BACKUP_SELECT; -}>; - -type BackupMeasurement = Prisma.MeasurementGetPayload<{ - select: typeof MEASUREMENT_BACKUP_SELECT; -}>; - -type BackupIntakeEvent = Prisma.MedicationIntakeEventGetPayload<{ - include: { medication: { select: { name: true } } }; -}>; - -/** - * The mood read, select and joins together. Named because two readers issue - * it — the materialising one and the paged one — and a select that lived in - * only one of them is exactly the drift the completeness guard cannot see. - */ -const MOOD_ENTRY_QUERY_SELECT = { - ...MOOD_ENTRY_BACKUP_SELECT, - // Every link, not only the rated ones. This read used to filter - // `rating: { not: null }`, and a BINARY link carries a NULL rating by - // definition — so the present/absent tags, the kind most people pick, - // were absent from the file and could not come back from it. - tagLinks: { select: MOOD_ENTRY_TAG_LINK_SELECT }, - // The day context, when the entry has one. Sparse by design — most - // entries carry none, and the join answers null on those. - context: { select: MOOD_CONTEXT_BACKUP_SELECT }, -} as const satisfies Prisma.MoodEntrySelect; - -type BackupMoodEntry = Prisma.MoodEntryGetPayload<{ - select: typeof MOOD_ENTRY_QUERY_SELECT; -}>; - -/** - * One context row, in whichever of the two wire shapes the caller asked for. - * - * The split is the same one `Measurement` makes: a disaster-recovery file - * carries the ciphertext verbatim, because the instance that reads it back - * holds the key; a portable export carries the plain text, because the point - * of a portable file is that somebody can read it. Every other column rides in - * both, since none of them is a storage detail — they are all answers somebody - * gave. - * - * The parameter is spelled `moodContext` rather than `c`, deliberately: the - * completeness guard proves a selected column reaches a payload by matching - * `moodContext.`, and a one-letter parameter shared with another - * mapper in this file would make that matcher unable to fail. - */ -function serialiseMoodContext( - moodContext: BackupMoodContext, - disasterRecovery: boolean, -) { - return { - ...(disasterRecovery - ? { - notesEncrypted: moodContext.notesEncrypted - ? Buffer.from(moodContext.notesEncrypted).toString("base64") - : null, - } - : { note: readNote(moodContext.notesEncrypted, null) }), - workStatus: moodContext.workStatus, - workMinutes: moodContext.workMinutes, - overtimeMinutes: moodContext.overtimeMinutes, - workLoad: moodContext.workLoad, - workSatisfaction: moodContext.workSatisfaction, - // The two multi-selects ride as the stored JSON string, unparsed. The - // restore writes the column back verbatim, so a file cannot arrive with a - // list the parser and the column disagree about. - contactCircles: moodContext.contactCircles, - contactForm: moodContext.contactForm, - contactExtent: moodContext.contactExtent, - contactQuality: moodContext.contactQuality, - contactSupport: moodContext.contactSupport, - leisureCategories: moodContext.leisureCategories, - leisureMinutes: moodContext.leisureMinutes, - leisureJoy: moodContext.leisureJoy, - leisureRecovery: moodContext.leisureRecovery, - eventType: moodContext.eventType, - eventValence: moodContext.eventValence, - eventAt: moodContext.eventAt?.toISOString() ?? null, - }; -} - -/** - * One measurement, in whichever of the two wire shapes the caller asked for. - * - * Named rather than inlined so the paged reader below and the materialising - * reader beside it cannot drift: both call this, so a column that reaches one - * shape reaches the other by construction. `measurement-backup-completeness. - * test.ts` proves a selected column reaches a payload by matching - * `: measurement.`, which is why the parameter keeps that spelling. - */ -function serialiseMeasurement( - measurement: BackupMeasurement, - disasterRecovery: boolean, -) { - return disasterRecovery - ? { - id: measurement.id, - type: measurement.type, - value: measurement.value, - valueMin: measurement.valueMin, - valueMax: measurement.valueMax, - unit: measurement.unit, - measuredAt: measurement.measuredAt.toISOString(), - source: measurement.source, - notes: measurement.notes, - notesEncrypted: measurement.notesEncrypted - ? Buffer.from(measurement.notesEncrypted).toString("base64") - : null, - externalId: measurement.externalId, - externalSourceVersion: measurement.externalSourceVersion, - aggregationProvenance: measurement.aggregationProvenance, - glucoseContext: measurement.glucoseContext, - sleepStage: measurement.sleepStage, - rhythmClassification: measurement.rhythmClassification, - deviceType: measurement.deviceType, - syncVersion: measurement.syncVersion, - deletedAt: measurement.deletedAt?.toISOString() ?? null, - createdAt: measurement.createdAt.toISOString(), - updatedAt: measurement.updatedAt.toISOString(), - } - : { - id: measurement.id, - type: measurement.type, - value: measurement.value, - unit: measurement.unit, - measuredAt: measurement.measuredAt.toISOString(), - source: measurement.source, - notes: readNote(measurement.notesEncrypted, measurement.notes), - deletedAt: measurement.deletedAt?.toISOString() ?? null, - }; -} - -/** One intake event, as it rides the wire. */ -function serialiseIntakeEvent(e: BackupIntakeEvent, disasterRecovery: boolean) { - return { - ...(disasterRecovery - ? { - id: e.id, - medicationId: e.medicationId, - autoMissed: e.autoMissed, - attributionSource: e.attributionSource, - idempotencyKey: e.idempotencyKey, - createdAt: e.createdAt.toISOString(), - injectionSite: e.injectionSite, - doseTaken: e.doseTaken, - inventoryConsumption: e.inventoryConsumption, - externalId: e.externalId, - updatedAt: e.updatedAt.toISOString(), - syncVersion: e.syncVersion, - deletedAt: e.deletedAt?.toISOString() ?? null, - } - : {}), - medication: e.medication.name, - scheduledFor: e.scheduledFor.toISOString(), - takenAt: e.takenAt?.toISOString() ?? null, - skipped: e.skipped, - source: e.source, - // v1.42.2 — both purposes: the history labels the dose "recorded - // automatically", and a restore that dropped the marker would relabel - // it as one the person confirmed. - autoRecorded: e.autoRecorded, - }; -} - -/** - * One mood entry, with its links partitioned and its day context folded in. - * - * The parameter is spelled out rather than the `e` every other mapper here - * uses, and deliberately: `mood-backup-completeness.test.ts` proves a - * selected column reaches a payload by matching `: moodEntry.`. A matcher - * on `: e.` would also match the intake-event mapper above it and could - * therefore never fail. - */ -function serialiseMoodEntry( - moodEntry: BackupMoodEntry, - disasterRecovery: boolean, -) { - // The two link arms partition the entry's links exhaustively: a link is - // a rated factor when its tag is RATED and it actually carries a score, - // and a structured tag otherwise. Total on purpose — a malformed link - // (a RATED tag whose rating went missing) rides as a tag key and comes - // back NAMED as unresolvable on restore, rather than falling between the - // two arms and disappearing the way every BINARY link used to. - const isRatedFactor = (link: (typeof moodEntry.tagLinks)[number]) => - link.moodTag.kind === "RATED" && link.rating !== null; - - const factors = moodEntry.tagLinks.filter(isRatedFactor).map((tagLink) => ({ - key: tagLink.moodTag.key, - rating: tagLink.rating as number, - })); - // BINARY link keys. Their own array rather than a nullable rating inside - // `factors`: the restore resolves `factors` against `kind: "RATED"` and - // reports what it cannot resolve, so a rating-less entry in that array - // would collide with both. A separate key leaves every backup file - // written before this parsing and restoring exactly as it did. - const structuredTags = moodEntry.tagLinks - .filter((link) => !isRatedFactor(link)) - .map((tagLink) => tagLink.moodTag.key); - - return { - ...(disasterRecovery - ? { - note: moodEntry.note, - noteEncrypted: moodEntry.noteEncrypted - ? Buffer.from(moodEntry.noteEncrypted).toString("base64") - : null, - syncedAt: moodEntry.syncedAt.toISOString(), - syncVersion: moodEntry.syncVersion, - createdAt: moodEntry.createdAt.toISOString(), - updatedAt: moodEntry.updatedAt.toISOString(), - } - : { note: readNote(moodEntry.noteEncrypted, moodEntry.note) }), - id: moodEntry.id, - date: moodEntry.date, - mood: moodEntry.mood, - score: moodEntry.score, - // The five level-A values, under the keys the write path takes. Both - // arms carry them: they are the entry's own answers, not a storage - // detail, and a portable export that dropped them would hand back a - // file describing the day less well than the row it came from. - a1: moodEntry.moodA1, - a2: moodEntry.stressA2, - a3: moodEntry.energyA3, - a4: moodEntry.connectionA4, - a5: moodEntry.stabilityA5, - tags: moodEntry.tags, - source: moodEntry.source, - loggedAt: moodEntry.moodLoggedAt.toISOString(), - externalId: moodEntry.externalId, - // The zone the `date` string is anchored to. Without it a restored row - // falls back to the legacy Europe/Berlin reading and an account that - // logged elsewhere gets its day boundaries quietly moved. - tz: moodEntry.tz, - deletedAt: moodEntry.deletedAt?.toISOString() ?? null, - factors, - structuredTags, - // The day context, or null. Null rather than an empty object, because - // "no context" and "a context whose every field is empty" are - // different facts and the restore has to keep them apart. - context: moodEntry.context - ? serialiseMoodContext(moodEntry.context, disasterRecovery) - : null, - }; -} - -/** - * How many rows one page of a bulk table carries. - * - * Smaller than the measurement pager's 10 000 on purpose: an intake event - * drags its medication's name along and a mood entry drags its links and its - * day context, so a page of either costs several times what a page of - * measurements costs. The number is a memory budget, not a throughput knob. - */ -const BULK_PAGE_SIZE = 5_000; - -/** - * Every backup measurement, one serialised row at a time. - * - * Keyset-paged through `iterateMeasurementPages`, so the caller decides - * whether to keep the rows (the materialising payload does) or hand each - * straight to a writer and forget it (the streaming writer does). Nothing here - * holds more than one page. - */ -export async function* iterateBackupMeasurements( - prisma: PrismaClient, - userId: string, - disasterRecovery: boolean, -): AsyncGenerator, void, void> { - const pages = iterateMeasurementPages( - prisma, - disasterRecovery ? { userId } : { userId, deletedAt: null }, - MEASUREMENT_BACKUP_SELECT, - ); - for await (const page of pages) { - for (const measurement of page) { - yield serialiseMeasurement(measurement, disasterRecovery); - } - } -} - -/** - * Every backup intake event, one serialised row at a time. - * - * The sort is TOTAL — `scheduledFor desc` then `id desc` — where the - * materialising read used to order by the timestamp alone. A keyset cursor - * needs a total order to page correctly, and a partial one also let two - * exports of the same account emit two different files whenever two doses - * shared a scheduled minute, which every multi-dose day has. - */ -export async function* iterateBackupIntakeEvents( - prisma: PrismaClient, - userId: string, - disasterRecovery: boolean, -): AsyncGenerator, void, void> { - let cursorId: string | null = null; - for (;;) { - const page: BackupIntakeEvent[] = - await prisma.medicationIntakeEvent.findMany({ - where: disasterRecovery ? { userId } : { userId, deletedAt: null }, - include: { medication: { select: { name: true } } }, - orderBy: [{ scheduledFor: "desc" }, { id: "desc" }], - take: BULK_PAGE_SIZE, - ...(cursorId !== null ? { cursor: { id: cursorId }, skip: 1 } : {}), - }); - if (page.length === 0) return; - for (const e of page) yield serialiseIntakeEvent(e, disasterRecovery); - if (page.length < BULK_PAGE_SIZE) return; - cursorId = page[page.length - 1]!.id; - } -} - -/** - * Every backup mood entry, one serialised row at a time. - * - * Same total-order argument as the intake events above: `moodLoggedAt desc` - * alone is not a cursor, and two entries logged in the same millisecond are - * exactly what a bulk import produces. - */ -export async function* iterateBackupMoodEntries( - prisma: PrismaClient, - userId: string, - disasterRecovery: boolean, -): AsyncGenerator, void, void> { - let cursorId: string | null = null; - for (;;) { - const page: BackupMoodEntry[] = await prisma.moodEntry.findMany({ - where: disasterRecovery ? { userId } : { userId, deletedAt: null }, - orderBy: [{ moodLoggedAt: "desc" }, { id: "desc" }], - take: BULK_PAGE_SIZE, - ...(cursorId !== null ? { cursor: { id: cursorId }, skip: 1 } : {}), - select: MOOD_ENTRY_QUERY_SELECT, - }); - if (page.length === 0) return; - for (const moodEntry of page) { - yield serialiseMoodEntry(moodEntry, disasterRecovery); - } - if (page.length < BULK_PAGE_SIZE) return; - cursorId = page[page.length - 1]!.id; - } -} - -/** Drain an async row source into an array. The materialising arm's only use. */ -async function collect(rows: AsyncIterable): Promise { - const out: T[] = []; - for await (const row of rows) out.push(row); - return out; -} - -/** - * A bulk table the payload declares but has NOT read. - * - * `buildFullBackupPayload({ deferBulk: true })` puts one of these where the - * row array would sit. The streaming writer recognises it, opens the JSON - * array itself and pulls rows through page by page, so a table with half a - * million rows never exists as an array at all. Every other caller leaves - * `deferBulk` unset and gets the arrays, unchanged. - * - * The marker key is a symbol so it cannot collide with a payload field and so - * the deferred payload can never be handed to `JSON.stringify` by accident and - * come back looking merely empty — `streamFullBackupJson` is the only reader. - */ -const DEFERRED_ROWS = Symbol("healthlog.backup.deferredRows"); - -export interface DeferredRows { - readonly [DEFERRED_ROWS]: true; - /** - * Open the row source. Called exactly once, by the streaming writer. A row - * is a plain object, or a `StreamedField` (`streamed-field.ts`) for a row - * with one field too large to hold as a string. - */ - rows(): AsyncIterable; -} - -function deferredRows(rows: () => AsyncIterable): DeferredRows { - return { [DEFERRED_ROWS]: true, rows }; -} - -/** True when a payload value is a deferred bulk table rather than an array. */ -export function isDeferredRows(value: unknown): value is DeferredRows { - return ( - typeof value === "object" && - value !== null && - (value as Record)[DEFERRED_ROWS] === true - ); -} /** * Build the canonical full-backup payload for `userId`. Portable exports omit * tombstones and document ciphertext; disaster-recovery payloads preserve * both so weekly and off-host snapshots share one restorable wire format. + * + * Every domain reads its own slice (`backup-export-sections.ts`), all of them + * at once; the slices are then laid out in the file's key order. A key two + * slices write is merged where it first appears (`manifest`). */ export async function buildFullBackupPayload( prisma: PrismaClient, @@ -720,801 +124,40 @@ export async function buildFullBackupPayload( options: FullBackupOptions = {}, ): Promise { const disasterRecovery = options.purpose === "disaster-recovery"; - const deferBulk = options.deferBulk === true; - const [ - appSettings, - measurements, - medications, - intakeEvents, - moodEntries, - customMoodTags, - customMoodTagCategories, - hiddenMoodTags, - cycle, - records, - profile, - intradayProfiles, - healthScoreRecords, - visits, - vaccinations, - reminders, - coach, - coachMemory, - sensitive, - documentFiling, - awards, - environment, - onboardingRecord, - ecg, - symptoms, - lifeEvents, - relationLinks, - nutrientDays, - accountSettings, - ] = await Promise.all([ - disasterRecovery - ? prisma.appSettings.findUnique({ where: { id: "singleton" } }) - : Promise.resolve(null), - deferBulk - ? null - : collect(iterateBackupMeasurements(prisma, userId, disasterRecovery)), - prisma.medication.findMany({ - where: { userId }, - // Side effects ride INSIDE their medication, exactly as the schedules - // beside them do. The alternative — a top-level array carrying - // `medicationId` — would have to survive a restore that mints new - // medication ids for a portable file, which is the id-remap the intake - // events already have to work around by drug name. A nested create has - // no id to remap: Prisma binds the child to whatever id the parent row - // actually got. - include: { - schedules: true, - sideEffects: { orderBy: { occurredAt: "desc" } }, - pauseEras: { orderBy: { pausedAt: "asc" } }, - courses: { orderBy: { startsOn: "asc" } }, - doseChanges: { orderBy: { effectiveFrom: "asc" } }, - inventoryItems: { orderBy: { createdAt: "asc" } }, - inventoryEvents: { orderBy: { occurredAt: "asc" } }, - phaseConfig: true, - // v1.43 — the composition per unit; ordered so two exports of the - // same account write it identically. - ingredients: { orderBy: { nutrient: "asc" } }, - // The archived schedule eras. Ordered TOTALLY, not just by the column - // that reads like the sort key: the position in this list is the - // identifier the supersede pointer travels as, so two eras sharing a - // `validFrom` must not be free to swap places between two exports of - // the same account. - scheduleRevisions: { - orderBy: [ - { validFrom: "asc" }, - { createdAt: "asc" }, - { id: "asc" }, - ] as const, - }, - // What the drug was supposed to move. The biomarker is read by NAME - // rather than by id, exactly as the lab results beside it are: a - // portable restore mints a fresh biomarker id, and `(userId, name)` is - // the natural key the catalogue is already unique on. - efficacyTargets: { - orderBy: { createdAt: "asc" }, - include: { biomarker: { select: { name: true } } }, - }, - }, - }), - deferBulk - ? null - : collect(iterateBackupIntakeEvents(prisma, userId, disasterRecovery)), - deferBulk - ? null - : collect(iterateBackupMoodEntries(prisma, userId, disasterRecovery)), - // The account's OWN mood tags. The seeded catalogue is reference data every - // instance has; a tag the user made lives only here. Without it the restore - // cannot resolve the entry's factor keys — and unlike the cycle path, which - // used to drop them silently, the mood path throws, so an account with one - // custom rated tag could not be restored at all. - prisma.moodTag.findMany({ - where: { userId }, - orderBy: { key: "asc" }, - }), - // The account's OWN categories. Seeded ones (`userId: null`) belong to the - // instance and are deliberately not carried — but a custom tag's - // `categoryId` is a real foreign key, so a category the account created - // itself has to travel or the tag above cannot be written at all. Before - // this section existed, that was not a gap in the restore: it was a - // constraint violation that took the WHOLE file down with it, so an - // account that had ever grouped its own mood factors could not be - // restored. - prisma.moodTagCategory.findMany({ - where: { userId }, - orderBy: { sortOrder: "asc" }, - }), - // Which tags the account chose to hide. Carried by KEY rather than id: - // what people hide is almost always a SEEDED tag, whose id differs on - // every instance, so an id would resolve to nothing on the way back. - prisma.moodTagHidden.findMany({ - where: { userId }, - select: { moodTag: { select: { key: true } }, createdAt: true }, - orderBy: { createdAt: "asc" }, - }), - buildCycleBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - buildRecordsBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - deferDocuments: deferBulk, - }), - // Durable self-context, user-defined metrics, and persisted pattern - // decisions. These account-owned rows cannot be reconstructed from an - // integration after restore. - buildProfileBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The hourly shape of a cumulative day. Not derived, despite looking it: - // the drain folds it out of the per-sample rows in the same transaction - // that deletes them, so past the grace window nothing can rebuild it. - buildIntradayProfileBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The score as it was SHOWN on each local day. Reads like a derived tier - // and is not one: today's number always comes from today's rows, so once - // the readings behind a past day have moved, nothing can recompute what - // that day said. The row is the only copy. - buildHealthScoreBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // Visits, the address book behind them, and the three link tables. Both - // ends live in `src/lib/export/visits-backup.ts` beside each other, so a - // grep of the restore route alone will not find them. - buildVisitsBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The immunization log and the pages it was transcribed from. Both ends - // live in `src/lib/export/vaccinations-backup.ts` beside each other, the - // same arrangement as the visits above and for the same reason. - buildVaccinationsBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The Vorsorge reminders and their completion ledger (v1.37.20, #223 / - // iOS #68). Both ends live in `src/lib/export/reminders-backup.ts` beside - // each other, the same arrangement as the two sections above and for the - // same reason. - buildRemindersBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The Coach's conversations, turns and document links. Both ends live in - // `src/lib/export/coach-backup.ts` beside each other, the same arrangement - // as the three sections above and for the same reason. - buildCoachBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // What the Coach keeps BETWEEN threads. Runs beside the transcript rather - // than inside it because the two answer different questions, and because a - // restore has to write them in that order. - buildCoachMemoryBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The screener history and the consent record. Both ride in a - // disaster-recovery file only, for two different reasons that are written - // out in `src/lib/export/sensitive-backup.ts` rather than summarised here. - buildSensitiveBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // Which conditions a document was filed against, and what the extraction - // pass read out of it. Both ends live in - // `src/lib/export/document-filing-backup.ts` beside each other, the same - // arrangement as the sections above and for the same reason. The documents - // themselves ride in `records-backup.ts`; these two say what was done with - // them, which a restored vault otherwise loses in silence. - buildDocumentFilingBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The bests and the badges. Both ends live in - // `src/lib/export/awards-backup.ts`, the same arrangement as the sections - // above. No `purpose`: neither model tombstones and neither holds - // ciphertext, so both purposes carry the same bytes. - buildAwardsBackupSection(prisma, userId), - // The per-day readings and the location periods that explain them, which - // is why one section carries both. Both ends live in - // `src/lib/export/environment-backup.ts`. Since v1.42 the locations are - // sealed, so the purpose decides whether they ride opened or sealed. - buildEnvironmentBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The needs-based setup answers. One small row, and the only copy of what - // the person told the flow. Both ends live in - // `src/lib/export/onboarding-backup.ts` beside each other. - buildOnboardingBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The ECG strips, their rhythm verdicts and their traces. Both ends live - // in `src/lib/export/ecg-backup.ts` beside each other, the same - // arrangement as the sections above and for the same reason. - buildEcgBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - defer: deferBulk, - }), - // The person's own symptoms with their occurrences nested inside. Both - // ends live in `src/lib/export/symptoms-backup.ts`. - buildSymptomsBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The person's life events (v1.42). Both ends live in - // `src/lib/export/life-events-backup.ts`. - buildLifeEventsBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // The five relation links that close the relation matrix (v1.43). Both - // ends live in `src/lib/export/relation-links-backup.ts`. - buildRelationLinksBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - // Nutrient day totals were absent from every export path, which - // contradicted the schema's own reason for denormalising the unit column - // ("rows stay self-describing in exports even if the catalog ever drifts"). - // `source` is part of the composite PK, so it has to ride along or a - // restore cannot tell a manual water entry from a synced day total. The - // table carries no tombstone, so both purposes read the same shape. - // `externalSourceVersion` is the source's own version marker, stored so - // the sync can replay it back unchanged; `createdAt` is when the day - // first arrived. Neither can be rebuilt after a restore. - prisma.nutrientIntakeDay.findMany({ - where: { userId }, - select: { - day: true, - nutrient: true, - amount: true, - unit: true, - source: true, - externalSourceVersion: true, - createdAt: true, - }, - orderBy: [{ day: "desc" }, { nutrient: "asc" }], - }), - // The account's own settings: units, language, modules, thresholds, the - // AI choices without the keys. Which columns those are is decided in - // `USER_COLUMN_BACKUP_CLASS`; both ends live in - // `src/lib/export/account-settings-backup.ts`. - buildAccountSettingsBackupSection(prisma, userId, { - purpose: disasterRecovery ? "disaster-recovery" : "portable-export", - }), - ]); - - // Pinned to the section interfaces so the spreads below cannot quietly widen - // (or narrow) what reaches the wire. - const cycleSection: CycleBackupSection = cycle; - const recordsSection: RecordsBackupSection = records; - const profileSection: ProfileBackupSection = profile; - const intradaySection: IntradayProfileBackupSection = intradayProfiles; - const healthScoreSection: HealthScoreBackupSection = healthScoreRecords; - const onboardingSection: OnboardingBackupSection = onboardingRecord; - const visitsSection: VisitsBackupSection = visits; - const vaccinationsSection: VaccinationsBackupSection = vaccinations; - const remindersSection: RemindersBackupSection = reminders; - const coachSection: CoachBackupSection = coach; - const coachMemorySection: CoachMemoryBackupSection = coachMemory; - const sensitiveSection: SensitiveBackupSection = { - mentalHealthAssessments: sensitive.mentalHealthAssessments, - consentReceipts: sensitive.consentReceipts, - }; - const sensitiveManifest: SensitiveBackupManifest = sensitive.manifest; - const documentFilingSection: DocumentFilingBackupSection = documentFiling; - const awardsSection: AwardsBackupSection = awards; - const environmentSection: EnvironmentBackupSection = environment; - const ecgSection: EcgBackupSection = ecg; - const symptomsSection: SymptomsBackupSection = symptoms; - const lifeEventsSection: LifeEventsBackupSection = lifeEvents; - const relationLinksSection: RelationLinksBackupSection = relationLinks; - const accountSettingsSection: AccountSettingsBackupSection = accountSettings; - - // The three unbounded tables, either read into arrays or left as markers - // the streaming writer will pull through. One place decides, so the payload - // literal below reads the same in both modes. - const bulk = { - measurements: deferBulk - ? deferredRows(() => - iterateBackupMeasurements(prisma, userId, disasterRecovery), - ) - : measurements!, - intakeEvents: deferBulk - ? deferredRows(() => - iterateBackupIntakeEvents(prisma, userId, disasterRecovery), - ) - : intakeEvents!, - moodEntries: deferBulk - ? deferredRows(() => - iterateBackupMoodEntries(prisma, userId, disasterRecovery), - ) - : moodEntries!, - }; - // The two sections whose rows carry stored bytes: the documents (up to a - // hundred megabytes each) and the ECG traces. Deferred, they keep the - // position their section gives them in the payload (a key that already - // exists keeps its place when it is assigned again) and are read a document - // or a page of strips at a time (#1239). - const purpose = disasterRecovery ? "disaster-recovery" : "portable-export"; - const deferredBytes = deferBulk - ? { - documents: deferredRows(() => - iterateBackupDocuments(prisma, userId, { purpose }), - ), - ecgRecordings: deferredRows(() => - iterateBackupEcgRecordings(prisma, userId, { purpose }), - ), - } - : {}; - - // The clinical category lives in its own table rather than on the - // medication row and is read through the one helper that normalises it. - // Before v1.39.4 no backup carried it and every restored medication came - // back as "Other". - const categoryByMedication = await getMedicationCategories( - medications.map((m) => m.id), + const ctx: BackupExportContext = { prisma, - ); - // v1.40 (#1041) — the account's own categories, restored before the - // medications that name them by key. Labels travel decrypted so a file - // restores on an instance with a different key; every one is carried, - // hidden ones included, because a medication can still be filed under it. - const customMedicationCategories = - await readMedicationCategoryLabelsForBackup(userId, prisma); - - // Where each archived era sits in its OWN drug's ordered list. Built once - // here rather than per row, and scoped per medication because the supersede - // pointer never crosses a drug — every route that writes it scopes the - // update to the medication it belongs to. - const revisionIndexByMedication = new Map( - medications.map((m) => [ - m.id, - new Map( - m.scheduleRevisions.map((revision, index) => [revision.id, index]), - ), - ]), + userId, + disasterRecovery, + purpose: disasterRecovery ? "disaster-recovery" : "portable-export", + deferBulk: options.deferBulk === true, + }; + const parts = await Promise.all( + BACKUP_EXPORT_SECTIONS.map((section) => section.build(ctx)), ); - const payload = { + const payload: Record = { schemaVersion: BACKUP_SCHEMA_VERSION, exportedAt: (options.exportedAt ?? new Date()).toISOString(), userId, - appSettings: - disasterRecovery && appSettings - ? { - ...appSettings, - adminCodexTokenExpiresAt: - appSettings.adminCodexTokenExpiresAt?.toISOString() ?? null, - adminCodexConnectedAt: - appSettings.adminCodexConnectedAt?.toISOString() ?? null, - documentQuotaBytes: appSettings.documentQuotaBytes.toString(), - } - : null, - ...accountSettingsSection, - measurements: bulk.measurements, - customMedicationCategories, - medications: medications.map((m) => ({ - ...(disasterRecovery - ? { - id: m.id, - treatmentClass: m.treatmentClass, - dosesPerUnit: m.dosesPerUnit, - unitsPerDose: m.unitsPerDose.toString(), - notificationsEnabled: m.notificationsEnabled, - pausedAt: m.pausedAt?.toISOString() ?? null, - snoozedUntil: m.snoozedUntil?.toISOString() ?? null, - startsOn: m.startsOn?.toISOString() ?? null, - endsOn: m.endsOn?.toISOString() ?? null, - oneShot: m.oneShot, - asNeeded: m.asNeeded, - deliveryForm: m.deliveryForm, - trackInjectionSites: m.trackInjectionSites, - allowedInjectionSites: m.allowedInjectionSites, - liveActivityEnabled: m.liveActivityEnabled, - criticalAlarmEnabled: m.criticalAlarmEnabled, - atcCode: m.atcCode, - rxNormCode: m.rxNormCode, - lowStockNotifiedAt: m.lowStockNotifiedAt?.toISOString() ?? null, - lowStockNotifiedThresholdDays: m.lowStockNotifiedThresholdDays, - reorderLeadDays: m.reorderLeadDays, - externalSource: m.externalSource, - externalId: m.externalId, - updatedAt: m.updatedAt.toISOString(), - } - : // No id: a portable restore mints a fresh one. The relation links - // still name this drug, so the file says which row they mean. - { ref: m.id }), - name: m.name, - dose: m.dose, - active: m.active, - // v1.39.1 (#1033) — the person's decision to keep this medication as a - // record only. Both purposes carry it: a restore that dropped it would - // start reminding them about a medication they chose not to track. - trackIntake: m.trackIntake, - // v1.42.2 — the person's choice to have doses recorded for them, and - // the floor before which none is. Both purposes: a restore without - // them would start reminding again, or fill history it never owned. - autoMarkTaken: m.autoMarkTaken, - autoMarkTakenSince: m.autoMarkTakenSince?.toISOString() ?? null, - // Both purposes: the category is something the person chose. - category: categoryByMedication[m.id] ?? "OTHER", - // Both purposes carry the creation instant: it is the floor of the - // medication's expected slots, so a restore stamped with the restore - // time would read the whole restored history as predating the - // medication and drop its misses from every rate (#1028). - createdAt: m.createdAt.toISOString(), - schedules: m.schedules.map((s) => ({ - ...(disasterRecovery - ? { - id: s.id, - daysOfWeek: s.daysOfWeek, - timesOfDay: s.timesOfDay, - reminderGraceMinutes: s.reminderGraceMinutes, - rrule: s.rrule, - rollingIntervalDays: s.rollingIntervalDays, - scheduleType: s.scheduleType, - cyclicOnWeeks: s.cyclicOnWeeks, - cyclicOffWeeks: s.cyclicOffWeeks, - doseWindows: s.doseWindows, - // #219 — per-schedule units per dose. Decimal → string like the - // medication-level column; NULL (inherit) stays NULL. - unitsPerDose: - s.unitsPerDose == null ? null : s.unitsPerDose.toString(), - } - : {}), - windowStart: s.windowStart, - windowEnd: s.windowEnd, - label: s.label, - dose: s.dose, - })), - // What the person recorded against the drug, and the reason a drug may - // have been stopped. `notes` is the same dual-column arrangement - // `Measurement` has and gets the same treatment: a portable export - // carries the DECRYPTED note and no ciphertext, a disaster-recovery - // payload carries the ciphertext verbatim plus whatever legacy plaintext - // the row still holds, and the restore re-encrypts the plaintext case. - sideEffects: m.sideEffects.map((s) => ({ - ...(disasterRecovery - ? { - id: s.id, - notes: s.notes, - notesEncrypted: s.notesEncrypted - ? Buffer.from(s.notesEncrypted).toString("base64") - : null, - createdAt: s.createdAt.toISOString(), - } - : { notes: readNote(s.notesEncrypted, s.notes) }), - occurredAt: s.occurredAt.toISOString(), - category: s.category, - entry: s.entry, - severity: s.severity, - })), - // Nested for the same reason the two above are: a pause era is bound to - // its medication by id, and a nested create binds it to whatever id the - // parent actually got. - // - // These spans are what separates "stopped taking it" from "was told to - // stop taking it". Without them the compliance recomputation counts a - // deliberate pause as missed doses, so a restored account reads as - // non-adherent for a stretch during which it did exactly what it was - // told. Nothing else in the payload can reconstruct them: an era is a - // decision, not an observation, and no intake row records its absence. - // - // `resumedAt` stays nullable end to end — an open era means the - // medication is still paused, and readers run it to `now`. - pauseEras: m.pauseEras.map((p) => ({ - ...(disasterRecovery ? { id: p.id } : {}), - pausedAt: p.pausedAt.toISOString(), - resumedAt: p.resumedAt ? p.resumedAt.toISOString() : null, - createdAt: p.createdAt.toISOString(), - })), - // v1.40 (#1024) — the courses: "when did I last take this" lives here - // and nowhere else once a second course exists. The medication's own - // window above is the projection of the latest one. - // v1.43 — what one unit contains. Both purposes: it is something the - // person typed off a label, and without it the supplement's doses stop - // counting toward the day's micronutrients. Canonical amount plus the - // amount as entered, so the form shows what was typed. - ingredients: m.ingredients.map((i) => ({ - ...(disasterRecovery ? { id: i.id } : {}), - nutrient: i.nutrient, - amountPerUnit: i.amountPerUnit, - unit: i.unit, - enteredAmount: i.enteredAmount, - enteredUnit: i.enteredUnit, - source: i.source, - createdAt: i.createdAt.toISOString(), - })), - courses: m.courses.map((c) => ({ - startsOn: dayKeyOfDate(c.startsOn), - endsOn: c.endsOn ? dayKeyOfDate(c.endsOn) : null, - note: c.noteEncrypted ? decryptFromBytes(c.noteEncrypted) : null, - createdAt: c.createdAt.toISOString(), - })), - // Titration: when the dose moved, to what, and why. The drug comes - // back at today's dose either way; without these the ramp that led to - // it is gone, and that ramp is what a prescriber asks about. - // - // The note follows the side-effect contract beside it: a portable - // export carries the DECRYPTED note and no ciphertext, a - // disaster-recovery payload carries the ciphertext verbatim plus - // whatever legacy plaintext the row still holds. - doseChanges: m.doseChanges.map((d) => ({ - ...(disasterRecovery - ? { - id: d.id, - note: d.note, - noteEncrypted: d.noteEncrypted - ? Buffer.from(d.noteEncrypted).toString("base64") - : null, - } - : { note: readNote(d.noteEncrypted, d.note) }), - effectiveFrom: d.effectiveFrom.toISOString(), - doseValue: d.doseValue, - doseUnit: d.doseUnit, - createdAt: d.createdAt.toISOString(), - })), - // The packs on the shelf, and the ledger behind the number on them. - // - // `unitsRemaining` is carried VERBATIM rather than recomputed from the - // events, for the same reason `nextDueAt` is: it is what the server had - // resolved and what the person was shown. Recomputing on the way back - // would silently correct a count the account may have adjusted by hand, - // and would disagree with every low-stock reminder already sent. - // - // Both Decimal columns cross the wire as strings. A pack of 0.5 mg - // halves is a real prescription, and JSON numbers would round it. - inventoryItems: m.inventoryItems.map((i) => ({ - ...(disasterRecovery - ? { - id: i.id, - notes: i.notes, - notesEncrypted: i.notesEncrypted - ? Buffer.from(i.notesEncrypted).toString("base64") - : null, - } - : { notes: readNote(i.notesEncrypted, i.notes) }), - state: i.state, - containerType: i.containerType, - unitsTotal: i.unitsTotal.toString(), - unitsRemaining: i.unitsRemaining.toString(), - firstUseAt: i.firstUseAt?.toISOString() ?? null, - expiresAt: i.expiresAt?.toISOString() ?? null, - printedExpiry: i.printedExpiry?.toISOString() ?? null, - purchasedAt: i.purchasedAt?.toISOString() ?? null, - manufacturer: i.manufacturer, - doseStrength: i.doseStrength, - createdAt: i.createdAt.toISOString(), - updatedAt: i.updatedAt.toISOString(), - })), - // The four reminder phase thresholds this drug was tuned to. One row or - // none, so it rides as an object rather than a list — and null means the - // person never tuned it, which is a different statement from "the file - // forgot to say". - phaseConfig: m.phaseConfig - ? { - ...(disasterRecovery ? { id: m.phaseConfig.id } : {}), - greenValue: m.phaseConfig.greenValue, - greenMode: m.phaseConfig.greenMode, - yellowValue: m.phaseConfig.yellowValue, - yellowMode: m.phaseConfig.yellowMode, - orangeValue: m.phaseConfig.orangeValue, - orangeMode: m.phaseConfig.orangeMode, - redValue: m.phaseConfig.redValue, - redMode: m.phaseConfig.redMode, - } - : null, - inventoryEvents: m.inventoryEvents.map((e) => ({ - ...(disasterRecovery ? { id: e.id } : {}), - delta: e.delta, - reason: e.reason, - occurredAt: e.occurredAt.toISOString(), - })), - // How the schedule got to where it is. The drug comes back on today's - // plan either way; without these the record of when the dose or the - // timing moved is gone, and the era minter reads every past day against - // the CURRENT schedule instead of the one that was live then — so a - // restored account's own compliance history quietly changes. - // - // `supersededByRevisionId` travels as a POSITION in this list rather - // than as an id. See the restore for the whole argument; the short form - // is that a portable restore mints fresh ids, so an id would point at - // nothing, and the list this array IS is the only stable thing both ends - // agree on. A pointer at a row outside this drug's own list cannot be - // expressed and is written as null — the schema allows it, no code path - // produces it, and inventing an index for it would be worse than saying - // nothing. - scheduleRevisions: m.scheduleRevisions.map((r) => ({ - ...(disasterRecovery ? { id: r.id } : {}), - validFrom: r.validFrom.toISOString(), - validUntil: r.validUntil.toISOString(), - payload: r.payload, - source: r.source, - supersededByIndex: r.supersededByRevisionId - ? (revisionIndexByMedication - .get(m.id) - ?.get(r.supersededByRevisionId) ?? null) - : null, - createdAt: r.createdAt.toISOString(), - })), - // What the drug is FOR, in the person's own words rather than the - // resolver's. A row exists only where they overrode the derived - // ATC / name inference, so it is the one durable statement of intent - // the "Wirkung" view holds — and the drug restores with no statement of - // what it was for without it. - // - // The lab arm travels as a NAME, the way a lab result's does. The id is - // fresh on a portable restore, and `Biomarker` is unique on - // `(userId, name)`, so the name is the only thing that survives the - // trip. It is resolved on the way back against the biomarkers the - // restore wrote. - efficacyTargets: m.efficacyTargets.map((t) => ({ - ...(disasterRecovery - ? { - id: t.id, - createdAt: t.createdAt.toISOString(), - updatedAt: t.updatedAt.toISOString(), - } - : {}), - measurementType: t.measurementType, - biomarkerName: t.biomarker?.name ?? null, - primary: t.primary, - })), - })), - intakeEvents: bulk.intakeEvents, - moodEntries: bulk.moodEntries, - customMoodTagCategories: customMoodTagCategories.map((category) => ({ - id: category.id, - key: category.key, - labelKey: category.labelKey, - icon: category.icon, - sortOrder: category.sortOrder, - isActive: category.isActive, - // The person's own words: ciphertext in a disaster-recovery payload, - // readable in a portable one (see `sealed-text.ts`). - ...(disasterRecovery - ? { labelEncrypted: category.labelEncrypted } - : { - label: category.labelEncrypted - ? openSealedForExport( - category.labelEncrypted, - "mood category label", - ) - : null, - }), - })), - hiddenMoodTags: hiddenMoodTags.map((hidden) => ({ - key: hidden.moodTag.key, - createdAt: hidden.createdAt.toISOString(), - })), - customMoodTags: customMoodTags.map((tag) => ({ - ...(disasterRecovery ? { id: tag.id } : {}), - key: tag.key, - labelKey: tag.labelKey, - categoryId: tag.categoryId, - kind: tag.kind, - isActive: tag.isActive, - icon: tag.icon, - sortOrder: tag.sortOrder, - // Ciphertext in a disaster-recovery payload (the same instance's key - // reads it back), readable in a portable one (see `sealed-text.ts`). - ...(disasterRecovery - ? { labelEncrypted: tag.labelEncrypted } - : { - label: tag.labelEncrypted - ? openSealedForExport(tag.labelEncrypted, "mood tag label") - : null, - }), - scaleMin: tag.scaleMin, - scaleMax: tag.scaleMax, - inverse: tag.inverse, - })), - // Both sections are SPREAD, not hand-picked key by key. - // - // Hand-picking is what lost the custom cycle symptoms. `buildCycleBackupSection` - // returned them, the wire schema declared them, `restoreCycleData` read them - // and — since v1.33.1 — THREW on a key it could not resolve. The list of keys - // copied out here was simply never extended, so every backup written carried - // `customSymptoms: []`. An account with one custom symptom did not get the - // silent drop that release removed; it got a restore that failed outright on - // a key nothing had written back, and the fix was inert in production. - // - // Spreading makes the section interface the single declaration of what rides - // the wire: a field added to `CycleBackupSection` / `RecordsBackupSection` - // reaches the payload by construction rather than by someone remembering. - // `manifest` (from the records section) discloses the two deliberate - // exclusions — document binaries, workout GPS/sample series — in the file - // itself, not just in the export UI copy. - ...cycleSection, - ...recordsSection, - ...profileSection, - ...intradaySection, - ...healthScoreSection, - ...onboardingSection, - ...visitsSection, - ...vaccinationsSection, - ...remindersSection, - ...coachSection, - ...coachMemorySection, - ...sensitiveSection, - // Merged rather than added beside it: the records section already owns a - // `manifest` key, and two of them would silently shadow each other - // depending on spread order. - manifest: { ...recordsSection.manifest, ...sensitiveManifest }, - ...documentFilingSection, - ...awardsSection, - ...environmentSection, - ...ecgSection, - ...symptomsSection, - ...lifeEventsSection, - ...relationLinksSection, - ...deferredBytes, - nutrientDays: nutrientDays.map((n) => ({ - day: n.day, - nutrient: n.nutrient, - amount: n.amount, - unit: n.unit, - source: n.source, - externalSourceVersion: n.externalSourceVersion, - createdAt: n.createdAt.toISOString(), - })), }; + for (const part of parts) { + for (const [key, value] of Object.entries(part.section)) { + payload[key] = + key === "manifest" && key in payload + ? { ...(payload.manifest as object), ...(value as object) } + : value; + } + } - return { - payload, - counts: { - measurements: measurements?.length ?? 0, - medications: medications.length, - intakeEvents: intakeEvents?.length ?? 0, - medicationSideEffects: medications.reduce( - (total, m) => total + m.sideEffects.length, - 0, - ), - medicationPauseEras: medications.reduce( - (total, m) => total + m.pauseEras.length, - 0, - ), - medicationDoseChanges: medications.reduce( - (total, m) => total + m.doseChanges.length, - 0, - ), - medicationInventoryItems: medications.reduce( - (total, m) => total + m.inventoryItems.length, - 0, - ), - medicationInventoryEvents: medications.reduce( - (total, m) => total + m.inventoryEvents.length, - 0, - ), - reminderPhaseConfigs: medications.filter((m) => m.phaseConfig).length, - medicationEfficacyTargets: medications.reduce( - (total, m) => total + m.efficacyTargets.length, - 0, - ), - medicationScheduleRevisions: medications.reduce( - (total, m) => total + m.scheduleRevisions.length, - 0, - ), - moodEntries: moodEntries?.length ?? 0, - customMoodTagCategories: customMoodTagCategories.length, - hiddenMoodTags: hiddenMoodTags.length, - cycles: cycle.cycles.length, - cycleDayLogs: cycle.cycleDayLogs.length, - nutrientDays: nutrientDays.length, - ...countRecordsBackupSection(records), - ...countProfileBackupSection(profile), - ...countIntradayProfileBackupSection(intradayProfiles), - ...countHealthScoreBackupSection(healthScoreRecords), - ...countOnboardingBackupSection(onboardingRecord), - ...countVisitsBackupSection(visits), - ...countVaccinationsBackupSection(vaccinations), - ...countRemindersBackupSection(reminders), - ...countCoachBackupSection(coach), - ...countCoachMemoryBackupSection(coachMemory), - ...countSensitiveBackupSection(sensitive), - ...countDocumentFilingBackupSection(documentFiling), - ...countAwardsBackupSection(awards), - ...countEnvironmentBackupSection(environment), - ...countEcgBackupSection(ecg), - ...countSymptomsBackupSection(symptoms), - ...countLifeEventsBackupSection(lifeEvents), - ...countRelationLinksBackupSection(relationLinks), - }, - }; + const partByName = new Map( + BACKUP_EXPORT_SECTIONS.map((section, index) => [ + section.name, + parts[index], + ]), + ); + const counts = Object.assign( + {}, + ...BACKUP_EXPORT_COUNT_ORDER.map((name) => partByName.get(name)!.counts), + ) as FullBackupCounts; + return { payload, counts }; } diff --git a/src/lib/export/illness-backup.ts b/src/lib/export/illness-backup.ts index 31c649900..64486bb95 100644 --- a/src/lib/export/illness-backup.ts +++ b/src/lib/export/illness-backup.ts @@ -2,13 +2,109 @@ * The illness history: episodes, their day logs, and the symptoms each day * recorded. Both backup ends in one file. */ +import { Buffer } from "node:buffer"; + +import type { PrismaClient } from "@/generated/prisma/client"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; +import { + toIllnessEpisodeDTO, + toIllnessDayLogDTO, + dayLogSymptomInclude, + type IllnessEpisodeDTO, + type IllnessDayLogDTO, +} from "@/lib/illness/dto"; import { decodeEncryptedBytes, type RestoreContext, } from "@/lib/export/restore-context"; import { recordUnknownKeys } from "@/lib/export/restore-skips"; +// ── Export ─────────────────────────────────────────────────────────────────── + +type CanonicalIllnessDayLog = IllnessDayLogDTO & { + noteEncrypted?: string | null; + tz?: string | null; + createdAt?: string; + deletedAt?: string | null; +}; + +export type CanonicalIllnessEpisode = IllnessEpisodeDTO & { + noteEncrypted?: string | null; + /** The body site's ciphertext, disaster-recovery purpose only (v1.39.2). */ + bodySiteEncrypted?: string | null; + deletedAt?: string | null; + dayLogs: CanonicalIllnessDayLog[]; +}; + +/** The illness slice of the records section: episodes with their day logs. */ +export async function buildIllnessBackupSlice( + prisma: Pick, + userId: string, + disasterRecovery: boolean, +): Promise<{ illnessEpisodes: CanonicalIllnessEpisode[] }> { + const episodeRows = await prisma.illnessEpisode.findMany({ + where: disasterRecovery ? { userId } : { userId, deletedAt: null }, + orderBy: { onsetAt: "desc" }, + include: { + dayLogs: { + ...(disasterRecovery ? {} : { where: { deletedAt: null } }), + orderBy: { date: "asc" }, + include: dayLogSymptomInclude, + }, + }, + }); + + const illnessEpisodes: CanonicalIllnessEpisode[] = disasterRecovery + ? episodeRows.map((row) => ({ + id: row.id, + label: row.label, + type: row.type, + lifecycle: row.lifecycle, + onsetAt: row.onsetAt.toISOString(), + resolvedAt: row.resolvedAt?.toISOString() ?? null, + parentConditionId: row.parentConditionId, + note: null, + noteEncrypted: row.noteEncrypted + ? Buffer.from(row.noteEncrypted).toString("base64") + : null, + // Ciphertext verbatim, like the note: the disaster-recovery file never + // carries the site in the clear. + bodySite: null, + bodySiteEncrypted: row.bodySiteEncrypted + ? Buffer.from(row.bodySiteEncrypted).toString("base64") + : null, + laterality: row.laterality, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + deletedAt: row.deletedAt?.toISOString() ?? null, + dayLogs: row.dayLogs.map((dayLog) => ({ + id: dayLog.id, + episodeId: dayLog.episodeId, + date: dayLog.date, + functionalImpact: dayLog.functionalImpact, + feverC: dayLog.feverC, + note: null, + noteEncrypted: dayLog.noteEncrypted + ? Buffer.from(dayLog.noteEncrypted).toString("base64") + : null, + tz: dayLog.tz, + createdAt: dayLog.createdAt.toISOString(), + updatedAt: dayLog.updatedAt.toISOString(), + deletedAt: dayLog.deletedAt?.toISOString() ?? null, + symptoms: dayLog.symptomLinks.map((link) => ({ + key: link.symptom.key, + severity: link.severity, + })), + })), + })) + : episodeRows.map((row) => ({ + ...toIllnessEpisodeDTO(row), + dayLogs: row.dayLogs.map(toIllnessDayLogDTO), + })); + + return { illnessEpisodes }; +} + // ── Restore ────────────────────────────────────────────────────────────────── export async function clearIllness(ctx: RestoreContext): Promise { diff --git a/src/lib/export/labs-backup.ts b/src/lib/export/labs-backup.ts index 10b4adfa9..f2e5291d5 100644 --- a/src/lib/export/labs-backup.ts +++ b/src/lib/export/labs-backup.ts @@ -6,15 +6,271 @@ * restore mints fresh marker ids, and `(userId, name)` is what the catalogue * is unique on, so the name is what survives the trip. */ -import { encryptContextToBytes } from "@/lib/labs/biomarker-store"; -import { encryptNoteToBytes } from "@/lib/labs/store"; +import { Buffer } from "node:buffer"; + +import type { PrismaClient } from "@/generated/prisma/client"; +import { + decryptContextFromBytes, + encryptContextToBytes, +} from "@/lib/labs/biomarker-store"; +import { decryptNoteFromBytes, encryptNoteToBytes } from "@/lib/labs/store"; import { foldForMatch } from "@/lib/i18n/fold-for-match"; -import { annotate } from "@/lib/logging/context"; +import { annotate, getEvent } from "@/lib/logging/context"; +import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker"; import { decodeEncryptedBytes, type RestoreContext, } from "@/lib/export/restore-context"; +// ── Export ─────────────────────────────────────────────────────────────────── + +/** + * Decrypt an encrypted free-text column for a portable export, or say plainly + * that this one could not be read. + * + * A fail-soft decrypt answers `null` on a wrong or dropped key. In a list view + * that is right — one bad row must not take the page down. In an export it is + * not: `null` is exactly what a column that was never written looks like, so + * the restore importer writes nothing and a note the person did write + * disappears without anyone being told. The Coach transcript and the doctor + * report both refuse that trade, one with a placeholder sentence and one with + * a rendered flag; this follows the same rule. + * + * The warning the soft helpers emitted is kept, so a systemic key gap still + * shows up in the wide event rather than only in the file. + */ +function readForExport( + buf: Uint8Array | null, + label: string, + decode: (bytes: Uint8Array) => string, +): string | null { + if (!buf || buf.byteLength === 0) return null; + try { + return decode(buf); + } catch (err) { + getEvent()?.addWarning( + `${label} decrypt failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return UNREADABLE_EXPORT_MARKER; + } +} + +export interface LabResultBackupEntry { + /** Present in canonical DR payloads so rows keep a stable identity. */ + id?: string; + /** The source row's id in a portable payload, for the relation links. */ + ref?: string; + panel: string | null; + analyte: string; + value: number | null; + valueText: string | null; + unit: string; + referenceLow: number | null; + referenceHigh: number | null; + /** The window the reading's own source report printed, verbatim + derived. */ + sourceReferenceLow: number | null; + sourceReferenceHigh: number | null; + sourceReferenceText: string | null; + /** + * The value and unit the source report printed, kept when a write + * converted the reading into its marker's unit. + */ + sourceValue?: number | null; + sourceUnit?: string | null; + takenAt: string; + source: string; + /** Human-readable cross-reference into `biomarkers` below, not an id. */ + /** Stable canonical FK; legacy portable payloads use biomarkerName. */ + biomarkerId?: string | null; + biomarkerName: string | null; + note: string | null; + /** Base64 ciphertext in canonical DR payloads. */ + noteEncrypted?: string | null; + createdAt?: string; + updatedAt?: string; + deletedAt?: string | null; +} + +export interface BiomarkerBackupEntry { + /** Present in canonical DR payloads; name remains the natural key. */ + id?: string; + name: string; + unit: string; + lowerBound: number | null; + upperBound: number | null; + panel: string | null; + hidden: boolean; + context: string | null; + /** The catalogue key the marker's name and unit resolve to. */ + analyteKey?: string | null; + /** + * The other spellings the marker answers to, with where + * each came from. Absent on files written before them. + */ + aliases?: BiomarkerAliasBackupEntry[]; + createdAt?: string; + updatedAt?: string; +} + +export interface BiomarkerAliasBackupEntry { + /** Present in canonical DR payloads. */ + id?: string; + name: string; + source: "MERGE" | "IMPORT_CHOICE" | "MANUAL"; + createdAt?: string; +} + +/** + * One marker folded into another. The target travels by id + * in a disaster-recovery file and by name in both, the way a lab result's + * marker does; the source is gone, so its id and name are the record. + */ +export interface BiomarkerMergeBackupEntry { + id?: string; + targetBiomarkerId?: string; + targetName: string; + sourceBiomarkerId: string; + sourceName: string; + readingCount: number; + mergedAt: string; +} + +/** The lab slice of the records section: results, markers, merge ledger. */ +export async function buildLabsBackupSlice( + prisma: Pick< + PrismaClient, + "labResult" | "biomarker" | "biomarkerAlias" | "biomarkerMerge" + >, + userId: string, + disasterRecovery: boolean, +): Promise<{ + labResults: LabResultBackupEntry[]; + biomarkers: BiomarkerBackupEntry[]; + biomarkerMerges: BiomarkerMergeBackupEntry[]; +}> { + const [labResultRows, biomarkerRows, biomarkerAliasRows, biomarkerMergeRows] = + await Promise.all([ + prisma.labResult.findMany({ + where: disasterRecovery ? { userId } : { userId, deletedAt: null }, + orderBy: { takenAt: "desc" }, + include: { biomarker: { select: { name: true } } }, + }), + prisma.biomarker.findMany({ + where: { userId }, + orderBy: { name: "asc" }, + }), + prisma.biomarkerAlias.findMany({ + where: { userId }, + orderBy: [{ biomarkerId: "asc" }, { nameFolded: "asc" }], + select: { + id: true, + biomarkerId: true, + name: true, + source: true, + createdAt: true, + }, + }), + prisma.biomarkerMerge.findMany({ + where: { userId }, + orderBy: { mergedAt: "asc" }, + include: { target: { select: { name: true } } }, + }), + ]); + + const labResults: LabResultBackupEntry[] = labResultRows.map((r) => ({ + ...(disasterRecovery + ? { + id: r.id, + biomarkerId: r.biomarkerId, + noteEncrypted: r.noteEncrypted + ? Buffer.from(r.noteEncrypted).toString("base64") + : null, + createdAt: r.createdAt.toISOString(), + updatedAt: r.updatedAt.toISOString(), + deletedAt: r.deletedAt?.toISOString() ?? null, + note: null, + } + : { + // No id: a portable restore mints a fresh one. The relation links + // still name this row, so the file says which row they mean. + ref: r.id, + note: readForExport( + r.noteEncrypted, + "lab note", + decryptNoteFromBytes, + ), + }), + panel: r.panel, + analyte: r.analyte, + value: r.value, + valueText: r.valueText, + unit: r.unit, + referenceLow: r.referenceLow, + referenceHigh: r.referenceHigh, + sourceReferenceLow: r.sourceReferenceLow, + sourceReferenceHigh: r.sourceReferenceHigh, + sourceReferenceText: r.sourceReferenceText, + sourceValue: r.sourceValue, + sourceUnit: r.sourceUnit, + takenAt: r.takenAt.toISOString(), + source: r.source, + biomarkerName: r.biomarker?.name ?? null, + })); + + const aliasesByMarker = new Map(); + for (const alias of biomarkerAliasRows) { + const list = aliasesByMarker.get(alias.biomarkerId) ?? []; + list.push({ + name: alias.name, + source: alias.source as BiomarkerAliasBackupEntry["source"], + ...(disasterRecovery + ? { id: alias.id, createdAt: alias.createdAt.toISOString() } + : {}), + }); + aliasesByMarker.set(alias.biomarkerId, list); + } + + const biomarkerMerges: BiomarkerMergeBackupEntry[] = biomarkerMergeRows.map( + (m) => ({ + ...(disasterRecovery + ? { id: m.id, targetBiomarkerId: m.targetBiomarkerId } + : {}), + targetName: m.target.name, + sourceBiomarkerId: m.sourceBiomarkerId, + sourceName: m.sourceName, + readingCount: m.readingCount, + mergedAt: m.mergedAt.toISOString(), + }), + ); + + const biomarkers: BiomarkerBackupEntry[] = biomarkerRows.map((b) => ({ + ...(disasterRecovery + ? { + id: b.id, + createdAt: b.createdAt.toISOString(), + updatedAt: b.updatedAt.toISOString(), + } + : {}), + name: b.name, + unit: b.unit, + lowerBound: b.lowerBound, + upperBound: b.upperBound, + panel: b.panel, + hidden: b.hidden, + context: readForExport( + b.contextEncrypted, + "biomarker context", + decryptContextFromBytes, + ), + analyteKey: b.analyteKey, + aliases: aliasesByMarker.get(b.id) ?? [], + })); + + return { labResults, biomarkers, biomarkerMerges }; +} + // ── Restore ────────────────────────────────────────────────────────────────── export async function clearLabResults(ctx: RestoreContext): Promise { diff --git a/src/lib/export/measurements-backup.ts b/src/lib/export/measurements-backup.ts index 5a8ebdb4d..63b17597d 100644 --- a/src/lib/export/measurements-backup.ts +++ b/src/lib/export/measurements-backup.ts @@ -1,429 +1,153 @@ /** - * The measurements, with both backup ends in one file. - * - * The restore reads the readings twice. The first read (`readStreamedBackup`, - * before anything is deleted) checks each one and notes what the second read - * has to know about the file as a whole: which `stats:` aggregates are live, - * and which nights the retired heart-rate variability filing shares with an - * RMSSD reading (`createMeasurementRestorePrepass`). The second read, inside - * the transaction, writes them a batch at a time (`restoreMeasurements`), so - * the restore never holds more than one batch. + * The measurements domain's export half: the select every carried column is + * named in, the two wire shapes, and the paged read. The restore half lives + * in `measurements-restore.ts`. */ -import type { Prisma } from "@/generated/prisma/client"; -import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client"; -import { encryptNote } from "@/lib/crypto/note-cipher"; -import { - isLegacyRmssdRow, - LEGACY_RMSSD_SOURCES, -} from "@/lib/export/legacy-rmssd"; -import { - insertMeasurementRows, - type MeasurementInsertRow, -} from "@/lib/export/measurement-bulk-insert"; -import { - decodeEncryptedBytes, - type RestoreContext, -} from "@/lib/export/restore-context"; -import { resolveUserTimezone } from "@/lib/measurements/consolidation-base"; -import { - FOLDED_TYPES, - compactionTombstoneCoverId, -} from "@/lib/measurements/folded-window"; -import type { BackupMeasurement } from "@/lib/validations/backup"; +import { Buffer } from "node:buffer"; -// ── Restore ────────────────────────────────────────────────────────────────── - -/** Both unique identities of a row, as it would be filed under `HRV_RMSSD`. */ -function rmssdTwinKeysOf(row: BackupMeasurement): string[] { - const keys = [ - `at|${row.source}|${new Date(row.measuredAt).toISOString()}|${row.sleepStage ?? ""}`, - ]; - if (row.externalId) keys.push(`ext|${row.source}|${row.externalId}`); - return keys; -} - -const foldStatsKey = (source: string, type: string, externalId: string) => - `${source}|${type}|${externalId}`; - -/** What the file's first read noted about its readings as a whole. */ -export interface MeasurementRestorePrepass { - /** Called for every reading as the first read checks it. */ - note(row: BackupMeasurement): void; - /** The live `stats:` ids of the folded types, keyed by source and type. */ - readonly liveFoldStatsIds: ReadonlySet; - /** A row the migration's rule leaves out under either type. */ - isShadowedLegacyRmssd(row: BackupMeasurement): boolean; -} +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; +import { readNote } from "@/lib/crypto/note-cipher"; +import { collect, deferredRows } from "@/lib/export/deferred-rows"; +import type { + BackupExportContext, + BackupExportPart, +} from "@/lib/export/export-context"; +import { iterateMeasurementPages } from "@/lib/export/paged-measurements"; /** - * The notes the measurement pass takes from the first read of the file. + * Every restorable `Measurement` scalar. `userId` is deliberately absent — the + * backup is user-scoped, so the owner is implicit in the query and restored + * from the target account rather than the payload. * - * The live `stats:` ids of the folded types, so the measurement pass can tell - * a compaction tombstone from a person's deletion without a third read of the - * file. Keyed by source as well: Health Connect folds into the same `stats:` - * ids as Apple Health, and an Apple Health hour must not make a Health - * Connect deletion look folded. - * - * Migration 0420 refiled Fitbit and Google Health HRV as `HRV_RMSSD`: both - * providers report a nightly RMSSD, which their mappers had written as the - * SDNN series. A file written before that release still carries those rows - * as `HEART_RATE_VARIABILITY`, and restoring it verbatim would undo the - * migration for the account. The restore applies the migration's rule: - * - * - a legacy row moves to RMSSD, unless a LIVE RMSSD row of its source - * holds the same night under either unique identity (external id, or - * measured-at and sleep stage). Then the twin holds the night and the - * legacy row is not written under either type: as RMSSD it would break - * the unique key, as SDNN it would put RMSSD back into the SDNN series; - * - a legacy row whose only twin is a tombstone wins: the tombstone is the - * row left out, since it shares the key and holds nothing; - * - a legacy tombstone with any RMSSD twin is left out. - * - * The keys are gathered on the first read. + * Named rather than inlined so `measurement-backup-completeness.test.ts` can + * compare it field-by-field against the model in `prisma/schema.prisma`. A + * column added to the model and forgotten here is how the aggregation + * provenance went missing from disaster-recovery backups for several releases. */ -export function createMeasurementRestorePrepass(): MeasurementRestorePrepass { - const liveFoldStatsIds = new Set(); - const foldedTypes = new Set(FOLDED_TYPES); - const foldSources = new Set(["APPLE_HEALTH", "HEALTH_CONNECT"]); - const noteLiveFoldRow = (row: BackupMeasurement) => { - if ( - row.deletedAt == null && - row.source !== undefined && - foldSources.has(row.source) && - foldedTypes.has(row.type) && - row.externalId?.startsWith("stats:") - ) { - liveFoldStatsIds.add(foldStatsKey(row.source, row.type, row.externalId)); - } - }; - const rmssdLiveKeys = new Set(); - const rmssdAnyKeys = new Set(); - const legacyLiveKeys = new Set(); - const noteRmssdTwin = (row: BackupMeasurement) => { - if (isLegacyRmssdRow(row)) { - if (row.deletedAt == null) { - for (const key of rmssdTwinKeysOf(row)) legacyLiveKeys.add(key); - } - return; - } - if ( - row.type === "HRV_RMSSD" && - row.source !== undefined && - LEGACY_RMSSD_SOURCES.has(row.source) - ) { - for (const key of rmssdTwinKeysOf(row)) { - rmssdAnyKeys.add(key); - if (row.deletedAt == null) rmssdLiveKeys.add(key); - } - } - }; - const anyKeyIn = (row: BackupMeasurement, keys: ReadonlySet) => - rmssdTwinKeysOf(row).some((key) => keys.has(key)); - const isShadowedLegacyRmssd = (row: BackupMeasurement): boolean => { - if (isLegacyRmssdRow(row)) { - return anyKeyIn( - row, - row.deletedAt == null ? rmssdLiveKeys : rmssdAnyKeys, - ); - } - // An RMSSD tombstone on the night a live legacy row is about to take. - return ( - row.type === "HRV_RMSSD" && - row.deletedAt != null && - row.source !== undefined && - LEGACY_RMSSD_SOURCES.has(row.source) && - anyKeyIn(row, legacyLiveKeys) && - !anyKeyIn(row, rmssdLiveKeys) - ); - }; - return { - note(row) { - noteLiveFoldRow(row); - noteRmssdTwin(row); - }, - liveFoldStatsIds, - isShadowedLegacyRmssd, - }; -} +const MEASUREMENT_BACKUP_SELECT = { + id: true, + type: true, + value: true, + valueMin: true, + valueMax: true, + unit: true, + measuredAt: true, + source: true, + notes: true, + notesEncrypted: true, + externalId: true, + externalSourceVersion: true, + aggregationProvenance: true, + glucoseContext: true, + sleepStage: true, + rhythmClassification: true, + deviceType: true, + syncVersion: true, + deletedAt: true, + createdAt: true, + updatedAt: true, +} as const satisfies Prisma.MeasurementSelect; -export async function clearMeasurements(ctx: RestoreContext): Promise { - const measurements = await ctx.tx.measurement.deleteMany({ - where: { userId: ctx.ownerId }, - }); - ctx.cleared.measurements = measurements.count; -} +type BackupMeasurementRow = Prisma.MeasurementGetPayload<{ + select: typeof MEASUREMENT_BACKUP_SELECT; +}>; /** - * The persisted measurement rollup partition, wiped for the same reason as - * the mood one: kept, it would hold the previous owner's daily means after - * the underlying measurements were replaced, and the chart's `source=rollup` - * fast-path could surface a stale 30-day mean built from rows that no longer - * exist. The rebuild after the commit mints fresh rows from the restored set. + * One measurement, in whichever of the two wire shapes the caller asked for. + * + * Named rather than inlined so the paged reader below and the materialising + * reader beside it cannot drift: both call this, so a column that reaches one + * shape reaches the other by construction. `measurement-backup-completeness. + * test.ts` proves a selected column reaches a payload by matching + * `: measurement.`, which is why the parameter keeps that spelling. */ -export async function clearMeasurementRollups( - ctx: RestoreContext, -): Promise { - await ctx.tx.measurementRollup.deleteMany({ where: { userId: ctx.ownerId } }); -} - -function toRestoredMeasurementData(measurement: BackupMeasurement) { - return { - type: isLegacyRmssdRow(measurement) - ? ("HRV_RMSSD" as const) - : measurement.type, - value: measurement.value, - valueMin: measurement.valueMin ?? null, - valueMax: measurement.valueMax ?? null, - unit: measurement.unit, - source: measurement.source ?? "MANUAL", - measuredAt: new Date(measurement.measuredAt), - notes: null, - notesEncrypted: - measurement.notesEncrypted == null - ? encryptNote(measurement.notes ?? null) - : decodeEncryptedBytes(measurement.notesEncrypted), - externalId: measurement.externalId ?? null, - externalSourceVersion: measurement.externalSourceVersion ?? null, - // Authority of an Apple Health `stats:` aggregate. A backup - // written before this field rode the payload carries nothing, and - // restoring NULL would flatten the authority ladder: an export.xml - // source-day estimate could then overwrite a native HealthKit - // statistic, which is the ambiguity migration 0263 already - // repaired once. Such a row is exactly the case LEGACY_UNKNOWN - // exists for, so it comes back repairable instead of ambiguous. - // - // The gate is scoped exactly as 0263 scoped its backfill and as - // the cumulative drain scopes its own LEGACY_UNKNOWN writes: - // Apple Health rows only. Other providers mint `stats:` ids for - // their cumulative dailies too (Fitbit, Google Health) and have - // never carried provenance — marking those would invent a state - // they were never in. Ordinary point rows keep NULL honestly. - // - // Only an ABSENT field is the old-backup case. A current backup - // writes the field for every row, and an explicit null is the row's - // real state: the nightly daily-mean fold writes Apple Health - // `stats:` rows with no provenance. Treating that null as absent - // turned every such row into LEGACY_UNKNOWN on restore (#1031). - aggregationProvenance: (measurement.aggregationProvenance !== undefined - ? measurement.aggregationProvenance - : measurement.source === "APPLE_HEALTH" && - measurement.externalId?.startsWith("stats:") - ? "LEGACY_UNKNOWN" - : null) as never, - glucoseContext: (measurement.glucoseContext ?? null) as never, - sleepStage: (measurement.sleepStage ?? null) as never, - rhythmClassification: (measurement.rhythmClassification ?? null) as never, - deviceType: measurement.deviceType ?? null, - syncVersion: measurement.syncVersion ?? 1, - deletedAt: measurement.deletedAt ? new Date(measurement.deletedAt) : null, - ...(measurement.createdAt - ? { createdAt: new Date(measurement.createdAt) } - : {}), - ...(measurement.updatedAt - ? { updatedAt: new Date(measurement.updatedAt) } - : {}), - }; -} - -/** - * Which deleted readings the file carries are not written back. - * - * Deleted readings the file carries are restored as deleted rows - * (tombstones), with one exception: a tombstone deleted longer ago than - * `TOMBSTONE_RETENTION_DAYS` is not written at all. - * - * Such a row is exactly what the nightly tombstone purge - * (`measurement-tombstone-cleanup.ts`) hard-deletes, so writing it costs the - * restore a row in thirteen indexes for a row that is gone by the next - * morning. Nothing reads it in between: the delta feed answers - * `cursorExpired` for every cursor older than the same horizon, so no client - * that would still be served this tombstone exists; the step-consolidation - * repair and the hourly rebuild only look inside the horizon; and the undo of - * a delete is offered for what the purge has not removed yet, which this row - * would not survive either. - * - * Younger tombstones are written. They still have readers: a paired client - * that has not yet drained that deletion from the delta feed, the undo of a - * recent delete, the consolidation repairs that rebuild from folded samples, - * and the unique `(user, type, source, externalId)` key that stops a device - * from uploading a folded sample again. The backup file itself is unchanged - * and keeps every row. - * - * Nor is a compaction tombstone written, whatever its age: a raw sample the - * folds soft-deleted before v1.42, whose whole hour or day the fold had - * reached when it was deleted and which a live `stats:` row of the same - * source in the same file covers. The folds delete those outright now and the - * backlog purge removes the old ones, so writing one back would only hand the - * purge the same row again; the `folded_window` ingest guard is what keeps - * the sample from being uploaded again. Classified by the rule the purge and - * the sync feed use (`folded-window.ts`), against the `stats:` ids gathered - * on the file's first read and the zone the account will have after the - * restore. Counted with the expired ones. - */ -async function expiredTombstoneFilter( - ctx: RestoreContext, -): Promise<(measurement: BackupMeasurement) => boolean> { - const { tx, ownerId, payload } = ctx; - const tombstoneHorizonMs = Date.now() - TOMBSTONE_RETENTION_DAYS * 86_400_000; - const foldTz = resolveUserTimezone( - payload.accountSettings?.timezone ?? - ( - await tx.user.findUnique({ - where: { id: ownerId }, - select: { timezone: true }, - }) - )?.timezone ?? - null, - ); - // Only once the fold repair has been through the means this file holds. - // The repair recomputes a mean an older release folded from part of its - // day, and these tombstones are what it reads. A file exported after the - // account's repair finished carries repaired means, so its compaction - // tombstones can go. Any other file is written back whole, the account's - // repair marker is removed, and the repair runs again once the transaction - // commits; the purge then removes the tombstones as usual. - const foldRepair = await tx.measurementFoldRepair.findUnique({ - where: { userId: ownerId }, - select: { completedAt: true }, - }); - const dropCompactionTombstones = - foldRepair !== null && - new Date(payload.exportedAt).getTime() > foldRepair.completedAt.getTime(); - if (!dropCompactionTombstones) { - await tx.measurementFoldRepair.deleteMany({ - where: { userId: ownerId }, - }); - ctx.outputs.foldRepairReset = true; - } - const { liveFoldStatsIds } = ctx.measurementPrepass; - const isCompactionTombstone = (measurement: BackupMeasurement) => { - if (!dropCompactionTombstones) return false; - if (measurement.deletedAt == null) return false; - const source = measurement.source ?? "MANUAL"; - const covering = compactionTombstoneCoverId( - { +function serialiseMeasurement( + measurement: BackupMeasurementRow, + disasterRecovery: boolean, +) { + return disasterRecovery + ? { + id: measurement.id, type: measurement.type, - source, - externalId: measurement.externalId ?? null, - measuredAt: new Date(measurement.measuredAt), - deletedAt: new Date(measurement.deletedAt), - }, - foldTz, - ); - return ( - covering !== null && - liveFoldStatsIds.has(foldStatsKey(source, measurement.type, covering)) - ); - }; - return (measurement: BackupMeasurement) => - measurement.deletedAt != null && - (new Date(measurement.deletedAt).getTime() < tombstoneHorizonMs || - isCompactionTombstone(measurement)); -} - -/** - * Two later sections point INTO the measurements (a record's - * `sourceMeasurementId`, an ECG strip's `measurementId`) and have to know - * which of those ids this transaction actually wrote. They used to be handed - * every written id; they only ever ask about the ones they reference, so only - * those are collected. - */ -function referencedMeasurementIdsOf( - payload: RestoreContext["payload"], -): Set { - const referenced = new Set(); - for (const record of payload.personalRecords) { - if (record.sourceMeasurementId) { - referenced.add(record.sourceMeasurementId); - } - } - for (const recording of payload.ecgRecordings) { - if (recording.measurementId) { - referenced.add(recording.measurementId); - } - } - return referenced; -} - -/** - * v1 payloads did not require stable ids. Their historical natural-key - * reconciliation is preserved without routing canonical v2 rows through it. - */ -async function reconcileWithoutId( - tx: Prisma.TransactionClient, - ownerId: string, - measurement: BackupMeasurement, -): Promise { - const restoredData = toRestoredMeasurementData(measurement); - const existing = await tx.measurement.findFirst({ - where: { - userId: ownerId, - type: measurement.type, - source: restoredData.source, - measuredAt: restoredData.measuredAt, - sleepStage: restoredData.sleepStage, - }, - select: { id: true }, - }); - if (existing) { - await tx.measurement.update({ - where: { id: existing.id, userId: ownerId }, - data: restoredData, - }); - } else { - await tx.measurement.create({ - data: { userId: ownerId, ...restoredData }, - }); - } -} - -/** - * The measurements, a batch at a time from a second read of the file, so the - * restore never holds more than one batch of them. - */ -export async function restoreMeasurements(ctx: RestoreContext): Promise { - const { tx, ownerId, payload, streamed } = ctx; - const referencedMeasurementIds = referencedMeasurementIdsOf(payload); - const writtenReferencedIds = ctx.links.writtenReferencedMeasurementIds; - const measurementBatchSize = 1_000; - const isExpiredTombstone = await expiredTombstoneFilter(ctx); - const { isShadowedLegacyRmssd } = ctx.measurementPrepass; - ctx.reportMeasurements(0); - await streamed.forEachMeasurementBatch( - measurementBatchSize, - async (fileBatch) => { - const kept = fileBatch.filter( - (measurement) => !isExpiredTombstone(measurement), - ); - ctx.outputs.expiredTombstonesSkipped += fileBatch.length - kept.length; - const batch = kept.filter( - (measurement) => !isShadowedLegacyRmssd(measurement), - ); - const stableRows = batch.flatMap((measurement): MeasurementInsertRow[] => - measurement.id - ? [ - { - id: measurement.id, - userId: ownerId, - ...toRestoredMeasurementData(measurement), - }, - ] - : [], - ); - // One statement per batch, not `createMany`: see - // `insertMeasurementRows` for what the latter cost here. - await insertMeasurementRows(tx, stableRows); - // Counted through the file, so the bar ends at the file's total. - ctx.reportMeasurements(fileBatch.length); - for (const row of stableRows) { - if (referencedMeasurementIds.has(row.id)) { - writtenReferencedIds.add(row.id); - } + value: measurement.value, + valueMin: measurement.valueMin, + valueMax: measurement.valueMax, + unit: measurement.unit, + measuredAt: measurement.measuredAt.toISOString(), + source: measurement.source, + notes: measurement.notes, + notesEncrypted: measurement.notesEncrypted + ? Buffer.from(measurement.notesEncrypted).toString("base64") + : null, + externalId: measurement.externalId, + externalSourceVersion: measurement.externalSourceVersion, + aggregationProvenance: measurement.aggregationProvenance, + glucoseContext: measurement.glucoseContext, + sleepStage: measurement.sleepStage, + rhythmClassification: measurement.rhythmClassification, + deviceType: measurement.deviceType, + syncVersion: measurement.syncVersion, + deletedAt: measurement.deletedAt?.toISOString() ?? null, + createdAt: measurement.createdAt.toISOString(), + updatedAt: measurement.updatedAt.toISOString(), } - for (const measurement of batch) { - if (measurement.id) continue; - await reconcileWithoutId(tx, ownerId, measurement); - } - }, + : { + id: measurement.id, + type: measurement.type, + value: measurement.value, + unit: measurement.unit, + measuredAt: measurement.measuredAt.toISOString(), + source: measurement.source, + notes: readNote(measurement.notesEncrypted, measurement.notes), + deletedAt: measurement.deletedAt?.toISOString() ?? null, + }; +} + +/** + * Every backup measurement, one serialised row at a time. + * + * Keyset-paged through `iterateMeasurementPages`, so the caller decides + * whether to keep the rows (the materialising payload does) or hand each + * straight to a writer and forget it (the streaming writer does). Nothing here + * holds more than one page. + */ +export async function* iterateBackupMeasurements( + prisma: PrismaClient, + userId: string, + disasterRecovery: boolean, +): AsyncGenerator, void, void> { + const pages = iterateMeasurementPages( + prisma, + disasterRecovery ? { userId } : { userId, deletedAt: null }, + MEASUREMENT_BACKUP_SELECT, ); + for await (const page of pages) { + for (const measurement of page) { + yield serialiseMeasurement(measurement, disasterRecovery); + } + } +} + +/** The measurements slice of the payload: the rows, or a marker for them. */ +export async function buildMeasurementsBackupPart( + ctx: BackupExportContext, +): Promise { + const { prisma, userId, disasterRecovery } = ctx; + if (ctx.deferBulk) { + return { + section: { + measurements: deferredRows(() => + iterateBackupMeasurements(prisma, userId, disasterRecovery), + ), + }, + counts: { measurements: 0 }, + }; + } + const measurements = await collect( + iterateBackupMeasurements(prisma, userId, disasterRecovery), + ); + return { + section: { measurements }, + counts: { measurements: measurements.length }, + }; } diff --git a/src/lib/export/measurements-restore.ts b/src/lib/export/measurements-restore.ts new file mode 100644 index 000000000..a6947d4be --- /dev/null +++ b/src/lib/export/measurements-restore.ts @@ -0,0 +1,419 @@ +/** + * The restore half of the measurements domain. The export half, with the + * select every column is carried by, lives in `measurements-backup.ts`. + * + * The restore reads the readings twice. The first read (`readStreamedBackup`, + * before anything is deleted) checks each one and notes what the second read + * has to know about the file as a whole: which `stats:` aggregates are live, + * and which nights the retired heart-rate variability filing shares with an + * RMSSD reading (`createMeasurementRestorePrepass`). The second read, inside + * the transaction, writes them a batch at a time (`restoreMeasurements`), so + * the restore never holds more than one batch. + */ +import type { Prisma } from "@/generated/prisma/client"; +import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client"; +import { encryptNote } from "@/lib/crypto/note-cipher"; +import { + isLegacyRmssdRow, + LEGACY_RMSSD_SOURCES, +} from "@/lib/export/legacy-rmssd"; +import { + insertMeasurementRows, + type MeasurementInsertRow, +} from "@/lib/export/measurement-bulk-insert"; +import { + decodeEncryptedBytes, + type MeasurementRestorePrepass, + type RestoreContext, +} from "@/lib/export/restore-context"; +import { resolveUserTimezone } from "@/lib/measurements/consolidation-base"; +import { + FOLDED_TYPES, + compactionTombstoneCoverId, +} from "@/lib/measurements/folded-window"; +import type { BackupMeasurement } from "@/lib/validations/backup"; + +/** Both unique identities of a row, as it would be filed under `HRV_RMSSD`. */ +function rmssdTwinKeysOf(row: BackupMeasurement): string[] { + const keys = [ + `at|${row.source}|${new Date(row.measuredAt).toISOString()}|${row.sleepStage ?? ""}`, + ]; + if (row.externalId) keys.push(`ext|${row.source}|${row.externalId}`); + return keys; +} + +const foldStatsKey = (source: string, type: string, externalId: string) => + `${source}|${type}|${externalId}`; + +/** + * The notes the measurement pass takes from the first read of the file. + * + * The live `stats:` ids of the folded types, so the measurement pass can tell + * a compaction tombstone from a person's deletion without a third read of the + * file. Keyed by source as well: Health Connect folds into the same `stats:` + * ids as Apple Health, and an Apple Health hour must not make a Health + * Connect deletion look folded. + * + * Migration 0420 refiled Fitbit and Google Health HRV as `HRV_RMSSD`: both + * providers report a nightly RMSSD, which their mappers had written as the + * SDNN series. A file written before that release still carries those rows + * as `HEART_RATE_VARIABILITY`, and restoring it verbatim would undo the + * migration for the account. The restore applies the migration's rule: + * + * - a legacy row moves to RMSSD, unless a LIVE RMSSD row of its source + * holds the same night under either unique identity (external id, or + * measured-at and sleep stage). Then the twin holds the night and the + * legacy row is not written under either type: as RMSSD it would break + * the unique key, as SDNN it would put RMSSD back into the SDNN series; + * - a legacy row whose only twin is a tombstone wins: the tombstone is the + * row left out, since it shares the key and holds nothing; + * - a legacy tombstone with any RMSSD twin is left out. + * + * The keys are gathered on the first read. + */ +export function createMeasurementRestorePrepass(): MeasurementRestorePrepass { + const liveFoldStatsIds = new Set(); + const foldedTypes = new Set(FOLDED_TYPES); + const foldSources = new Set(["APPLE_HEALTH", "HEALTH_CONNECT"]); + const noteLiveFoldRow = (row: BackupMeasurement) => { + if ( + row.deletedAt == null && + row.source !== undefined && + foldSources.has(row.source) && + foldedTypes.has(row.type) && + row.externalId?.startsWith("stats:") + ) { + liveFoldStatsIds.add(foldStatsKey(row.source, row.type, row.externalId)); + } + }; + const rmssdLiveKeys = new Set(); + const rmssdAnyKeys = new Set(); + const legacyLiveKeys = new Set(); + const noteRmssdTwin = (row: BackupMeasurement) => { + if (isLegacyRmssdRow(row)) { + if (row.deletedAt == null) { + for (const key of rmssdTwinKeysOf(row)) legacyLiveKeys.add(key); + } + return; + } + if ( + row.type === "HRV_RMSSD" && + row.source !== undefined && + LEGACY_RMSSD_SOURCES.has(row.source) + ) { + for (const key of rmssdTwinKeysOf(row)) { + rmssdAnyKeys.add(key); + if (row.deletedAt == null) rmssdLiveKeys.add(key); + } + } + }; + const anyKeyIn = (row: BackupMeasurement, keys: ReadonlySet) => + rmssdTwinKeysOf(row).some((key) => keys.has(key)); + const isShadowedLegacyRmssd = (row: BackupMeasurement): boolean => { + if (isLegacyRmssdRow(row)) { + return anyKeyIn( + row, + row.deletedAt == null ? rmssdLiveKeys : rmssdAnyKeys, + ); + } + // An RMSSD tombstone on the night a live legacy row is about to take. + return ( + row.type === "HRV_RMSSD" && + row.deletedAt != null && + row.source !== undefined && + LEGACY_RMSSD_SOURCES.has(row.source) && + anyKeyIn(row, legacyLiveKeys) && + !anyKeyIn(row, rmssdLiveKeys) + ); + }; + return { + note(row) { + noteLiveFoldRow(row); + noteRmssdTwin(row); + }, + liveFoldStatsIds, + isShadowedLegacyRmssd, + }; +} + +export async function clearMeasurements(ctx: RestoreContext): Promise { + const measurements = await ctx.tx.measurement.deleteMany({ + where: { userId: ctx.ownerId }, + }); + ctx.cleared.measurements = measurements.count; +} + +/** + * The persisted measurement rollup partition, wiped for the same reason as + * the mood one: kept, it would hold the previous owner's daily means after + * the underlying measurements were replaced, and the chart's `source=rollup` + * fast-path could surface a stale 30-day mean built from rows that no longer + * exist. The rebuild after the commit mints fresh rows from the restored set. + */ +export async function clearMeasurementRollups( + ctx: RestoreContext, +): Promise { + await ctx.tx.measurementRollup.deleteMany({ where: { userId: ctx.ownerId } }); +} + +function toRestoredMeasurementData(measurement: BackupMeasurement) { + return { + type: isLegacyRmssdRow(measurement) + ? ("HRV_RMSSD" as const) + : measurement.type, + value: measurement.value, + valueMin: measurement.valueMin ?? null, + valueMax: measurement.valueMax ?? null, + unit: measurement.unit, + source: measurement.source ?? "MANUAL", + measuredAt: new Date(measurement.measuredAt), + notes: null, + notesEncrypted: + measurement.notesEncrypted == null + ? encryptNote(measurement.notes ?? null) + : decodeEncryptedBytes(measurement.notesEncrypted), + externalId: measurement.externalId ?? null, + externalSourceVersion: measurement.externalSourceVersion ?? null, + // Authority of an Apple Health `stats:` aggregate. A backup + // written before this field rode the payload carries nothing, and + // restoring NULL would flatten the authority ladder: an export.xml + // source-day estimate could then overwrite a native HealthKit + // statistic, which is the ambiguity migration 0263 already + // repaired once. Such a row is exactly the case LEGACY_UNKNOWN + // exists for, so it comes back repairable instead of ambiguous. + // + // The gate is scoped exactly as 0263 scoped its backfill and as + // the cumulative drain scopes its own LEGACY_UNKNOWN writes: + // Apple Health rows only. Other providers mint `stats:` ids for + // their cumulative dailies too (Fitbit, Google Health) and have + // never carried provenance — marking those would invent a state + // they were never in. Ordinary point rows keep NULL honestly. + // + // Only an ABSENT field is the old-backup case. A current backup + // writes the field for every row, and an explicit null is the row's + // real state: the nightly daily-mean fold writes Apple Health + // `stats:` rows with no provenance. Treating that null as absent + // turned every such row into LEGACY_UNKNOWN on restore (#1031). + aggregationProvenance: (measurement.aggregationProvenance !== undefined + ? measurement.aggregationProvenance + : measurement.source === "APPLE_HEALTH" && + measurement.externalId?.startsWith("stats:") + ? "LEGACY_UNKNOWN" + : null) as never, + glucoseContext: (measurement.glucoseContext ?? null) as never, + sleepStage: (measurement.sleepStage ?? null) as never, + rhythmClassification: (measurement.rhythmClassification ?? null) as never, + deviceType: measurement.deviceType ?? null, + syncVersion: measurement.syncVersion ?? 1, + deletedAt: measurement.deletedAt ? new Date(measurement.deletedAt) : null, + ...(measurement.createdAt + ? { createdAt: new Date(measurement.createdAt) } + : {}), + ...(measurement.updatedAt + ? { updatedAt: new Date(measurement.updatedAt) } + : {}), + }; +} + +/** + * Which deleted readings the file carries are not written back. + * + * Deleted readings the file carries are restored as deleted rows + * (tombstones), with one exception: a tombstone deleted longer ago than + * `TOMBSTONE_RETENTION_DAYS` is not written at all. + * + * Such a row is exactly what the nightly tombstone purge + * (`measurement-tombstone-cleanup.ts`) hard-deletes, so writing it costs the + * restore a row in thirteen indexes for a row that is gone by the next + * morning. Nothing reads it in between: the delta feed answers + * `cursorExpired` for every cursor older than the same horizon, so no client + * that would still be served this tombstone exists; the step-consolidation + * repair and the hourly rebuild only look inside the horizon; and the undo of + * a delete is offered for what the purge has not removed yet, which this row + * would not survive either. + * + * Younger tombstones are written. They still have readers: a paired client + * that has not yet drained that deletion from the delta feed, the undo of a + * recent delete, the consolidation repairs that rebuild from folded samples, + * and the unique `(user, type, source, externalId)` key that stops a device + * from uploading a folded sample again. The backup file itself is unchanged + * and keeps every row. + * + * Nor is a compaction tombstone written, whatever its age: a raw sample the + * folds soft-deleted before v1.42, whose whole hour or day the fold had + * reached when it was deleted and which a live `stats:` row of the same + * source in the same file covers. The folds delete those outright now and the + * backlog purge removes the old ones, so writing one back would only hand the + * purge the same row again; the `folded_window` ingest guard is what keeps + * the sample from being uploaded again. Classified by the rule the purge and + * the sync feed use (`folded-window.ts`), against the `stats:` ids gathered + * on the file's first read and the zone the account will have after the + * restore. Counted with the expired ones. + */ +async function expiredTombstoneFilter( + ctx: RestoreContext, +): Promise<(measurement: BackupMeasurement) => boolean> { + const { tx, ownerId, payload } = ctx; + const tombstoneHorizonMs = Date.now() - TOMBSTONE_RETENTION_DAYS * 86_400_000; + const foldTz = resolveUserTimezone( + payload.accountSettings?.timezone ?? + ( + await tx.user.findUnique({ + where: { id: ownerId }, + select: { timezone: true }, + }) + )?.timezone ?? + null, + ); + // Only once the fold repair has been through the means this file holds. + // The repair recomputes a mean an older release folded from part of its + // day, and these tombstones are what it reads. A file exported after the + // account's repair finished carries repaired means, so its compaction + // tombstones can go. Any other file is written back whole, the account's + // repair marker is removed, and the repair runs again once the transaction + // commits; the purge then removes the tombstones as usual. + const foldRepair = await tx.measurementFoldRepair.findUnique({ + where: { userId: ownerId }, + select: { completedAt: true }, + }); + const dropCompactionTombstones = + foldRepair !== null && + new Date(payload.exportedAt).getTime() > foldRepair.completedAt.getTime(); + if (!dropCompactionTombstones) { + await tx.measurementFoldRepair.deleteMany({ + where: { userId: ownerId }, + }); + ctx.outputs.foldRepairReset = true; + } + const { liveFoldStatsIds } = ctx.measurementPrepass; + const isCompactionTombstone = (measurement: BackupMeasurement) => { + if (!dropCompactionTombstones) return false; + if (measurement.deletedAt == null) return false; + const source = measurement.source ?? "MANUAL"; + const covering = compactionTombstoneCoverId( + { + type: measurement.type, + source, + externalId: measurement.externalId ?? null, + measuredAt: new Date(measurement.measuredAt), + deletedAt: new Date(measurement.deletedAt), + }, + foldTz, + ); + return ( + covering !== null && + liveFoldStatsIds.has(foldStatsKey(source, measurement.type, covering)) + ); + }; + return (measurement: BackupMeasurement) => + measurement.deletedAt != null && + (new Date(measurement.deletedAt).getTime() < tombstoneHorizonMs || + isCompactionTombstone(measurement)); +} + +/** + * Two later sections point INTO the measurements (a record's + * `sourceMeasurementId`, an ECG strip's `measurementId`) and have to know + * which of those ids this transaction actually wrote. They used to be handed + * every written id; they only ever ask about the ones they reference, so only + * those are collected. + */ +function referencedMeasurementIdsOf( + payload: RestoreContext["payload"], +): Set { + const referenced = new Set(); + for (const record of payload.personalRecords) { + if (record.sourceMeasurementId) { + referenced.add(record.sourceMeasurementId); + } + } + for (const recording of payload.ecgRecordings) { + if (recording.measurementId) { + referenced.add(recording.measurementId); + } + } + return referenced; +} + +/** + * v1 payloads did not require stable ids. Their historical natural-key + * reconciliation is preserved without routing canonical v2 rows through it. + */ +async function reconcileWithoutId( + tx: Prisma.TransactionClient, + ownerId: string, + measurement: BackupMeasurement, +): Promise { + const restoredData = toRestoredMeasurementData(measurement); + const existing = await tx.measurement.findFirst({ + where: { + userId: ownerId, + type: measurement.type, + source: restoredData.source, + measuredAt: restoredData.measuredAt, + sleepStage: restoredData.sleepStage, + }, + select: { id: true }, + }); + if (existing) { + await tx.measurement.update({ + where: { id: existing.id, userId: ownerId }, + data: restoredData, + }); + } else { + await tx.measurement.create({ + data: { userId: ownerId, ...restoredData }, + }); + } +} + +/** + * The measurements, a batch at a time from a second read of the file, so the + * restore never holds more than one batch of them. + */ +export async function restoreMeasurements(ctx: RestoreContext): Promise { + const { tx, ownerId, payload, streamed } = ctx; + const referencedMeasurementIds = referencedMeasurementIdsOf(payload); + const writtenReferencedIds = ctx.links.writtenReferencedMeasurementIds; + const measurementBatchSize = 1_000; + const isExpiredTombstone = await expiredTombstoneFilter(ctx); + const { isShadowedLegacyRmssd } = ctx.measurementPrepass; + ctx.reportMeasurements(0); + await streamed.forEachMeasurementBatch( + measurementBatchSize, + async (fileBatch) => { + const kept = fileBatch.filter( + (measurement) => !isExpiredTombstone(measurement), + ); + ctx.outputs.expiredTombstonesSkipped += fileBatch.length - kept.length; + const batch = kept.filter( + (measurement) => !isShadowedLegacyRmssd(measurement), + ); + const stableRows = batch.flatMap((measurement): MeasurementInsertRow[] => + measurement.id + ? [ + { + id: measurement.id, + userId: ownerId, + ...toRestoredMeasurementData(measurement), + }, + ] + : [], + ); + // One statement per batch, not `createMany`: see + // `insertMeasurementRows` for what the latter cost here. + await insertMeasurementRows(tx, stableRows); + // Counted through the file, so the bar ends at the file's total. + ctx.reportMeasurements(fileBatch.length); + for (const row of stableRows) { + if (referencedMeasurementIds.has(row.id)) { + writtenReferencedIds.add(row.id); + } + } + for (const measurement of batch) { + if (measurement.id) continue; + await reconcileWithoutId(tx, ownerId, measurement); + } + }, + ); +} diff --git a/src/lib/export/medications-backup.ts b/src/lib/export/medications-backup.ts new file mode 100644 index 000000000..2ffbfd80c --- /dev/null +++ b/src/lib/export/medications-backup.ts @@ -0,0 +1,521 @@ +/** + * The medications domain's export half: every drug with everything that hangs + * off it, the account's own categories, and the dose history. The restore + * half lives in `medications-restore.ts`. + */ +import { Buffer } from "node:buffer"; + +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; +import { decryptFromBytes } from "@/lib/ai/coach/bytes-codec"; +import { readNote } from "@/lib/crypto/note-cipher"; +import { + BULK_PAGE_SIZE, + collect, + deferredRows, +} from "@/lib/export/deferred-rows"; +import type { + BackupExportContext, + BackupExportPart, +} from "@/lib/export/export-context"; +import { + getMedicationCategories, + readMedicationCategoryLabelsForBackup, +} from "@/lib/medication-category"; +import { dayKeyOfDate } from "@/lib/medications/course-window"; + +// Side effects ride INSIDE their medication, exactly as the schedules +// beside them do. The alternative — a top-level array carrying +// `medicationId` — would have to survive a restore that mints new +// medication ids for a portable file, which is the id-remap the intake +// events already have to work around by drug name. A nested create has +// no id to remap: Prisma binds the child to whatever id the parent row +// actually got. +const MEDICATION_BACKUP_INCLUDE = { + schedules: true, + sideEffects: { orderBy: { occurredAt: "desc" } }, + pauseEras: { orderBy: { pausedAt: "asc" } }, + courses: { orderBy: { startsOn: "asc" } }, + doseChanges: { orderBy: { effectiveFrom: "asc" } }, + inventoryItems: { orderBy: { createdAt: "asc" } }, + inventoryEvents: { orderBy: { occurredAt: "asc" } }, + phaseConfig: true, + // The composition per unit; ordered so two exports of the + // same account write it identically. + ingredients: { orderBy: { nutrient: "asc" } }, + // The archived schedule eras. Ordered TOTALLY, not just by the column + // that reads like the sort key: the position in this list is the + // identifier the supersede pointer travels as, so two eras sharing a + // `validFrom` must not be free to swap places between two exports of + // the same account. + scheduleRevisions: { + orderBy: [ + { validFrom: "asc" }, + { createdAt: "asc" }, + { id: "asc" }, + ] as const, + }, + // What the drug was supposed to move. The biomarker is read by NAME + // rather than by id, exactly as the lab results beside it are: a + // portable restore mints a fresh biomarker id, and `(userId, name)` is + // the natural key the catalogue is already unique on. + efficacyTargets: { + orderBy: { createdAt: "asc" }, + include: { biomarker: { select: { name: true } } }, + }, +} satisfies Prisma.MedicationInclude; + +type BackupMedicationRow = Prisma.MedicationGetPayload<{ + include: typeof MEDICATION_BACKUP_INCLUDE; +}>; + +/** The drug's own columns, before the rows that hang off it. */ +function serialiseMedicationRow( + m: BackupMedicationRow, + disasterRecovery: boolean, + category: string, +) { + return { + ...(disasterRecovery + ? { + id: m.id, + treatmentClass: m.treatmentClass, + dosesPerUnit: m.dosesPerUnit, + unitsPerDose: m.unitsPerDose.toString(), + notificationsEnabled: m.notificationsEnabled, + pausedAt: m.pausedAt?.toISOString() ?? null, + snoozedUntil: m.snoozedUntil?.toISOString() ?? null, + startsOn: m.startsOn?.toISOString() ?? null, + endsOn: m.endsOn?.toISOString() ?? null, + oneShot: m.oneShot, + asNeeded: m.asNeeded, + deliveryForm: m.deliveryForm, + trackInjectionSites: m.trackInjectionSites, + allowedInjectionSites: m.allowedInjectionSites, + liveActivityEnabled: m.liveActivityEnabled, + criticalAlarmEnabled: m.criticalAlarmEnabled, + atcCode: m.atcCode, + rxNormCode: m.rxNormCode, + lowStockNotifiedAt: m.lowStockNotifiedAt?.toISOString() ?? null, + lowStockNotifiedThresholdDays: m.lowStockNotifiedThresholdDays, + reorderLeadDays: m.reorderLeadDays, + externalSource: m.externalSource, + externalId: m.externalId, + updatedAt: m.updatedAt.toISOString(), + } + : // No id: a portable restore mints a fresh one. The relation links + // still name this drug, so the file says which row they mean. + { ref: m.id }), + name: m.name, + dose: m.dose, + active: m.active, + // The person's decision to keep this medication as a + // record only. Both purposes carry it: a restore that dropped it would + // start reminding them about a medication they chose not to track. + trackIntake: m.trackIntake, + // The person's choice to have doses recorded for them, and + // the floor before which none is. Both purposes: a restore without + // them would start reminding again, or fill history it never owned. + autoMarkTaken: m.autoMarkTaken, + autoMarkTakenSince: m.autoMarkTakenSince?.toISOString() ?? null, + // Both purposes: the category is something the person chose. + category, + // Both purposes carry the creation instant: it is the floor of the + // medication's expected slots, so a restore stamped with the restore + // time would read the whole restored history as predating the + // medication and drop its misses from every rate (#1028). + createdAt: m.createdAt.toISOString(), + }; +} + +/** What was planned, recorded and changed: schedules to dose changes. */ +function serialiseMedicationHistory( + m: BackupMedicationRow, + disasterRecovery: boolean, +) { + return { + schedules: m.schedules.map((s) => ({ + ...(disasterRecovery + ? { + id: s.id, + daysOfWeek: s.daysOfWeek, + timesOfDay: s.timesOfDay, + reminderGraceMinutes: s.reminderGraceMinutes, + rrule: s.rrule, + rollingIntervalDays: s.rollingIntervalDays, + scheduleType: s.scheduleType, + cyclicOnWeeks: s.cyclicOnWeeks, + cyclicOffWeeks: s.cyclicOffWeeks, + doseWindows: s.doseWindows, + // #219 — per-schedule units per dose. Decimal → string like the + // medication-level column; NULL (inherit) stays NULL. + unitsPerDose: + s.unitsPerDose == null ? null : s.unitsPerDose.toString(), + } + : {}), + windowStart: s.windowStart, + windowEnd: s.windowEnd, + label: s.label, + dose: s.dose, + })), + // What the person recorded against the drug, and the reason a drug may + // have been stopped. `notes` is the same dual-column arrangement + // `Measurement` has and gets the same treatment: a portable export + // carries the DECRYPTED note and no ciphertext, a disaster-recovery + // payload carries the ciphertext verbatim plus whatever legacy plaintext + // the row still holds, and the restore re-encrypts the plaintext case. + sideEffects: m.sideEffects.map((s) => ({ + ...(disasterRecovery + ? { + id: s.id, + notes: s.notes, + notesEncrypted: s.notesEncrypted + ? Buffer.from(s.notesEncrypted).toString("base64") + : null, + createdAt: s.createdAt.toISOString(), + } + : { notes: readNote(s.notesEncrypted, s.notes) }), + occurredAt: s.occurredAt.toISOString(), + category: s.category, + entry: s.entry, + severity: s.severity, + })), + // Nested for the same reason the two above are: a pause era is bound to + // its medication by id, and a nested create binds it to whatever id the + // parent actually got. + // + // These spans are what separates "stopped taking it" from "was told to + // stop taking it". Without them the compliance recomputation counts a + // deliberate pause as missed doses, so a restored account reads as + // non-adherent for a stretch during which it did exactly what it was + // told. Nothing else in the payload can reconstruct them: an era is a + // decision, not an observation, and no intake row records its absence. + // + // `resumedAt` stays nullable end to end — an open era means the + // medication is still paused, and readers run it to `now`. + pauseEras: m.pauseEras.map((p) => ({ + ...(disasterRecovery ? { id: p.id } : {}), + pausedAt: p.pausedAt.toISOString(), + resumedAt: p.resumedAt ? p.resumedAt.toISOString() : null, + createdAt: p.createdAt.toISOString(), + })), + // The courses: "when did I last take this" lives here + // and nowhere else once a second course exists. The medication's own + // window above is the projection of the latest one. + // What one unit contains. Both purposes: it is something the + // person typed off a label, and without it the supplement's doses stop + // counting toward the day's micronutrients. Canonical amount plus the + // amount as entered, so the form shows what was typed. + ingredients: m.ingredients.map((i) => ({ + ...(disasterRecovery ? { id: i.id } : {}), + nutrient: i.nutrient, + amountPerUnit: i.amountPerUnit, + unit: i.unit, + enteredAmount: i.enteredAmount, + enteredUnit: i.enteredUnit, + source: i.source, + createdAt: i.createdAt.toISOString(), + })), + courses: m.courses.map((c) => ({ + startsOn: dayKeyOfDate(c.startsOn), + endsOn: c.endsOn ? dayKeyOfDate(c.endsOn) : null, + note: c.noteEncrypted ? decryptFromBytes(c.noteEncrypted) : null, + createdAt: c.createdAt.toISOString(), + })), + // Titration: when the dose moved, to what, and why. The drug comes + // back at today's dose either way; without these the ramp that led to + // it is gone, and that ramp is what a prescriber asks about. + // + // The note follows the side-effect contract beside it: a portable + // export carries the DECRYPTED note and no ciphertext, a + // disaster-recovery payload carries the ciphertext verbatim plus + // whatever legacy plaintext the row still holds. + doseChanges: m.doseChanges.map((d) => ({ + ...(disasterRecovery + ? { + id: d.id, + note: d.note, + noteEncrypted: d.noteEncrypted + ? Buffer.from(d.noteEncrypted).toString("base64") + : null, + } + : { note: readNote(d.noteEncrypted, d.note) }), + effectiveFrom: d.effectiveFrom.toISOString(), + doseValue: d.doseValue, + doseUnit: d.doseUnit, + createdAt: d.createdAt.toISOString(), + })), + }; +} + +/** + * The stock, the phase timings, the archived schedule eras and the targets. + * `revisionIndex` is where each era sits in this drug's own ordered list. + */ +function serialiseMedicationStock( + m: BackupMedicationRow, + disasterRecovery: boolean, + revisionIndex: ReadonlyMap | undefined, +) { + return { + // The packs on the shelf, and the ledger behind the number on them. + // + // `unitsRemaining` is carried VERBATIM rather than recomputed from the + // events, for the same reason `nextDueAt` is: it is what the server had + // resolved and what the person was shown. Recomputing on the way back + // would silently correct a count the account may have adjusted by hand, + // and would disagree with every low-stock reminder already sent. + // + // Both Decimal columns cross the wire as strings. A pack of 0.5 mg + // halves is a real prescription, and JSON numbers would round it. + inventoryItems: m.inventoryItems.map((i) => ({ + ...(disasterRecovery + ? { + id: i.id, + notes: i.notes, + notesEncrypted: i.notesEncrypted + ? Buffer.from(i.notesEncrypted).toString("base64") + : null, + } + : { notes: readNote(i.notesEncrypted, i.notes) }), + state: i.state, + containerType: i.containerType, + unitsTotal: i.unitsTotal.toString(), + unitsRemaining: i.unitsRemaining.toString(), + firstUseAt: i.firstUseAt?.toISOString() ?? null, + expiresAt: i.expiresAt?.toISOString() ?? null, + printedExpiry: i.printedExpiry?.toISOString() ?? null, + purchasedAt: i.purchasedAt?.toISOString() ?? null, + manufacturer: i.manufacturer, + doseStrength: i.doseStrength, + createdAt: i.createdAt.toISOString(), + updatedAt: i.updatedAt.toISOString(), + })), + // The four reminder phase thresholds this drug was tuned to. One row or + // none, so it rides as an object rather than a list — and null means the + // person never tuned it, which is a different statement from "the file + // forgot to say". + phaseConfig: m.phaseConfig + ? { + ...(disasterRecovery ? { id: m.phaseConfig.id } : {}), + greenValue: m.phaseConfig.greenValue, + greenMode: m.phaseConfig.greenMode, + yellowValue: m.phaseConfig.yellowValue, + yellowMode: m.phaseConfig.yellowMode, + orangeValue: m.phaseConfig.orangeValue, + orangeMode: m.phaseConfig.orangeMode, + redValue: m.phaseConfig.redValue, + redMode: m.phaseConfig.redMode, + } + : null, + inventoryEvents: m.inventoryEvents.map((e) => ({ + ...(disasterRecovery ? { id: e.id } : {}), + delta: e.delta, + reason: e.reason, + occurredAt: e.occurredAt.toISOString(), + })), + // How the schedule got to where it is. The drug comes back on today's + // plan either way; without these the record of when the dose or the + // timing moved is gone, and the era minter reads every past day against + // the CURRENT schedule instead of the one that was live then — so a + // restored account's own compliance history quietly changes. + // + // `supersededByRevisionId` travels as a POSITION in this list rather + // than as an id. See the restore for the whole argument; the short form + // is that a portable restore mints fresh ids, so an id would point at + // nothing, and the list this array IS is the only stable thing both ends + // agree on. A pointer at a row outside this drug's own list cannot be + // expressed and is written as null — the schema allows it, no code path + // produces it, and inventing an index for it would be worse than saying + // nothing. + scheduleRevisions: m.scheduleRevisions.map((r) => ({ + ...(disasterRecovery ? { id: r.id } : {}), + validFrom: r.validFrom.toISOString(), + validUntil: r.validUntil.toISOString(), + payload: r.payload, + source: r.source, + supersededByIndex: r.supersededByRevisionId + ? (revisionIndex?.get(r.supersededByRevisionId) ?? null) + : null, + createdAt: r.createdAt.toISOString(), + })), + // What the drug is FOR, in the person's own words rather than the + // resolver's. A row exists only where they overrode the derived + // ATC / name inference, so it is the one durable statement of intent + // the "Wirkung" view holds — and the drug restores with no statement of + // what it was for without it. + // + // The lab arm travels as a NAME, the way a lab result's does. The id is + // fresh on a portable restore, and `Biomarker` is unique on + // `(userId, name)`, so the name is the only thing that survives the + // trip. It is resolved on the way back against the biomarkers the + // restore wrote. + efficacyTargets: m.efficacyTargets.map((t) => ({ + ...(disasterRecovery + ? { + id: t.id, + createdAt: t.createdAt.toISOString(), + updatedAt: t.updatedAt.toISOString(), + } + : {}), + measurementType: t.measurementType, + biomarkerName: t.biomarker?.name ?? null, + primary: t.primary, + })), + }; +} + +type BackupIntakeEvent = Prisma.MedicationIntakeEventGetPayload<{ + include: { medication: { select: { name: true } } }; +}>; + +/** One intake event, as it rides the wire. */ +function serialiseIntakeEvent(e: BackupIntakeEvent, disasterRecovery: boolean) { + return { + ...(disasterRecovery + ? { + id: e.id, + medicationId: e.medicationId, + autoMissed: e.autoMissed, + attributionSource: e.attributionSource, + idempotencyKey: e.idempotencyKey, + createdAt: e.createdAt.toISOString(), + injectionSite: e.injectionSite, + doseTaken: e.doseTaken, + inventoryConsumption: e.inventoryConsumption, + externalId: e.externalId, + updatedAt: e.updatedAt.toISOString(), + syncVersion: e.syncVersion, + deletedAt: e.deletedAt?.toISOString() ?? null, + } + : {}), + medication: e.medication.name, + scheduledFor: e.scheduledFor.toISOString(), + takenAt: e.takenAt?.toISOString() ?? null, + skipped: e.skipped, + source: e.source, + // Both purposes: the history labels the dose "recorded + // automatically", and a restore that dropped the marker would relabel + // it as one the person confirmed. + autoRecorded: e.autoRecorded, + }; +} + +/** + * Every backup intake event, one serialised row at a time. + * + * The sort is TOTAL — `scheduledFor desc` then `id desc` — where the + * materialising read used to order by the timestamp alone. A keyset cursor + * needs a total order to page correctly, and a partial one also let two + * exports of the same account emit two different files whenever two doses + * shared a scheduled minute, which every multi-dose day has. + */ +export async function* iterateBackupIntakeEvents( + prisma: PrismaClient, + userId: string, + disasterRecovery: boolean, +): AsyncGenerator, void, void> { + let cursorId: string | null = null; + for (;;) { + const page: BackupIntakeEvent[] = + await prisma.medicationIntakeEvent.findMany({ + where: disasterRecovery ? { userId } : { userId, deletedAt: null }, + include: { medication: { select: { name: true } } }, + orderBy: [{ scheduledFor: "desc" }, { id: "desc" }], + take: BULK_PAGE_SIZE, + ...(cursorId !== null ? { cursor: { id: cursorId }, skip: 1 } : {}), + }); + if (page.length === 0) return; + for (const e of page) yield serialiseIntakeEvent(e, disasterRecovery); + if (page.length < BULK_PAGE_SIZE) return; + cursorId = page[page.length - 1]!.id; + } +} + +/** Totals of the rows nested in the drugs, for the audit trail. */ +function nestedCounts(medications: BackupMedicationRow[]) { + const total = (rows: (m: BackupMedicationRow) => number) => + medications.reduce((sum, m) => sum + rows(m), 0); + return { + medicationSideEffects: total((m) => m.sideEffects.length), + medicationPauseEras: total((m) => m.pauseEras.length), + medicationDoseChanges: total((m) => m.doseChanges.length), + medicationInventoryItems: total((m) => m.inventoryItems.length), + medicationInventoryEvents: total((m) => m.inventoryEvents.length), + reminderPhaseConfigs: medications.filter((m) => m.phaseConfig).length, + medicationEfficacyTargets: total((m) => m.efficacyTargets.length), + medicationScheduleRevisions: total((m) => m.scheduleRevisions.length), + }; +} + +/** + * The medications slice of the payload: the account's own categories, the + * drugs, and the dose history (rows, or a marker for them). + */ +export async function buildMedicationsBackupPart( + ctx: BackupExportContext, +): Promise { + const { prisma, userId, disasterRecovery } = ctx; + const [medications, intakeEvents] = await Promise.all([ + prisma.medication.findMany({ + where: { userId }, + include: MEDICATION_BACKUP_INCLUDE, + }), + ctx.deferBulk + ? null + : collect(iterateBackupIntakeEvents(prisma, userId, disasterRecovery)), + ]); + // The clinical category lives in its own table rather than on the + // medication row and is read through the one helper that normalises it. + // Before v1.39.4 no backup carried it and every restored medication came + // back as "Other". + const categoryByMedication = await getMedicationCategories( + medications.map((m) => m.id), + prisma, + ); + // The account's own categories, restored before the medications that name + // them by key. Labels travel decrypted so a file restores on an instance + // with a different key; every one is carried, hidden ones included, + // because a medication can still be filed under it. + const customMedicationCategories = + await readMedicationCategoryLabelsForBackup(userId, prisma); + // Where each archived era sits in its OWN drug's ordered list. Built once + // here rather than per row, and scoped per medication because the + // supersede pointer never crosses a drug — every route that writes it + // scopes the update to the medication it belongs to. + const revisionIndexByMedication = new Map( + medications.map((m) => [ + m.id, + new Map( + m.scheduleRevisions.map((revision, index) => [revision.id, index]), + ), + ]), + ); + return { + section: { + customMedicationCategories, + medications: medications.map((m) => ({ + ...serialiseMedicationRow( + m, + disasterRecovery, + categoryByMedication[m.id] ?? "OTHER", + ), + ...serialiseMedicationHistory(m, disasterRecovery), + ...serialiseMedicationStock( + m, + disasterRecovery, + revisionIndexByMedication.get(m.id), + ), + })), + intakeEvents: ctx.deferBulk + ? deferredRows(() => + iterateBackupIntakeEvents(prisma, userId, disasterRecovery), + ) + : intakeEvents!, + }, + counts: { + medications: medications.length, + intakeEvents: intakeEvents?.length ?? 0, + ...nestedCounts(medications), + }, + }; +} diff --git a/src/lib/export/mood-backup.ts b/src/lib/export/mood-backup.ts new file mode 100644 index 000000000..439a3f4d9 --- /dev/null +++ b/src/lib/export/mood-backup.ts @@ -0,0 +1,391 @@ +/** + * The mood domain's export half: the entries with their day context and + * their links, and the account's own categories, tags and hidden set. The + * restore half lives in `mood-restore.ts`. + */ +import { Buffer } from "node:buffer"; + +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; +import { readNote } from "@/lib/crypto/note-cipher"; +import { + BULK_PAGE_SIZE, + collect, + deferredRows, +} from "@/lib/export/deferred-rows"; +import type { + BackupExportContext, + BackupExportPart, +} from "@/lib/export/export-context"; +import { openSealedForExport } from "@/lib/export/sealed-text"; + +/** + * Every restorable `MoodEntry` scalar. `userId` is absent for the same reason + * it is absent above: the backup is user-scoped and the owner is re-derived + * from the restoring account. + * + * Named rather than inlined so `mood-backup-completeness.test.ts` can compare + * it field-by-field against the model in `prisma/schema.prisma`, exactly as the + * measurement guard does for the select above. + */ +const MOOD_ENTRY_BACKUP_SELECT = { + id: true, + date: true, + mood: true, + score: true, + moodA1: true, + stressA2: true, + energyA3: true, + connectionA4: true, + stabilityA5: true, + tags: true, + note: true, + noteEncrypted: true, + source: true, + externalId: true, + moodLoggedAt: true, + tz: true, + syncedAt: true, + syncVersion: true, + deletedAt: true, + createdAt: true, + updatedAt: true, +} as const satisfies Prisma.MoodEntrySelect; + +/** + * A structured-tag link, as it rides the wire. The link is carried by the + * tag's KEY, not by its id: a restore resolves the key against the target + * instance's catalogue, where the source instance's row id may not exist at + * all. `kind` comes along so the mapper can split RATED factors from BINARY + * tags without a second query. + */ +const MOOD_ENTRY_TAG_LINK_SELECT = { + rating: true, + moodTag: { select: { key: true, kind: true } }, +} as const satisfies Prisma.MoodEntryTagLinkSelect; + +/** + * Every restorable `MoodContext` scalar. + * + * `moodEntryId` and `userId` are absent for the reasons the exclusion list in + * `mood-backup-completeness.test.ts` states: the context rides inside its + * entry and is re-bound to whatever id that entry got, and the owner is + * re-derived from the restoring account. The two row stamps are absent for the + * third reason there — they describe when the sub-row was written, not when + * the day happened, and the entry already carries the moment that matters. + * + * Named rather than inlined so the guard can compare it field-by-field against + * the model, exactly as it does for the two selects above. This is the whole + * point of the pair: a column added to `MoodContext` without a line here + * fails, and it fails in the release that added it. + */ +const MOOD_CONTEXT_BACKUP_SELECT = { + workStatus: true, + workMinutes: true, + overtimeMinutes: true, + workLoad: true, + workSatisfaction: true, + contactCircles: true, + contactForm: true, + contactExtent: true, + contactQuality: true, + contactSupport: true, + leisureCategories: true, + leisureMinutes: true, + leisureJoy: true, + leisureRecovery: true, + eventType: true, + eventValence: true, + eventAt: true, + notesEncrypted: true, +} as const satisfies Prisma.MoodContextSelect; + +type BackupMoodContext = Prisma.MoodContextGetPayload<{ + select: typeof MOOD_CONTEXT_BACKUP_SELECT; +}>; + +/** + * The mood read, select and joins together. Named because two readers issue + * it — the materialising one and the paged one — and a select that lived in + * only one of them is exactly the drift the completeness guard cannot see. + */ +const MOOD_ENTRY_QUERY_SELECT = { + ...MOOD_ENTRY_BACKUP_SELECT, + // Every link, not only the rated ones. This read used to filter + // `rating: { not: null }`, and a BINARY link carries a NULL rating by + // definition — so the present/absent tags, the kind most people pick, + // were absent from the file and could not come back from it. + tagLinks: { select: MOOD_ENTRY_TAG_LINK_SELECT }, + // The day context, when the entry has one. Sparse by design — most + // entries carry none, and the join answers null on those. + context: { select: MOOD_CONTEXT_BACKUP_SELECT }, +} as const satisfies Prisma.MoodEntrySelect; + +type BackupMoodEntry = Prisma.MoodEntryGetPayload<{ + select: typeof MOOD_ENTRY_QUERY_SELECT; +}>; + +/** + * One context row, in whichever of the two wire shapes the caller asked for. + * + * The split is the same one `Measurement` makes: a disaster-recovery file + * carries the ciphertext verbatim, because the instance that reads it back + * holds the key; a portable export carries the plain text, because the point + * of a portable file is that somebody can read it. Every other column rides in + * both, since none of them is a storage detail — they are all answers somebody + * gave. + * + * The parameter is spelled `moodContext` rather than `c`, deliberately: the + * completeness guard proves a selected column reaches a payload by matching + * `moodContext.`, and a one-letter parameter shared with another + * mapper in this file would make that matcher unable to fail. + */ +function serialiseMoodContext( + moodContext: BackupMoodContext, + disasterRecovery: boolean, +) { + return { + ...(disasterRecovery + ? { + notesEncrypted: moodContext.notesEncrypted + ? Buffer.from(moodContext.notesEncrypted).toString("base64") + : null, + } + : { note: readNote(moodContext.notesEncrypted, null) }), + workStatus: moodContext.workStatus, + workMinutes: moodContext.workMinutes, + overtimeMinutes: moodContext.overtimeMinutes, + workLoad: moodContext.workLoad, + workSatisfaction: moodContext.workSatisfaction, + // The two multi-selects ride as the stored JSON string, unparsed. The + // restore writes the column back verbatim, so a file cannot arrive with a + // list the parser and the column disagree about. + contactCircles: moodContext.contactCircles, + contactForm: moodContext.contactForm, + contactExtent: moodContext.contactExtent, + contactQuality: moodContext.contactQuality, + contactSupport: moodContext.contactSupport, + leisureCategories: moodContext.leisureCategories, + leisureMinutes: moodContext.leisureMinutes, + leisureJoy: moodContext.leisureJoy, + leisureRecovery: moodContext.leisureRecovery, + eventType: moodContext.eventType, + eventValence: moodContext.eventValence, + eventAt: moodContext.eventAt?.toISOString() ?? null, + }; +} + +/** + * One mood entry, with its links partitioned and its day context folded in. + * + * The parameter is spelled out rather than the `e` every other mapper here + * uses, and deliberately: `mood-backup-completeness.test.ts` proves a + * selected column reaches a payload by matching `: moodEntry.`. A matcher + * on `: e.` would also match any other mapper's one-letter parameter and could + * therefore never fail. + */ +function serialiseMoodEntry( + moodEntry: BackupMoodEntry, + disasterRecovery: boolean, +) { + // The two link arms partition the entry's links exhaustively: a link is + // a rated factor when its tag is RATED and it actually carries a score, + // and a structured tag otherwise. Total on purpose — a malformed link + // (a RATED tag whose rating went missing) rides as a tag key and comes + // back NAMED as unresolvable on restore, rather than falling between the + // two arms and disappearing the way every BINARY link used to. + const isRatedFactor = (link: (typeof moodEntry.tagLinks)[number]) => + link.moodTag.kind === "RATED" && link.rating !== null; + + const factors = moodEntry.tagLinks.filter(isRatedFactor).map((tagLink) => ({ + key: tagLink.moodTag.key, + rating: tagLink.rating as number, + })); + // BINARY link keys. Their own array rather than a nullable rating inside + // `factors`: the restore resolves `factors` against `kind: "RATED"` and + // reports what it cannot resolve, so a rating-less entry in that array + // would collide with both. A separate key leaves every backup file + // written before this parsing and restoring exactly as it did. + const structuredTags = moodEntry.tagLinks + .filter((link) => !isRatedFactor(link)) + .map((tagLink) => tagLink.moodTag.key); + + return { + ...(disasterRecovery + ? { + note: moodEntry.note, + noteEncrypted: moodEntry.noteEncrypted + ? Buffer.from(moodEntry.noteEncrypted).toString("base64") + : null, + syncedAt: moodEntry.syncedAt.toISOString(), + syncVersion: moodEntry.syncVersion, + createdAt: moodEntry.createdAt.toISOString(), + updatedAt: moodEntry.updatedAt.toISOString(), + } + : { note: readNote(moodEntry.noteEncrypted, moodEntry.note) }), + id: moodEntry.id, + date: moodEntry.date, + mood: moodEntry.mood, + score: moodEntry.score, + // The five level-A values, under the keys the write path takes. Both + // arms carry them: they are the entry's own answers, not a storage + // detail, and a portable export that dropped them would hand back a + // file describing the day less well than the row it came from. + a1: moodEntry.moodA1, + a2: moodEntry.stressA2, + a3: moodEntry.energyA3, + a4: moodEntry.connectionA4, + a5: moodEntry.stabilityA5, + tags: moodEntry.tags, + source: moodEntry.source, + loggedAt: moodEntry.moodLoggedAt.toISOString(), + externalId: moodEntry.externalId, + // The zone the `date` string is anchored to. Without it a restored row + // falls back to the legacy Europe/Berlin reading and an account that + // logged elsewhere gets its day boundaries quietly moved. + tz: moodEntry.tz, + deletedAt: moodEntry.deletedAt?.toISOString() ?? null, + factors, + structuredTags, + // The day context, or null. Null rather than an empty object, because + // "no context" and "a context whose every field is empty" are + // different facts and the restore has to keep them apart. + context: moodEntry.context + ? serialiseMoodContext(moodEntry.context, disasterRecovery) + : null, + }; +} + +/** + * Every backup mood entry, one serialised row at a time. + * + * Same total-order argument as the intake events above: `moodLoggedAt desc` + * alone is not a cursor, and two entries logged in the same millisecond are + * exactly what a bulk import produces. + */ +export async function* iterateBackupMoodEntries( + prisma: PrismaClient, + userId: string, + disasterRecovery: boolean, +): AsyncGenerator, void, void> { + let cursorId: string | null = null; + for (;;) { + const page: BackupMoodEntry[] = await prisma.moodEntry.findMany({ + where: disasterRecovery ? { userId } : { userId, deletedAt: null }, + orderBy: [{ moodLoggedAt: "desc" }, { id: "desc" }], + take: BULK_PAGE_SIZE, + ...(cursorId !== null ? { cursor: { id: cursorId }, skip: 1 } : {}), + select: MOOD_ENTRY_QUERY_SELECT, + }); + if (page.length === 0) return; + for (const moodEntry of page) { + yield serialiseMoodEntry(moodEntry, disasterRecovery); + } + if (page.length < BULK_PAGE_SIZE) return; + cursorId = page[page.length - 1]!.id; + } +} + +/** + * The mood slice of the payload: the entries (rows, or a marker for them), + * then the account's own categories, hidden set and tags. + */ +export async function buildMoodBackupPart( + ctx: BackupExportContext, +): Promise { + const { prisma, userId, disasterRecovery } = ctx; + const [moodEntries, customMoodTags, customMoodTagCategories, hiddenMoodTags] = + await Promise.all([ + ctx.deferBulk + ? null + : collect(iterateBackupMoodEntries(prisma, userId, disasterRecovery)), + // The account's OWN mood tags. The seeded catalogue is reference data every + // instance has; a tag the user made lives only here. Without it the restore + // cannot resolve the entry's factor keys — and unlike the cycle path, which + // used to drop them silently, the mood path throws, so an account with one + // custom rated tag could not be restored at all. + prisma.moodTag.findMany({ + where: { userId }, + orderBy: { key: "asc" }, + }), + // The account's OWN categories. Seeded ones (`userId: null`) belong to the + // instance and are deliberately not carried — but a custom tag's + // `categoryId` is a real foreign key, so a category the account created + // itself has to travel or the tag above cannot be written at all. Before + // this section existed, that was not a gap in the restore: it was a + // constraint violation that took the WHOLE file down with it, so an + // account that had ever grouped its own mood factors could not be + // restored. + prisma.moodTagCategory.findMany({ + where: { userId }, + orderBy: { sortOrder: "asc" }, + }), + // Which tags the account chose to hide. Carried by KEY rather than id: + // what people hide is almost always a SEEDED tag, whose id differs on + // every instance, so an id would resolve to nothing on the way back. + prisma.moodTagHidden.findMany({ + where: { userId }, + select: { moodTag: { select: { key: true } }, createdAt: true }, + orderBy: { createdAt: "asc" }, + }), + ]); + return { + section: { + moodEntries: ctx.deferBulk + ? deferredRows(() => + iterateBackupMoodEntries(prisma, userId, disasterRecovery), + ) + : moodEntries!, + customMoodTagCategories: customMoodTagCategories.map((category) => ({ + id: category.id, + key: category.key, + labelKey: category.labelKey, + icon: category.icon, + sortOrder: category.sortOrder, + isActive: category.isActive, + // The person's own words: ciphertext in a disaster-recovery payload, + // readable in a portable one (see `sealed-text.ts`). + ...(disasterRecovery + ? { labelEncrypted: category.labelEncrypted } + : { + label: category.labelEncrypted + ? openSealedForExport( + category.labelEncrypted, + "mood category label", + ) + : null, + }), + })), + hiddenMoodTags: hiddenMoodTags.map((hidden) => ({ + key: hidden.moodTag.key, + createdAt: hidden.createdAt.toISOString(), + })), + customMoodTags: customMoodTags.map((tag) => ({ + ...(disasterRecovery ? { id: tag.id } : {}), + key: tag.key, + labelKey: tag.labelKey, + categoryId: tag.categoryId, + kind: tag.kind, + isActive: tag.isActive, + icon: tag.icon, + sortOrder: tag.sortOrder, + // Ciphertext in a disaster-recovery payload (the same instance's key + // reads it back), readable in a portable one (see `sealed-text.ts`). + ...(disasterRecovery + ? { labelEncrypted: tag.labelEncrypted } + : { + label: tag.labelEncrypted + ? openSealedForExport(tag.labelEncrypted, "mood tag label") + : null, + }), + scaleMin: tag.scaleMin, + scaleMax: tag.scaleMax, + inverse: tag.inverse, + })), + }, + counts: { + moodEntries: moodEntries?.length ?? 0, + customMoodTagCategories: customMoodTagCategories.length, + hiddenMoodTags: hiddenMoodTags.length, + }, + }; +} diff --git a/src/lib/export/nutrients-backup.ts b/src/lib/export/nutrients-backup.ts index c0efe157b..09693fc4d 100644 --- a/src/lib/export/nutrients-backup.ts +++ b/src/lib/export/nutrients-backup.ts @@ -1,8 +1,57 @@ /** * The nutrient day totals, with both backup ends in one file. */ +import type { + BackupExportContext, + BackupExportPart, +} from "@/lib/export/export-context"; import type { RestoreContext } from "@/lib/export/restore-context"; +// ── Export ─────────────────────────────────────────────────────────────────── + +/** + * Nutrient day totals were absent from every export path, which contradicted + * the schema's own reason for denormalising the unit column ("rows stay + * self-describing in exports even if the catalog ever drifts"). `source` is + * part of the composite PK, so it has to ride along or a restore cannot tell + * a manual water entry from a synced day total. The table carries no + * tombstone, so both purposes read the same shape. `externalSourceVersion` + * is the source's own version marker, stored so the sync can replay it back + * unchanged; `createdAt` is when the day first arrived. Neither can be + * rebuilt after a restore. + */ +export async function buildNutrientsBackupPart( + ctx: BackupExportContext, +): Promise { + const nutrientDays = await ctx.prisma.nutrientIntakeDay.findMany({ + where: { userId: ctx.userId }, + select: { + day: true, + nutrient: true, + amount: true, + unit: true, + source: true, + externalSourceVersion: true, + createdAt: true, + }, + orderBy: [{ day: "desc" }, { nutrient: "asc" }], + }); + return { + section: { + nutrientDays: nutrientDays.map((n) => ({ + day: n.day, + nutrient: n.nutrient, + amount: n.amount, + unit: n.unit, + source: n.source, + externalSourceVersion: n.externalSourceVersion, + createdAt: n.createdAt.toISOString(), + })), + }, + counts: { nutrientDays: nutrientDays.length }, + }; +} + // ── Restore ────────────────────────────────────────────────────────────────── /** diff --git a/src/lib/export/records-backup.ts b/src/lib/export/records-backup.ts index c003e9db7..756b89f37 100644 --- a/src/lib/export/records-backup.ts +++ b/src/lib/export/records-backup.ts @@ -1,5 +1,12 @@ /** - * Structured records, workouts, and document backup section. + * The structured-records section: labs, illness history, allergies, family + * history, workout summaries and the document vault, with the manifest that + * discloses what each of them leaves out. + * + * Each record kind is a domain of its own, with its export and its restore in + * one module (`labs-backup.ts`, `illness-backup.ts`, `allergies-backup.ts`, + * `family-history-backup.ts`, `workouts-backup.ts`, `documents-backup.ts`). + * This file lays their slices out in the order the file has always had. * * Portable exports keep the historical v1.28 metadata-only document shape. * The weekly disaster-recovery writer opts into the canonical mode, which @@ -10,30 +17,33 @@ * restore cannot resurrect tombstones. Other encrypted free-text fields are * carried as plaintext and re-encrypted by the restore importer; one this * instance cannot decrypt is carried as a visible marker rather than as - * nothing (see `readForExport` below). + * nothing. */ -import { Buffer } from "node:buffer"; -import type { Prisma, PrismaClient } from "@/generated/prisma/client"; -import { decryptNoteFromBytes } from "@/lib/labs/store"; -import { decryptContextFromBytes } from "@/lib/labs/biomarker-store"; -import { UNREADABLE_EXPORT_MARKER } from "./unreadable-marker"; -import { decryptDocumentSummary } from "@/lib/documents/store"; -import { getEvent } from "@/lib/logging/context"; +import type { PrismaClient } from "@/generated/prisma/client"; +import type { FamilyHistoryEntryDTO } from "@/lib/records/dto"; import { - toIllnessEpisodeDTO, - toIllnessDayLogDTO, - dayLogSymptomInclude, - type IllnessEpisodeDTO, - type IllnessDayLogDTO, -} from "@/lib/illness/dto"; + buildAllergiesBackupSlice, + type CanonicalAllergy, +} from "@/lib/export/allergies-backup"; import { - toAllergyDTO, - toFamilyHistoryEntryDTO, - type AllergyDTO, - type FamilyHistoryEntryDTO, -} from "@/lib/records/dto"; -import { dateOnlyKey } from "@/lib/tz/date-only"; -import { streamedField, type StreamedField } from "./streamed-field"; + buildDocumentsBackupSlice, + type DocumentBackupEntry, +} from "@/lib/export/documents-backup"; +import { buildFamilyHistoryBackupSlice } from "@/lib/export/family-history-backup"; +import { + buildIllnessBackupSlice, + type CanonicalIllnessEpisode, +} from "@/lib/export/illness-backup"; +import { + buildLabsBackupSlice, + type BiomarkerBackupEntry, + type BiomarkerMergeBackupEntry, + type LabResultBackupEntry, +} from "@/lib/export/labs-backup"; +import { + buildWorkoutsBackupSlice, + type WorkoutBackupEntry, +} from "@/lib/export/workouts-backup"; /** Disclosed in the payload manifest AND mirrored in the export UI copy. */ export const DOCUMENTS_MANIFEST_NOTE = @@ -48,183 +58,6 @@ export const WORKOUTS_MANIFEST_NOTE = "steps) are included. GPS routes and per-sample heart-rate/pace time " + "series are not included in this export."; -/** - * Decrypt an encrypted free-text column for a portable export, or say plainly - * that this one could not be read. - * - * A fail-soft decrypt answers `null` on a wrong or dropped key. In a list view - * that is right — one bad row must not take the page down. In an export it is - * not: `null` is exactly what a column that was never written looks like, so - * the restore importer writes nothing and a note the person did write - * disappears without anyone being told. The Coach transcript and the doctor - * report both refuse that trade, one with a placeholder sentence and one with - * a rendered flag; this follows the same rule. - * - * The warning the soft helpers emitted is kept, so a systemic key gap still - * shows up in the wide event rather than only in the file. - */ -function readForExport( - buf: Uint8Array | null, - label: string, - decode: (bytes: Uint8Array) => string, -): string | null { - if (!buf || buf.byteLength === 0) return null; - try { - return decode(buf); - } catch (err) { - getEvent()?.addWarning( - `${label} decrypt failed: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - return UNREADABLE_EXPORT_MARKER; - } -} - -export interface LabResultBackupEntry { - /** Present in canonical DR payloads so rows keep a stable identity. */ - id?: string; - /** The source row's id in a portable payload, for the relation links. */ - ref?: string; - panel: string | null; - analyte: string; - value: number | null; - valueText: string | null; - unit: string; - referenceLow: number | null; - referenceHigh: number | null; - /** The window the reading's own source report printed, verbatim + derived. */ - sourceReferenceLow: number | null; - sourceReferenceHigh: number | null; - sourceReferenceText: string | null; - /** - * The value and unit the source report printed, kept when a write - * converted the reading into its marker's unit. - */ - sourceValue?: number | null; - sourceUnit?: string | null; - takenAt: string; - source: string; - /** Human-readable cross-reference into `biomarkers` below, not an id. */ - /** Stable canonical FK; legacy portable payloads use biomarkerName. */ - biomarkerId?: string | null; - biomarkerName: string | null; - note: string | null; - /** Base64 ciphertext in canonical DR payloads. */ - noteEncrypted?: string | null; - createdAt?: string; - updatedAt?: string; - deletedAt?: string | null; -} - -export interface BiomarkerBackupEntry { - /** Present in canonical DR payloads; name remains the natural key. */ - id?: string; - name: string; - unit: string; - lowerBound: number | null; - upperBound: number | null; - panel: string | null; - hidden: boolean; - context: string | null; - /** The catalogue key the marker's name and unit resolve to. */ - analyteKey?: string | null; - /** - * v1.43 — the other spellings the marker answers to, with where - * each came from. Absent on files written before them. - */ - aliases?: BiomarkerAliasBackupEntry[]; - createdAt?: string; - updatedAt?: string; -} - -export interface BiomarkerAliasBackupEntry { - /** Present in canonical DR payloads. */ - id?: string; - name: string; - source: "MERGE" | "IMPORT_CHOICE" | "MANUAL"; - createdAt?: string; -} - -/** - * v1.43 — one marker folded into another. The target travels by id - * in a disaster-recovery file and by name in both, the way a lab result's - * marker does; the source is gone, so its id and name are the record. - */ -export interface BiomarkerMergeBackupEntry { - id?: string; - targetBiomarkerId?: string; - targetName: string; - sourceBiomarkerId: string; - sourceName: string; - readingCount: number; - mergedAt: string; -} - -export interface WorkoutBackupEntry { - /** Present in canonical DR payloads for manual workouts without externalId. */ - id?: string; - createdAt?: string; - updatedAt?: string; - sportType: string; - startedAt: string; - endedAt: string; - durationSec: number; - totalEnergyKcal: number | null; - totalDistanceM: number | null; - avgHeartRate: number | null; - maxHeartRate: number | null; - minHeartRate: number | null; - stepCount: number | null; - elevationM: number | null; - pauseDurationSec: number | null; - source: string; - externalId: string | null; - /** Canonical DR only; portable workout summaries keep their existing shape. */ - metadata?: Prisma.JsonValue; - /** Canonical DR only, beside `metadata`: the source's own version marker. */ - externalSourceVersion?: string | null; -} - -export interface DocumentBackupEntry { - id: string; - kind: string; - title: string | null; - filename: string | null; - mimeType: string; - byteSize: number; - status: string; - reportDate: string | null; - documentDate: string | null; - /** Portable export only; canonical DR carries summaryEncrypted instead. */ - summary?: string | null; - createdAt: string; - updatedAt?: string; - /** Base64 of the already-encrypted BYTEA value; never plaintext. */ - contentEncrypted?: string; - contentSha256?: string | null; - contentCodec?: string; - providerType?: string | null; - errorReason?: string | null; - /** Base64 of the already-encrypted summary BYTEA value. */ - summaryEncrypted?: string | null; - summaryGeneratedAt?: string | null; - summaryState?: string; - /** Refs #776 — the index-attempt record, carried verbatim (canonical DR). */ - lastIndexAttemptAt?: string | null; - lastIndexOutcome?: string | null; - /** - * v1.39.2 (#1038) — where an imported document came from, both payload - * kinds. Optional so a file written before the fields existed still parses. - */ - sourceSystem?: string | null; - sourceId?: string | null; - /** v1.39.3 — the source instance (origin); absent in older files. */ - sourceInstance?: string | null; - /** Canonical DR only: the import's hold on automatic AI reading. */ - aiReadDeferred?: boolean; -} - export interface RecordsBackupOptions { purpose?: "portable-export" | "disaster-recovery"; /** @@ -234,27 +67,6 @@ export interface RecordsBackupOptions { deferDocuments?: boolean; } -type CanonicalIllnessDayLog = IllnessDayLogDTO & { - noteEncrypted?: string | null; - tz?: string | null; - createdAt?: string; - deletedAt?: string | null; -}; - -type CanonicalIllnessEpisode = IllnessEpisodeDTO & { - noteEncrypted?: string | null; - /** The body site's ciphertext, disaster-recovery purpose only (v1.39.2). */ - bodySiteEncrypted?: string | null; - deletedAt?: string | null; - dayLogs: CanonicalIllnessDayLog[]; -}; - -type CanonicalAllergy = AllergyDTO & { - reactionEncrypted?: string | null; - notesEncrypted?: string | null; - deletedAt?: string | null; -}; - export interface RecordsBackupSection { labResults: LabResultBackupEntry[]; biomarkers: BiomarkerBackupEntry[]; @@ -284,223 +96,6 @@ export interface RecordsBackupCounts { documents: number; } -/** The document columns both purposes carry. */ -const DOCUMENT_PORTABLE_SELECT = { - id: true, - kind: true, - title: true, - filename: true, - mimeType: true, - byteSize: true, - status: true, - reportDate: true, - documentDate: true, - summaryEncrypted: true, - sourceSystem: true, - sourceId: true, - sourceInstance: true, - createdAt: true, -} as const satisfies Prisma.InboundDocumentSelect; - -/** - * A disaster-recovery document without its stored bytes. The paged reader - * reads this for every document at once and the bytes one slice at a time. - */ -const DOCUMENT_DR_METADATA_SELECT = { - ...DOCUMENT_PORTABLE_SELECT, - contentSha256: true, - contentCodec: true, - providerType: true, - errorReason: true, - summaryGeneratedAt: true, - summaryState: true, - lastIndexAttemptAt: true, - lastIndexOutcome: true, - aiReadDeferred: true, - updatedAt: true, -} as const satisfies Prisma.InboundDocumentSelect; - -const DOCUMENT_DR_SELECT = { - ...DOCUMENT_DR_METADATA_SELECT, - contentEncrypted: true, -} as const satisfies Prisma.InboundDocumentSelect; - -/** Total, so two exports of the same account list the documents alike. */ -const DOCUMENT_ORDER = [ - { createdAt: "desc" }, - { id: "desc" }, -] as const satisfies Prisma.InboundDocumentOrderByWithRelationInput[]; - -type PortableDocumentRow = Prisma.InboundDocumentGetPayload<{ - select: typeof DOCUMENT_PORTABLE_SELECT; -}>; -type DisasterRecoveryDocumentRow = Prisma.InboundDocumentGetPayload<{ - select: typeof DOCUMENT_DR_METADATA_SELECT; -}>; -type DocumentRow = - | PortableDocumentRow - | Prisma.InboundDocumentGetPayload<{ select: typeof DOCUMENT_DR_SELECT }>; - -/** - * One document as it rides the wire. `contentEncrypted` is the base64 of the - * stored bytes for a disaster-recovery entry and null for a portable one, - * which carries the readable summary instead. Shared by the materialising - * reader and the paged one, so the two cannot drift. - */ -function serialiseDocument( - d: PortableDocumentRow | DisasterRecoveryDocumentRow, - contentEncrypted: string | null, -): DocumentBackupEntry { - const metadata = { - id: d.id, - kind: d.kind, - title: d.title, - filename: d.filename, - mimeType: d.mimeType, - byteSize: d.byteSize, - status: d.status, - reportDate: d.reportDate ? dateOnlyKey(d.reportDate) : null, - documentDate: d.documentDate ? dateOnlyKey(d.documentDate) : null, - sourceSystem: d.sourceSystem, - sourceId: d.sourceId, - sourceInstance: d.sourceInstance, - createdAt: d.createdAt.toISOString(), - }; - - if (contentEncrypted !== null && "contentCodec" in d) { - const dr: DisasterRecoveryDocumentRow = d; - return { - ...metadata, - reportDate: dr.reportDate?.toISOString() ?? null, - documentDate: dr.documentDate?.toISOString() ?? null, - updatedAt: dr.updatedAt.toISOString(), - contentEncrypted, - contentSha256: dr.contentSha256, - contentCodec: dr.contentCodec, - providerType: dr.providerType, - errorReason: dr.errorReason, - summaryEncrypted: dr.summaryEncrypted - ? Buffer.from(dr.summaryEncrypted).toString("base64") - : null, - summaryGeneratedAt: dr.summaryGeneratedAt?.toISOString() ?? null, - summaryState: dr.summaryState, - lastIndexAttemptAt: dr.lastIndexAttemptAt?.toISOString() ?? null, - lastIndexOutcome: dr.lastIndexOutcome, - aiReadDeferred: dr.aiReadDeferred, - }; - } - - return { - ...metadata, - summary: decryptSummarySoft(d.summaryEncrypted), - }; -} - -/** - * How much of a document's stored bytes one read takes. A multiple of three, - * so the base64 of every slice joins into the base64 of the whole. - */ -const DOCUMENT_SLICE_BYTES = 1536 * 1024; - -/** - * How much of a document's base64 the streaming writer shows its observer - * (the restore preview). The key id sits in the first characters; the cap is - * above the longest value the preview keeps as a key sample, so a shortened - * value is never kept as one. A multiple of four, so it still reads as base64. - */ -const DOCUMENT_OBSERVED_CHARS = 128 * 1024; - -/** - * Every document of the account, one at a time, for the streaming backup - * writer (#1239). - * - * A document is up to a hundred megabytes, and the materialising read holds - * every document's bytes, their base64 and the section's JSON at once: about - * five times the size of the vault in heap, which a 524 MB process does not - * have for a vault of 100 MB. Here the metadata of all documents is read - * once, and each document's bytes are read a slice at a time and handed on as - * base64 pieces (`streamedField`), so what is held is one slice. - * - * A document that disappears between the metadata read and its first slice - * is left out, as a later read would not have seen it. One that disappears - * halfway fails the copy: half a document is not a document. - */ -export async function* iterateBackupDocuments( - prisma: Pick, - userId: string, - options: RecordsBackupOptions = {}, -): AsyncGenerator { - if (options.purpose !== "disaster-recovery") { - const rows = await prisma.inboundDocument.findMany({ - where: { userId, deletedAt: null }, - orderBy: DOCUMENT_ORDER, - select: DOCUMENT_PORTABLE_SELECT, - }); - for (const row of rows) yield serialiseDocument(row, null); - return; - } - const rows = await prisma.inboundDocument.findMany({ - where: { userId, deletedAt: null }, - orderBy: DOCUMENT_ORDER, - select: DOCUMENT_DR_METADATA_SELECT, - }); - const slice = async (id: string, from: number): Promise => { - const [row] = await prisma.$queryRaw>` - SELECT substring(content_encrypted FROM ${from + 1}::int - FOR ${DOCUMENT_SLICE_BYTES}::int) AS part - FROM inbound_documents - WHERE id = ${id} AND user_id = ${userId} - `; - if (!row) return null; - const part = row.part ?? new Uint8Array(0); - // A view, not a copy: the slice is read once and dropped. - return Buffer.from(part.buffer, part.byteOffset, part.byteLength); - }; - for (const row of rows) { - const first = await slice(row.id, 0); - if (first === null) continue; - const head = first.toString("base64"); - const pieces = async function* (): AsyncGenerator { - yield head; - let last = first.byteLength; - for (let at = last; last === DOCUMENT_SLICE_BYTES; at += last) { - const next = await slice(row.id, at); - if (next === null) { - throw new Error( - `Document ${row.id} was removed while it was being backed up; ` + - "the next run copies the account again.", - ); - } - last = next.byteLength; - if (last > 0) yield next.toString("base64"); - } - }; - yield streamedField( - serialiseDocument(row, ""), - "contentEncrypted", - head.length > DOCUMENT_OBSERVED_CHARS - ? head.slice(0, DOCUMENT_OBSERVED_CHARS) - : head, - pieces, - ); - } -} - -/** Decrypt a document's optional AI summary fail-soft (null on any error). */ -function decryptSummarySoft(buf: Uint8Array | null): string | null { - if (!buf || buf.byteLength === 0) return null; - try { - return decryptDocumentSummary(buf); - } catch (err) { - getEvent()?.addWarning( - `document summary decrypt failed: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - return null; - } -} - /** * Build the structured-records slice. The default is the established portable * export. Only the weekly DR writer passes `purpose: "disaster-recovery"`. @@ -522,286 +117,28 @@ export async function buildRecordsBackupSection( options: RecordsBackupOptions = {}, ): Promise { const disasterRecovery = options.purpose === "disaster-recovery"; - const documentQuery = options.deferDocuments - ? Promise.resolve([] as DocumentRow[]) - : disasterRecovery - ? prisma.inboundDocument.findMany({ - where: { userId, deletedAt: null }, - orderBy: DOCUMENT_ORDER, - select: DOCUMENT_DR_SELECT, - }) - : prisma.inboundDocument.findMany({ - where: { userId, deletedAt: null }, - orderBy: DOCUMENT_ORDER, - select: DOCUMENT_PORTABLE_SELECT, - }); - const [ - labResultRows, - biomarkerRows, - biomarkerAliasRows, - biomarkerMergeRows, - episodeRows, - allergyRows, - familyRows, - workoutRows, - documentRows, - ] = await Promise.all([ - prisma.labResult.findMany({ - where: disasterRecovery ? { userId } : { userId, deletedAt: null }, - orderBy: { takenAt: "desc" }, - include: { biomarker: { select: { name: true } } }, - }), - prisma.biomarker.findMany({ - where: { userId }, - orderBy: { name: "asc" }, - }), - prisma.biomarkerAlias.findMany({ - where: { userId }, - orderBy: [{ biomarkerId: "asc" }, { nameFolded: "asc" }], - select: { - id: true, - biomarkerId: true, - name: true, - source: true, - createdAt: true, - }, - }), - prisma.biomarkerMerge.findMany({ - where: { userId }, - orderBy: { mergedAt: "asc" }, - include: { target: { select: { name: true } } }, - }), - prisma.illnessEpisode.findMany({ - where: disasterRecovery ? { userId } : { userId, deletedAt: null }, - orderBy: { onsetAt: "desc" }, - include: { - dayLogs: { - ...(disasterRecovery ? {} : { where: { deletedAt: null } }), - orderBy: { date: "asc" }, - include: dayLogSymptomInclude, - }, - }, - }), - prisma.allergy.findMany({ - where: disasterRecovery ? { userId } : { userId, deletedAt: null }, - orderBy: { createdAt: "desc" }, - }), - prisma.familyHistoryEntry.findMany({ - where: { userId, deletedAt: null }, - orderBy: { createdAt: "desc" }, - }), - prisma.workout.findMany({ - where: { userId }, - orderBy: { startedAt: "desc" }, - }), - documentQuery, - ]); - - const labResults: LabResultBackupEntry[] = labResultRows.map((r) => ({ - ...(disasterRecovery - ? { - id: r.id, - biomarkerId: r.biomarkerId, - noteEncrypted: r.noteEncrypted - ? Buffer.from(r.noteEncrypted).toString("base64") - : null, - createdAt: r.createdAt.toISOString(), - updatedAt: r.updatedAt.toISOString(), - deletedAt: r.deletedAt?.toISOString() ?? null, - note: null, - } - : { - // No id: a portable restore mints a fresh one. The relation links - // still name this row, so the file says which row they mean. - ref: r.id, - note: readForExport( - r.noteEncrypted, - "lab note", - decryptNoteFromBytes, - ), - }), - panel: r.panel, - analyte: r.analyte, - value: r.value, - valueText: r.valueText, - unit: r.unit, - referenceLow: r.referenceLow, - referenceHigh: r.referenceHigh, - sourceReferenceLow: r.sourceReferenceLow, - sourceReferenceHigh: r.sourceReferenceHigh, - sourceReferenceText: r.sourceReferenceText, - sourceValue: r.sourceValue, - sourceUnit: r.sourceUnit, - takenAt: r.takenAt.toISOString(), - source: r.source, - biomarkerName: r.biomarker?.name ?? null, - })); - - const aliasesByMarker = new Map(); - for (const alias of biomarkerAliasRows) { - const list = aliasesByMarker.get(alias.biomarkerId) ?? []; - list.push({ - name: alias.name, - source: alias.source as BiomarkerAliasBackupEntry["source"], - ...(disasterRecovery - ? { id: alias.id, createdAt: alias.createdAt.toISOString() } - : {}), - }); - aliasesByMarker.set(alias.biomarkerId, list); - } - - const biomarkerMerges: BiomarkerMergeBackupEntry[] = biomarkerMergeRows.map( - (m) => ({ - ...(disasterRecovery - ? { id: m.id, targetBiomarkerId: m.targetBiomarkerId } - : {}), - targetName: m.target.name, - sourceBiomarkerId: m.sourceBiomarkerId, - sourceName: m.sourceName, - readingCount: m.readingCount, - mergedAt: m.mergedAt.toISOString(), - }), - ); - - const biomarkers: BiomarkerBackupEntry[] = biomarkerRows.map((b) => ({ - ...(disasterRecovery - ? { - id: b.id, - createdAt: b.createdAt.toISOString(), - updatedAt: b.updatedAt.toISOString(), - } - : {}), - name: b.name, - unit: b.unit, - lowerBound: b.lowerBound, - upperBound: b.upperBound, - panel: b.panel, - hidden: b.hidden, - context: readForExport( - b.contextEncrypted, - "biomarker context", - decryptContextFromBytes, - ), - analyteKey: b.analyteKey, - aliases: aliasesByMarker.get(b.id) ?? [], - })); - - const illnessEpisodes: CanonicalIllnessEpisode[] = disasterRecovery - ? episodeRows.map((row) => ({ - id: row.id, - label: row.label, - type: row.type, - lifecycle: row.lifecycle, - onsetAt: row.onsetAt.toISOString(), - resolvedAt: row.resolvedAt?.toISOString() ?? null, - parentConditionId: row.parentConditionId, - note: null, - noteEncrypted: row.noteEncrypted - ? Buffer.from(row.noteEncrypted).toString("base64") - : null, - // Ciphertext verbatim, like the note: the disaster-recovery file never - // carries the site in the clear. - bodySite: null, - bodySiteEncrypted: row.bodySiteEncrypted - ? Buffer.from(row.bodySiteEncrypted).toString("base64") - : null, - laterality: row.laterality, - createdAt: row.createdAt.toISOString(), - updatedAt: row.updatedAt.toISOString(), - deletedAt: row.deletedAt?.toISOString() ?? null, - dayLogs: row.dayLogs.map((dayLog) => ({ - id: dayLog.id, - episodeId: dayLog.episodeId, - date: dayLog.date, - functionalImpact: dayLog.functionalImpact, - feverC: dayLog.feverC, - note: null, - noteEncrypted: dayLog.noteEncrypted - ? Buffer.from(dayLog.noteEncrypted).toString("base64") - : null, - tz: dayLog.tz, - createdAt: dayLog.createdAt.toISOString(), - updatedAt: dayLog.updatedAt.toISOString(), - deletedAt: dayLog.deletedAt?.toISOString() ?? null, - symptoms: dayLog.symptomLinks.map((link) => ({ - key: link.symptom.key, - severity: link.severity, - })), - })), - })) - : episodeRows.map((row) => ({ - ...toIllnessEpisodeDTO(row), - dayLogs: row.dayLogs.map(toIllnessDayLogDTO), - })); - - const allergies: CanonicalAllergy[] = disasterRecovery - ? allergyRows.map((row) => ({ - id: row.id, - substance: row.substance, - category: row.category, - type: row.type, - severity: row.severity, - status: row.status, - onsetAt: row.onsetAt?.toISOString() ?? null, - reaction: null, - note: null, - reactionEncrypted: row.reactionEncrypted - ? Buffer.from(row.reactionEncrypted).toString("base64") - : null, - notesEncrypted: row.notesEncrypted - ? Buffer.from(row.notesEncrypted).toString("base64") - : null, - createdAt: row.createdAt.toISOString(), - updatedAt: row.updatedAt.toISOString(), - deletedAt: row.deletedAt?.toISOString() ?? null, - })) - : allergyRows.map(toAllergyDTO); - const familyHistory = familyRows.map(toFamilyHistoryEntryDTO); - - const workouts: WorkoutBackupEntry[] = workoutRows.map((w) => ({ - ...(disasterRecovery - ? { - id: w.id, - createdAt: w.createdAt.toISOString(), - updatedAt: w.updatedAt.toISOString(), - ...(w.metadata == null ? {} : { metadata: w.metadata }), - externalSourceVersion: w.externalSourceVersion, - } - : {}), - sportType: w.sportType, - startedAt: w.startedAt.toISOString(), - endedAt: w.endedAt.toISOString(), - durationSec: w.durationSec, - totalEnergyKcal: w.totalEnergyKcal, - totalDistanceM: w.totalDistanceM, - avgHeartRate: w.avgHeartRate, - maxHeartRate: w.maxHeartRate, - minHeartRate: w.minHeartRate, - stepCount: w.stepCount, - elevationM: w.elevationM, - pauseDurationSec: w.pauseDurationSec, - source: w.source, - externalId: w.externalId, - })); - - const documents: DocumentBackupEntry[] = documentRows.map((d) => - serialiseDocument( - d, - disasterRecovery && "contentEncrypted" in d - ? Buffer.from(d.contentEncrypted).toString("base64") - : null, - ), - ); + const [labs, illness, allergies, family, workouts, documents] = + await Promise.all([ + buildLabsBackupSlice(prisma, userId, disasterRecovery), + buildIllnessBackupSlice(prisma, userId, disasterRecovery), + buildAllergiesBackupSlice(prisma, userId, disasterRecovery), + buildFamilyHistoryBackupSlice(prisma, userId), + buildWorkoutsBackupSlice(prisma, userId, disasterRecovery), + buildDocumentsBackupSlice( + prisma, + userId, + disasterRecovery, + options.deferDocuments === true, + ), + ]); return { - labResults, - biomarkers, - biomarkerMerges, - illnessEpisodes, - allergies, - familyHistory, - workouts, - documents, + ...labs, + ...illness, + ...allergies, + ...family, + ...workouts, + ...documents, manifest: { documents: disasterRecovery ? { diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index 9b80706ec..737a39044 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -35,14 +35,12 @@ import { RESTORE_CLEAR_STEPS, RESTORE_STEPS, } from "@/lib/export/backup-domains"; -import { - createMeasurementRestorePrepass, - type MeasurementRestorePrepass, -} from "@/lib/export/measurements-backup"; +import { createMeasurementRestorePrepass } from "@/lib/export/measurements-restore"; import { emptyRestoreLinks, orderedCleared, RESTORE_SECTION_STEPS, + type MeasurementRestorePrepass, type RestoreCleared, type RestoreContext, type RestoreSectionStep, diff --git a/src/lib/export/restore-context.ts b/src/lib/export/restore-context.ts index 2a4f1dad7..80734d5a9 100644 --- a/src/lib/export/restore-context.ts +++ b/src/lib/export/restore-context.ts @@ -12,11 +12,13 @@ import { Buffer } from "node:buffer"; import type { Prisma } from "@/generated/prisma/client"; -import type { BackupPayload } from "@/lib/validations/backup"; +import type { + BackupMeasurement, + BackupPayload, +} from "@/lib/validations/backup"; import type { StreamedBackup } from "@/lib/export/streamed-backup"; import type { RestoreSkipLog } from "@/lib/export/restore-skips"; import type { AccountSettingsRestoreResult } from "@/lib/export/account-settings-restore"; -import type { MeasurementRestorePrepass } from "@/lib/export/measurements-backup"; /** * The steps after the measurements that report as sections, in order. The @@ -144,6 +146,21 @@ export interface RestoreLinks { episodeIds: Set; } +/** + * What the file's first read noted about its readings as a whole, for the + * measurement step (`createMeasurementRestorePrepass` in + * `measurements-restore.ts` builds it). Declared here so a module that only + * names the restore context does not import the restore itself. + */ +export interface MeasurementRestorePrepass { + /** Called for every reading as the first read checks it. */ + note(row: BackupMeasurement): void; + /** The live `stats:` ids of the folded types, keyed by source and type. */ + readonly liveFoldStatsIds: ReadonlySet; + /** A row the migration's rule leaves out under either type. */ + isShadowedLegacyRmssd(row: BackupMeasurement): boolean; +} + /** What the steps report back to the orchestrator once they are done. */ export interface RestoreOutputs { /** Measurement tombstones left out (expired or folded). */ diff --git a/src/lib/export/restore-preflight.ts b/src/lib/export/restore-preflight.ts index 5695bf254..07e5f65fa 100644 --- a/src/lib/export/restore-preflight.ts +++ b/src/lib/export/restore-preflight.ts @@ -15,7 +15,7 @@ import { describeBackupKeyProblem, } from "@/lib/export/backup-key-ids"; import { findIncompleteDocument } from "@/lib/export/documents-backup"; -import type { MeasurementRestorePrepass } from "@/lib/export/measurements-backup"; +import type { MeasurementRestorePrepass } from "@/lib/export/restore-context"; import type { RestoreBackupInput, RestoreBackupOutcome, diff --git a/src/lib/export/workouts-backup.ts b/src/lib/export/workouts-backup.ts index 2dec89ca1..af3915364 100644 --- a/src/lib/export/workouts-backup.ts +++ b/src/lib/export/workouts-backup.ts @@ -2,9 +2,76 @@ * The workout summaries, with both backup ends in one file. GPS routes and * per-sample series do not travel, and the file's manifest says so. */ +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; import { toJson } from "@/lib/db"; import type { RestoreContext } from "@/lib/export/restore-context"; +// ── Export ─────────────────────────────────────────────────────────────────── + +export interface WorkoutBackupEntry { + /** Present in canonical DR payloads for manual workouts without externalId. */ + id?: string; + createdAt?: string; + updatedAt?: string; + sportType: string; + startedAt: string; + endedAt: string; + durationSec: number; + totalEnergyKcal: number | null; + totalDistanceM: number | null; + avgHeartRate: number | null; + maxHeartRate: number | null; + minHeartRate: number | null; + stepCount: number | null; + elevationM: number | null; + pauseDurationSec: number | null; + source: string; + externalId: string | null; + /** Canonical DR only; portable workout summaries keep their existing shape. */ + metadata?: Prisma.JsonValue; + /** Canonical DR only, beside `metadata`: the source's own version marker. */ + externalSourceVersion?: string | null; +} + +/** The workout slice of the records section: the summaries. */ +export async function buildWorkoutsBackupSlice( + prisma: Pick, + userId: string, + disasterRecovery: boolean, +): Promise<{ workouts: WorkoutBackupEntry[] }> { + const workoutRows = await prisma.workout.findMany({ + where: { userId }, + orderBy: { startedAt: "desc" }, + }); + + const workouts: WorkoutBackupEntry[] = workoutRows.map((w) => ({ + ...(disasterRecovery + ? { + id: w.id, + createdAt: w.createdAt.toISOString(), + updatedAt: w.updatedAt.toISOString(), + ...(w.metadata == null ? {} : { metadata: w.metadata }), + externalSourceVersion: w.externalSourceVersion, + } + : {}), + sportType: w.sportType, + startedAt: w.startedAt.toISOString(), + endedAt: w.endedAt.toISOString(), + durationSec: w.durationSec, + totalEnergyKcal: w.totalEnergyKcal, + totalDistanceM: w.totalDistanceM, + avgHeartRate: w.avgHeartRate, + maxHeartRate: w.maxHeartRate, + minHeartRate: w.minHeartRate, + stepCount: w.stepCount, + elevationM: w.elevationM, + pauseDurationSec: w.pauseDurationSec, + source: w.source, + externalId: w.externalId, + })); + return { workouts }; +} + // ── Restore ────────────────────────────────────────────────────────────────── export async function clearWorkouts(ctx: RestoreContext): Promise {