fix(backup): read ECG traces a strip at a time

The shared backup reader kept documents without their stored bytes but
kept every ECG strip whole, trace included. A long watch history is
hundreds of strips of tens to hundreds of kilobytes each, so an upload,
preview or restore of such a copy held the whole history in memory.

The first read now keeps each strip without its trace (a short stand-in,
after the key it was written under is recorded and its form checked),
and forEachEcgTrace reads the traces again one strip at a time. The ECG
restore writes each strip from that second read, as the document restore
already did, and refuses a file whose two reads disagree on the count.

The upload memory test gains an ECG-heavy file: 150 strips, about 200 MB.
The old reader held 197 MB of it and failed the 48 MB bound; the new one
holds about 1 MB, also under a 524 MB heap.

src/lib/export/backup-domains.ts

@@ -420,6 +420,7 @@ export const RESTORE_STEPS: readonly RestoreStep[] = [
           ctx.payload,
           ctx.links.writtenReferencedMeasurementIds,
           ctx.skips,
+          (fn) => ctx.streamed.forEachEcgTrace(fn),
         ),
       );
     },

src/lib/export/ecg-backup.ts

@@ -75,6 +75,7 @@ import {
   recordUnknownKeys,
   type RestoreSkipLog,
 } from "@/lib/export/restore-skips";
+import type { EcgTrace } from "@/lib/export/streamed-backup";
 
 export interface EcgBackupOptions {
   purpose?: "portable-export" | "disaster-recovery";
@@ -278,6 +279,11 @@ export function countEcgBackupSection(
   return { ecgRecordings: section.ecgRecordings.length };
 }
 
+/** Hands over each strip's trace from a second read of the file. */
+export type EcgTraceSource = (
+  fn: (trace: EcgTrace, index: number) => Promise<void>,
+) => Promise<void>;
+
 /** Counts the ECG restore wiped, for the audit trail. */
 export interface EcgRestoreCleared {
   ecgRecordings: number;
@@ -351,13 +357,37 @@ export async function restoreEcgData(
   payload: EcgRestoreInput,
   measurementIds: ReadonlySet<string>,
   skips: RestoreSkipLog,
+  forEachTrace: EcgTraceSource,
 ): Promise<EcgRestoreCleared> {
   const cleared = await tx.ecgRecording.deleteMany({
     where: { userId: ownerId },
   });
 
   const droppedMeasurementRefs: string[] = [];
-  for (const recording of payload.ecgRecordings) {
+  // The traces come from a second read of the file, one strip at a time
+  // (`forEachEcgTrace`): the first read kept each strip without its trace,
+  // because a long ECG history held whole is a vault of its own. The strips
+  // are in file order on both reads.
+  let traces = 0;
+  if (payload.ecgRecordings.length > 0) {
+    await forEachTrace(async (trace, index) => {
+      traces++;
+      const listed = payload.ecgRecordings[index];
+      if (!listed) {
+        throw new Error(
+          `ECG recording ${index} of the backup is missing from the first read`,
+        );
+      }
+      await writeRecording({ ...listed, ...trace });
+    });
+  }
+  if (traces !== payload.ecgRecordings.length) {
+    throw new Error(
+      `The backup lists ${payload.ecgRecordings.length} ECG recordings and the second read found ${traces}`,
+    );
+  }
+
+  async function writeRecording(recording: RestoredEcgRecording) {
     let measurementId = recording.measurementId ?? null;
     if (measurementId && !measurementIds.has(measurementId)) {
       droppedMeasurementRefs.push(measurementId);

src/lib/export/streamed-backup.ts

@@ -12,7 +12,9 @@
  * megabytes, and a vault of 190 MB held whole cost more heap than a 524 MB
  * process has (#1239). The first read keeps each document without its stored
  * bytes (`DOCUMENT_CONTENT_LEFT_OUT` in their place), and
- * `forEachDocumentContent` reads them again one document at a time.
+ * `forEachDocumentContent` reads them again one document at a time. The ECG
+ * strips are kept without their traces the same way, and `forEachEcgTrace`
+ * reads those again one strip at a time.
  *
  * The other sections that grow with a record (`BACKUP_STREAMED_SECTIONS`) are
  * read an element at a time too and kept as they are, so no section of the
@@ -60,21 +62,22 @@ export type BackupSource = () => AsyncIterable<Uint8Array>;
 
 const MEASUREMENTS = "measurements";
 const DOCUMENTS = "documents";
+const ECG_RECORDINGS = "ecgRecordings";
 
 /**
  * The sections a backup reader takes an element at a time, never whole: the
  * ones that grow with a record, in rows (measurements, intake events, mood
  * entries) or in stored bytes (documents, ECG traces). How the first read
  * treats each: the measurements are checked and counted, not kept; the
- * documents are kept without their stored bytes; the rest are kept element
- * by element.
+ * documents are kept without their stored bytes and the ECG strips without
+ * their traces; the rest are kept element by element.
  */
 export const BACKUP_STREAMED_SECTIONS = [
   MEASUREMENTS,
   "intakeEvents",
   "moodEntries",
   DOCUMENTS,
-  "ecgRecordings",
+  ECG_RECORDINGS,
 ] as const;
 
 /** `BACKUP_STREAMED_SECTIONS` as `scanBackupJson` takes it. */
@@ -84,6 +87,7 @@ export const BACKUP_STREAM_KEYS: ReadonlySet<string> = new Set(
 
 const MEASUREMENTS_ONLY = new Set([MEASUREMENTS]);
 const DOCUMENTS_ONLY = new Set([DOCUMENTS]);
+const ECG_ONLY = new Set([ECG_RECORDINGS]);
 
 /**
  * What a document's `contentEncrypted` reads as in `raw`, where the stored
@@ -94,6 +98,21 @@ const DOCUMENTS_ONLY = new Set([DOCUMENTS]);
  */
 const DOCUMENT_CONTENT_LEFT_OUT = "AA==";
 
+/**
+ * What an ECG strip's trace reads as in `raw`: the stored ciphertext as the
+ * same short stand-in, or an empty sample array on a portable file. The
+ * restore writes the real trace, from `forEachEcgTrace`.
+ */
+const ECG_CIPHERTEXT_LEFT_OUT = "AA==";
+
+/** One strip's trace, as the second read hands it over. */
+export interface EcgTrace {
+  /** The stored ciphertext as base64 (a disaster-recovery file). */
+  waveformEncrypted?: string;
+  /** The micro-volt samples (a portable file). */
+  waveform?: number[];
+}
+
 /** Base64, as the backup schema accepts it for stored bytes. */
 const BASE64 = /^[A-Za-z0-9+/]+={0,2}$/;
 
@@ -138,6 +157,15 @@ export interface StreamedBackup {
   forEachDocumentContent(
     fn: (content: string | undefined, index: number) => Promise<void>,
   ): Promise<void>;
+  /**
+   * Read the file again and hand over each ECG strip's trace (the stored
+   * ciphertext or the sample array the first read left out of `raw`) with
+   * the strip's position, one strip at a time. Awaits `fn` before reading
+   * further.
+   */
+  forEachEcgTrace(
+    fn: (trace: EcgTrace, index: number) => Promise<void>,
+  ): Promise<void>;
 }
 
 function parseMeasurement(
@@ -200,7 +228,11 @@ export async function readStreamedBackup(
       let section = kept.get(key);
       if (!section) kept.set(key, (section = []));
       section.push(
-        key === DOCUMENTS ? withoutContent(element, index, keys) : element,
+        key === DOCUMENTS
+          ? withoutContent(element, index, keys)
+          : key === ECG_RECORDINGS
+            ? withoutTrace(element, index, keys)
+            : element,
       );
     },
   });
@@ -228,6 +260,9 @@ export async function readStreamedBackup(
   const documentSkipKeys = new Set(
     Object.keys(document).filter((key) => key !== DOCUMENTS),
   );
+  const ecgSkipKeys = new Set(
+    Object.keys(document).filter((key) => key !== ECG_RECORDINGS),
+  );
 
   return {
     raw: document,
@@ -261,9 +296,74 @@ export async function readStreamedBackup(
         },
       });
     },
+    async forEachEcgTrace(fn) {
+      await scanBackupJson(source(), {
+        streamKeys: ECG_ONLY,
+        skipKeys: ecgSkipKeys,
+        onElement: async (_key, element, index) => {
+          const { waveformEncrypted, waveform } = (element ?? {}) as EcgTrace;
+          await fn(
+            {
+              ...(typeof waveformEncrypted === "string"
+                ? { waveformEncrypted }
+                : {}),
+              ...(Array.isArray(waveform) ? { waveform } : {}),
+            },
+            index,
+          );
+        },
+      });
+    },
   };
 }
 
+/**
+ * An ECG strip as `raw` keeps it: its trace replaced by a stand-in, after the
+ * key the ciphertext was written under is recorded and the trace's form is
+ * checked, since the whole-file schema only sees the stand-in. A long Withings
+ * or Apple Watch history is hundreds of strips of tens to hundreds of
+ * kilobytes each; held whole it is the next vault.
+ */
+function withoutTrace(
+  element: unknown,
+  index: number,
+  keys: BackupKeyIdCollector,
+): unknown {
+  if (!element || typeof element !== "object") return element;
+  const { waveformEncrypted, waveform, ...rest } = element as Record<
+    string,
+    unknown
+  >;
+  if (typeof waveformEncrypted === "string") {
+    if (
+      waveformEncrypted.length === 0 ||
+      waveformEncrypted.length % 4 !== 0 ||
+      !BASE64.test(waveformEncrypted)
+    ) {
+      throw new StreamedBackupInvalidError(
+        `${ECG_RECORDINGS}.${index}.waveformEncrypted`,
+        "Expected base64-encoded encrypted bytes",
+      );
+    }
+    keys.add(ECG_RECORDINGS, waveformEncrypted, "waveformEncrypted");
+    return {
+      ...rest,
+      waveformEncrypted: ECG_CIPHERTEXT_LEFT_OUT,
+      ...(waveform === undefined ? {} : { waveform }),
+    };
+  }
+  if (Array.isArray(waveform)) {
+    if (!waveform.every((sample) => typeof sample === "number")) {
+      throw new StreamedBackupInvalidError(
+        `${ECG_RECORDINGS}.${index}.waveform`,
+        "Expected an array of numbers",
+      );
+    }
+    return { ...rest, waveform: [] };
+  }
+  return element;
+}
+
 /**
  * A document as `raw` keeps it: its stored bytes replaced by
  * `DOCUMENT_CONTENT_LEFT_OUT`, after the key they were written under is