HealthLog/src/lib/import/__tests__/csv-stream.test.ts

/**
 * The streaming CSV reader: the spreadsheet shapes, every chunk boundary, a
 * fuzzed round trip, and a large file read with bounded memory.
 */
import { describe, expect, it } from "vitest";

import {
  CsvTokenizer,
  CsvTooLargeError,
  MAX_COLUMNS,
  MAX_FIELD_CHARS,
  detectDelimiter,
  readCsvRecords,
  textToStream,
  type CsvRecord,
} from "../csv-stream";

function all(text: string, delimiter: "auto" | "," | ";" | "\t" = "auto") {
  const t = new CsvTokenizer(delimiter);
  return [...t.push(text), ...t.end()];
}

/** Feed the text split at every position, one boundary at a time. */
function everySplit(text: string): CsvRecord[][] {
  const out: CsvRecord[][] = [];
  for (let i = 0; i <= text.length; i++) {
    const t = new CsvTokenizer();
    out.push([
      ...t.push(text.slice(0, i)),
      ...t.push(text.slice(i)),
      ...t.end(),
    ]);
  }
  return out;
}

const fields = (records: CsvRecord[]) => records.map((r) => r.fields);

describe("detectDelimiter", () => {
  it.each([
    ["a,b,c", ","],
    ["a;b;c", ";"],
    ["a\tb\tc", "\t"],
    ['"x;y",b,c', ","],
    ["single", ","],
  ] as const)("%j → %j", (line, expected) => {
    expect(detectDelimiter(line)).toBe(expected);
  });
});

describe("CsvTokenizer", () => {
  it("drops a byte-order mark and reads a semicolon file with decimal commas", () => {
    expect(fields(all("Analyt;Wert;Einheit\nHbA1c;5,4;%\n"))).toEqual([
      ["Analyt", "Wert", "Einheit"],
      ["HbA1c", "5,4", "%"],
    ]);
  });

  it("keeps separators, quotes and line breaks inside quoted fields", () => {
    const text = 'a,b\n"x, y","say ""hi""\nthere"\n';
    expect(fields(all(text))).toEqual([
      ["a", "b"],
      ["x, y", 'say "hi"\nthere'],
    ]);
  });

  it("reads LF, CRLF and lone CR line ends and skips blank lines", () => {
    expect(fields(all("a,b\r\n1,2\r3,4\n\n\r\n5,6"))).toEqual([
      ["a", "b"],
      ["1", "2"],
      ["3", "4"],
      ["5", "6"],
    ]);
  });

  it("reports the physical line each record starts on", () => {
    const records = all('h\n"multi\nline"\n\nlast\n');
    expect(records.map((r) => r.line)).toEqual([1, 2, 5]);
  });

  it("keeps empty fields and a trailing empty field", () => {
    expect(fields(all("a,,c,\n"))).toEqual([["a", "", "c", ""]]);
  });

  it("gives the same records whatever the chunk boundaries", () => {
    const text =
      'name;dose;"note"\r\n"Ram;ipril";"5 mg";"two\r\nlines ""q"""\r\nX;1;\r';
    const expected = fields(all(text));
    for (const records of everySplit(text)) {
      expect(fields(records)).toEqual(expected);
    }
  });

  it("marks a runaway field instead of growing without end", () => {
    const records = all(`a\n"${"x".repeat(MAX_FIELD_CHARS + 50)}\n`);
    expect(records[1].oversized).toBe(true);
    expect(records[1].fields[0].length).toBe(MAX_FIELD_CHARS);
  });

  it("marks a record wider than the column cap", () => {
    const wide = Array.from({ length: MAX_COLUMNS + 5 }, (_, i) =>
      String(i),
    ).join(",");
    const [record] = all(`${wide}\n`);
    expect(record.oversized).toBe(true);
    expect(record.fields).toHaveLength(MAX_COLUMNS);
  });
});

/** Deterministic pseudo-random so a failure reproduces. */
function rng(seed: number) {
  let s = seed;
  return () => {
    s = (s * 1103515245 + 12345) & 0x7fffffff;
    return s / 0x7fffffff;
  };
}

function quoteField(value: string) {
  return /[",;\t\r\n]/.test(value) || value === ""
    ? `"${value.replace(/"/g, '""')}"`
    : value;
}

describe("fuzzed round trip", () => {
  it("reads back exactly what was written, for random tables and chunkings", () => {
    const alphabet = [
      "a",
      "Z",
      "1",
      ",",
      ";",
      '"',
      "\n",
      "\r\n",
      " ",
      "ä",
      "漢",
      "\t",
      ".",
    ];
    const rand = rng(1215);
    for (let round = 0; round < 300; round++) {
      const delimiter = ([",", ";", "\t"] as const)[Math.floor(rand() * 3)];
      const cols = 1 + Math.floor(rand() * 5);
      const rows = 1 + Math.floor(rand() * 6);
      const table: string[][] = [];
      for (let r = 0; r < rows; r++) {
        const row: string[] = [];
        for (let c = 0; c < cols; c++) {
          let v = "";
          const len = Math.floor(rand() * 6);
          for (let k = 0; k < len; k++)
            v += alphabet[Math.floor(rand() * alphabet.length)];
          // A field that is all whitespace-free and quotable stays as written.
          row.push(r === 0 ? `h${c}` : v);
        }
        // A record of only empty fields is skipped by design; keep one cell.
        if (!row.some((f) => f.length > 0)) row[0] = "x";
        table.push(row);
      }
      const text =
        table
          .map((row) => row.map((f) => quoteField(f)).join(delimiter))
          .join(rand() < 0.5 ? "\n" : "\r\n") + (rand() < 0.5 ? "\n" : "");
      const t = new CsvTokenizer(delimiter);
      const out: CsvRecord[] = [];
      let i = 0;
      while (i < text.length) {
        const step = 1 + Math.floor(rand() * 7);
        out.push(...t.push(text.slice(i, i + step)));
        i += step;
      }
      out.push(...t.end());
      // Quoted line breaks come back as "\n": the reader normalises them.
      const expected = table.map((row) =>
        row.map((f) => f.replace(/\r\n/g, "\n")),
      );
      expect(fields(out), `round ${round}: ${JSON.stringify(text)}`).toEqual(
        expected,
      );
    }
  });
});

describe("readCsvRecords", () => {
  it("streams a large file record by record", async () => {
    const rows = 200_000;
    const lines = ["analyte;value;unit;date"];
    for (let i = 0; i < rows; i++)
      lines.push(`Marker ${i % 50};${i % 9},${i % 10};mg/dL;01.02.2024`);
    const text = lines.join("\n");
    let count = 0;
    let maxFieldLength = 0;
    for await (const record of readCsvRecords(textToStream(text, 8192), {
      maxBytes: 64 * 1024 * 1024,
    })) {
      count += 1;
      for (const f of record.fields)
        maxFieldLength = Math.max(maxFieldLength, f.length);
    }
    expect(count).toBe(rows + 1);
    expect(maxFieldLength).toBeLessThan(20);
  });

  it("stops past the byte cap", async () => {
    const text = "a,b\n" + "1,2\n".repeat(1000);
    const consume = async () => {
      for await (const _ of readCsvRecords(textToStream(text, 100), {
        maxBytes: 500,
      })) {
        void _;
      }
    };
    await expect(consume()).rejects.toBeInstanceOf(CsvTooLargeError);
  });

  it("decodes a multi-byte character split across chunks", async () => {
    const text = "name\nÄrztin 漢字\n";
    const out: string[] = [];
    for await (const r of readCsvRecords(textToStream(text, 1), {
      maxBytes: 1000,
    }))
      out.push(r.fields[0]);
    expect(out).toEqual(["name", "Ärztin 漢字"]);
  });
});