mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-15 23:12:04 +00:00
One-shot prettier pass so the new format:check gate starts from a clean baseline. Formatting only — no behaviour change. Generator-owned and vendored artifacts stay untouched via .prettierignore.
372 lines
11 KiB
TypeScript
372 lines
11 KiB
TypeScript
/**
|
||
* CSV measurement import — parse + normalise + per-row validation (v1.17.1).
|
||
*
|
||
* A cold-start escape hatch for self-hosters migrating from a spreadsheet,
|
||
* a glucose-meter export, or another tracker. The genuine import gap left
|
||
* by the JSON (`/api/import`) and Apple-Health (`/api/import/apple-health-export`)
|
||
* routes is plain CSV, which neither covers.
|
||
*
|
||
* This module is the pure, side-effect-free core: it turns raw CSV text
|
||
* into a per-row result set carrying either a normalised measurement (ready
|
||
* for the `/api/import` write loop) or a `skipped` status with a stable
|
||
* machine-readable `reason`. The route (`/api/import/csv`) owns the DB write
|
||
* + rollup re-fold; everything testable without a database lives here.
|
||
*
|
||
* Documented column schema (header row required, order-independent):
|
||
*
|
||
* type,value,unit,measuredAt[,glucoseContext][,notes][,externalId]
|
||
*
|
||
* - `type` — a `measurementTypeEnum` value (e.g. WEIGHT). Unknown → skipped.
|
||
* - `value` — a number in the row's `unit`.
|
||
* - `unit` — the canonical unit for the type, OR a recognised alias:
|
||
* glucose mmol/L → mg/dL (× 18.016),
|
||
* weight lb → kg (× 0.453592).
|
||
* Any other non-canonical unit → skipped (never silently
|
||
* mis-stored).
|
||
* - `measuredAt` — ISO-8601 WITH an explicit offset (`Z` or ±HH:MM). A row
|
||
* without an offset is skipped — the importer never guesses
|
||
* a timezone. The instant is bounded by `validateEntryInstant`
|
||
* (no future beyond a 5-min skew, no instant before 1900).
|
||
* - `glucoseContext` — required for BLOOD_GLUCOSE, forbidden otherwise.
|
||
* - `notes` — free text, ≤ 200 chars.
|
||
* - `externalId` — optional source-stable id. When present, the write loop
|
||
* upserts on `(userId, type, source=IMPORT, externalId)` so a
|
||
* re-upload is idempotent. Absent → re-upload duplicates.
|
||
*/
|
||
import {
|
||
measurementTypeEnum,
|
||
glucoseContextEnum,
|
||
getUnitForType,
|
||
validateMeasurementRange,
|
||
MEASUREMENT_NOTES_MAX_LENGTH,
|
||
} from "@/lib/validations/measurement";
|
||
import { isPlausibleEntryInstant } from "@/lib/validations/entry-instant";
|
||
|
||
/** Canonical conversion factors keyed by `${type}:${loweredAlias}`. */
|
||
const GLUCOSE_MMOL_TO_MGDL = 18.016;
|
||
const LB_TO_KG = 0.453592;
|
||
|
||
/** A normalised, write-ready row in the `/api/import` measurement shape. */
|
||
export interface NormalisedMeasurementRow {
|
||
type: string;
|
||
value: number;
|
||
unit: string;
|
||
measuredAt: Date;
|
||
glucoseContext?: string;
|
||
notes?: string;
|
||
externalId?: string;
|
||
}
|
||
|
||
/** Stable machine-readable skip reasons, mirroring the batch route's vocabulary. */
|
||
export type CsvRowReason =
|
||
| "missing_required_column"
|
||
| "unknown_type"
|
||
| "invalid_value"
|
||
| "value_out_of_range"
|
||
| "unknown_unit"
|
||
| "missing_timezone_offset"
|
||
| "invalid_timestamp"
|
||
| "implausible_timestamp"
|
||
| "missing_glucose_context"
|
||
| "unexpected_glucose_context"
|
||
| "invalid_glucose_context"
|
||
| "notes_too_long"
|
||
| "external_id_too_long"
|
||
| "wrong_column_count";
|
||
|
||
export type CsvRowStatus = "ok" | "skipped";
|
||
|
||
export interface CsvRowResult {
|
||
/** 1-based source line number (header is line 1, first data row is line 2). */
|
||
line: number;
|
||
status: CsvRowStatus;
|
||
reason?: CsvRowReason;
|
||
/** Present only when `status === "ok"`. */
|
||
row?: NormalisedMeasurementRow;
|
||
}
|
||
|
||
export interface CsvParseOutcome {
|
||
/** A fatal, file-level error (e.g. missing required header). When set, `rows` is empty. */
|
||
fatal?: { reason: CsvRowReason; message: string };
|
||
rows: CsvRowResult[];
|
||
}
|
||
|
||
const REQUIRED_COLUMNS = ["type", "value", "unit", "measuredat"] as const;
|
||
const OPTIONAL_COLUMNS = ["glucosecontext", "notes", "externalid"] as const;
|
||
|
||
/**
|
||
* Split CSV text into rows of cells. Hand-rolled, zero-dependency, RFC-4180-ish:
|
||
* handles quoted fields, escaped quotes (`""`), commas + newlines inside quotes,
|
||
* a leading UTF-8 BOM, and CRLF / LF line endings. Trailing blank lines are
|
||
* dropped. Sufficient for the flat measurement schema; not a general CSV engine.
|
||
*/
|
||
export function splitCsvRows(text: string): string[][] {
|
||
// Strip a leading BOM so the first header cell isn't "type".
|
||
const input = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
||
const rows: string[][] = [];
|
||
let cell = "";
|
||
let row: string[] = [];
|
||
let inQuotes = false;
|
||
let sawAnyChar = false;
|
||
|
||
for (let i = 0; i < input.length; i++) {
|
||
const ch = input[i];
|
||
if (inQuotes) {
|
||
if (ch === '"') {
|
||
if (input[i + 1] === '"') {
|
||
cell += '"';
|
||
i++;
|
||
} else {
|
||
inQuotes = false;
|
||
}
|
||
} else {
|
||
cell += ch;
|
||
}
|
||
continue;
|
||
}
|
||
if (ch === '"') {
|
||
inQuotes = true;
|
||
sawAnyChar = true;
|
||
continue;
|
||
}
|
||
if (ch === ",") {
|
||
row.push(cell);
|
||
cell = "";
|
||
sawAnyChar = true;
|
||
continue;
|
||
}
|
||
if (ch === "\r") {
|
||
// Swallow; the \n (if any) finalises the row.
|
||
continue;
|
||
}
|
||
if (ch === "\n") {
|
||
row.push(cell);
|
||
rows.push(row);
|
||
row = [];
|
||
cell = "";
|
||
sawAnyChar = false;
|
||
continue;
|
||
}
|
||
cell += ch;
|
||
sawAnyChar = true;
|
||
}
|
||
// Flush a trailing row that did not end in a newline.
|
||
if (sawAnyChar || cell.length > 0 || row.length > 0) {
|
||
row.push(cell);
|
||
rows.push(row);
|
||
}
|
||
// Drop fully-empty rows (e.g. a trailing blank line that produced [""]).
|
||
return rows.filter((r) => !(r.length === 1 && r[0].trim() === ""));
|
||
}
|
||
|
||
/** True when an ISO-8601 datetime string carries an explicit offset (`Z` or ±HH:MM). */
|
||
function hasExplicitOffset(value: string): boolean {
|
||
// Offset is in the time portion, after the `T`. Match a trailing Z or ±HH:MM.
|
||
return /([zZ]|[+-]\d{2}:?\d{2})$/.test(value.trim());
|
||
}
|
||
|
||
/**
|
||
* Resolve the row's unit to the canonical unit + converted value, or return
|
||
* `null` when the unit is neither canonical nor a recognised alias.
|
||
*/
|
||
function convertToCanonicalUnit(
|
||
type: string,
|
||
value: number,
|
||
rawUnit: string,
|
||
): { value: number; unit: string } | null {
|
||
const canonical = getUnitForType(type);
|
||
const unit = rawUnit.trim();
|
||
// Case-insensitive match against the canonical unit (mg/dL vs MG/DL).
|
||
if (unit.toLowerCase() === canonical.toLowerCase()) {
|
||
return { value, unit: canonical };
|
||
}
|
||
const lower = unit.toLowerCase();
|
||
// Glucose: mmol/L → mg/dL.
|
||
if (type === "BLOOD_GLUCOSE" && (lower === "mmol/l" || lower === "mmol")) {
|
||
return { value: value * GLUCOSE_MMOL_TO_MGDL, unit: canonical };
|
||
}
|
||
// Weight: lb → kg.
|
||
if (
|
||
type === "WEIGHT" &&
|
||
(lower === "lb" ||
|
||
lower === "lbs" ||
|
||
lower === "pound" ||
|
||
lower === "pounds")
|
||
) {
|
||
return { value: value * LB_TO_KG, unit: canonical };
|
||
}
|
||
return null;
|
||
}
|
||
|
||
const KNOWN_TYPES = new Set(measurementTypeEnum.options as readonly string[]);
|
||
const KNOWN_GLUCOSE_CONTEXTS = new Set(
|
||
glucoseContextEnum.options as readonly string[],
|
||
);
|
||
|
||
/**
|
||
* Parse + validate + normalise a CSV measurement file. Returns a per-row
|
||
* result set: each data row is either `ok` with a normalised measurement or
|
||
* `skipped` with a stable reason. A missing required header column is the
|
||
* only fatal, file-level failure.
|
||
*
|
||
* `now` is injectable so tests pin the entry-instant clock.
|
||
*/
|
||
export function parseCsvMeasurements(
|
||
text: string,
|
||
opts: { now?: number } = {},
|
||
): CsvParseOutcome {
|
||
const now = opts.now ?? Date.now();
|
||
const grid = splitCsvRows(text);
|
||
|
||
if (grid.length === 0) {
|
||
return {
|
||
fatal: {
|
||
reason: "missing_required_column",
|
||
message: "The CSV is empty — a header row is required.",
|
||
},
|
||
rows: [],
|
||
};
|
||
}
|
||
|
||
const header = grid[0].map((h) => h.trim().toLowerCase());
|
||
const colIndex = new Map<string, number>();
|
||
header.forEach((name, idx) => {
|
||
if (!colIndex.has(name)) colIndex.set(name, idx);
|
||
});
|
||
|
||
const missing = REQUIRED_COLUMNS.filter((c) => !colIndex.has(c));
|
||
if (missing.length > 0) {
|
||
return {
|
||
fatal: {
|
||
reason: "missing_required_column",
|
||
message: `Missing required column(s): ${missing.join(", ")}. Required header: ${REQUIRED_COLUMNS.join(",")}[,${OPTIONAL_COLUMNS.join(",")}].`,
|
||
},
|
||
rows: [],
|
||
};
|
||
}
|
||
|
||
const idxType = colIndex.get("type")!;
|
||
const idxValue = colIndex.get("value")!;
|
||
const idxUnit = colIndex.get("unit")!;
|
||
const idxMeasuredAt = colIndex.get("measuredat")!;
|
||
const idxContext = colIndex.get("glucosecontext");
|
||
const idxNotes = colIndex.get("notes");
|
||
const idxExternalId = colIndex.get("externalid");
|
||
|
||
const rows: CsvRowResult[] = [];
|
||
|
||
for (let r = 1; r < grid.length; r++) {
|
||
const cells = grid[r];
|
||
const line = r + 1; // 1-based, header is line 1.
|
||
const cellAt = (idx: number | undefined): string =>
|
||
idx === undefined ? "" : (cells[idx] ?? "").trim();
|
||
|
||
const type = cellAt(idxType);
|
||
const rawValue = cellAt(idxValue);
|
||
const rawUnit = cellAt(idxUnit);
|
||
const rawMeasuredAt = cellAt(idxMeasuredAt);
|
||
const rawContext = cellAt(idxContext);
|
||
const rawNotes = idxNotes === undefined ? "" : (cells[idxNotes] ?? "");
|
||
const rawExternalId = cellAt(idxExternalId);
|
||
|
||
const skip = (reason: CsvRowReason): void => {
|
||
rows.push({ line, status: "skipped", reason });
|
||
};
|
||
|
||
if (!KNOWN_TYPES.has(type)) {
|
||
skip("unknown_type");
|
||
continue;
|
||
}
|
||
|
||
const value = Number(rawValue);
|
||
if (rawValue === "" || !Number.isFinite(value)) {
|
||
skip("invalid_value");
|
||
continue;
|
||
}
|
||
|
||
const converted = convertToCanonicalUnit(type, value, rawUnit);
|
||
if (converted === null) {
|
||
skip("unknown_unit");
|
||
continue;
|
||
}
|
||
|
||
if (validateMeasurementRange(type, converted.value) !== null) {
|
||
skip("value_out_of_range");
|
||
continue;
|
||
}
|
||
|
||
if (!rawMeasuredAt) {
|
||
skip("invalid_timestamp");
|
||
continue;
|
||
}
|
||
if (!hasExplicitOffset(rawMeasuredAt)) {
|
||
skip("missing_timezone_offset");
|
||
continue;
|
||
}
|
||
const measuredAt = new Date(rawMeasuredAt);
|
||
if (Number.isNaN(measuredAt.getTime())) {
|
||
skip("invalid_timestamp");
|
||
continue;
|
||
}
|
||
if (!isPlausibleEntryInstant(measuredAt, { now })) {
|
||
skip("implausible_timestamp");
|
||
continue;
|
||
}
|
||
|
||
// Glucose context: required for BLOOD_GLUCOSE, forbidden otherwise.
|
||
let glucoseContext: string | undefined;
|
||
if (type === "BLOOD_GLUCOSE") {
|
||
if (!rawContext) {
|
||
skip("missing_glucose_context");
|
||
continue;
|
||
}
|
||
if (!KNOWN_GLUCOSE_CONTEXTS.has(rawContext)) {
|
||
skip("invalid_glucose_context");
|
||
continue;
|
||
}
|
||
glucoseContext = rawContext;
|
||
} else if (rawContext) {
|
||
skip("unexpected_glucose_context");
|
||
continue;
|
||
}
|
||
|
||
const notes = rawNotes.trim();
|
||
if (notes.length > MEASUREMENT_NOTES_MAX_LENGTH) {
|
||
skip("notes_too_long");
|
||
continue;
|
||
}
|
||
|
||
if (rawExternalId.length > 120) {
|
||
skip("external_id_too_long");
|
||
continue;
|
||
}
|
||
|
||
rows.push({
|
||
line,
|
||
status: "ok",
|
||
row: {
|
||
type,
|
||
value: converted.value,
|
||
unit: converted.unit,
|
||
measuredAt,
|
||
...(glucoseContext ? { glucoseContext } : {}),
|
||
...(notes ? { notes } : {}),
|
||
...(rawExternalId ? { externalId: rawExternalId } : {}),
|
||
},
|
||
});
|
||
}
|
||
|
||
return { rows };
|
||
}
|
||
|
||
/** Column order for the downloadable example + docs. */
|
||
export const CSV_EXAMPLE_COLUMNS = [
|
||
"type",
|
||
"value",
|
||
"unit",
|
||
"measuredAt",
|
||
"glucoseContext",
|
||
"notes",
|
||
"externalId",
|
||
] as const;
|