mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-07 04:12:04 +00:00
Merge pull request #1140 from MBombeck/ready/pr-1131
fix(import): leave out Apple Health samples HealthLog wrote itself
This commit is contained in:
15 files changed
+1095
-46
No files matched your search
@@ -5375,6 +5375,9 @@
|
||||
"rowsImported": "Bisher {count} Einträge importiert",
|
||||
"doneSummary": "Fertig — {imported} von {read} Datensätzen importiert, {skipped} klinische Datensätze übersprungen.",
|
||||
"doneNothing": "Aus diesem Archiv wurde nichts importiert: 0 von {read} Datensätzen sind angekommen.",
|
||||
"writtenByHealthLogOne": "{count} Eintrag, den HealthLog selbst in Apple Health geschrieben hat, wurde ausgelassen, weil er schon in deinen Daten ist.",
|
||||
"writtenByHealthLogOther": "{count} Einträge, die HealthLog selbst in Apple Health geschrieben hat, wurden ausgelassen, weil sie schon in deinen Daten sind.",
|
||||
"writtenByHealthLogFew": "{count} Einträge, die HealthLog selbst in Apple Health geschrieben hat, wurden ausgelassen, weil sie schon in deinen Daten sind.",
|
||||
"ecgResult": "EKGs: {discovered} gefunden, {imported} importiert, {updated} aktualisiert, {skipped} übersprungen, {failed} fehlgeschlagen. Fehler bei EKGs verwerfen keine unterstützten Datensätze, die aus export.xml importiert wurden.",
|
||||
"estimateWarning": "Kumulative Summen für {count} Tag(e) wurden anhand der größten Exportquelle geschätzt. Native Apple-Health-Statistiken bleiben maßgeblich.",
|
||||
"phase": {
|
||||
|
||||
@@ -5375,6 +5375,9 @@
|
||||
"rowsImported": "{count} rows imported so far",
|
||||
"doneSummary": "Done — {imported} of {read} records imported, {skipped} clinical records skipped.",
|
||||
"doneNothing": "Nothing was imported from that archive: 0 of {read} records landed.",
|
||||
"writtenByHealthLogOne": "{count} record that HealthLog wrote into Apple Health itself was left out, because it is already in your data.",
|
||||
"writtenByHealthLogOther": "{count} records that HealthLog wrote into Apple Health itself were left out, because they are already in your data.",
|
||||
"writtenByHealthLogFew": "{count} records that HealthLog wrote into Apple Health itself were left out, because they are already in your data.",
|
||||
"ecgResult": "ECGs: {discovered} discovered, {imported} imported, {updated} updated, {skipped} skipped, {failed} failed. ECG failures do not discard supported records imported from export.xml.",
|
||||
"estimateWarning": "Cumulative totals for {count} day(s) were estimated from the largest export source. Native Apple Health statistics remain authoritative.",
|
||||
"phase": {
|
||||
|
||||
@@ -5375,6 +5375,9 @@
|
||||
"rowsImported": "{count} filas importadas hasta ahora",
|
||||
"doneSummary": "Listo: {imported} de {read} registros importados, {skipped} registros clínicos omitidos.",
|
||||
"doneNothing": "No se importó nada de ese archivo: llegaron 0 de {read} registros.",
|
||||
"writtenByHealthLogOne": "Se omitió {count} registro que HealthLog escribió en Apple Health, porque ya está en tus datos.",
|
||||
"writtenByHealthLogOther": "Se omitieron {count} registros que HealthLog escribió en Apple Health, porque ya están en tus datos.",
|
||||
"writtenByHealthLogFew": "Se omitieron {count} registros que HealthLog escribió en Apple Health, porque ya están en tus datos.",
|
||||
"ecgResult": "ECG: {discovered} detectados, {imported} importados, {updated} actualizados, {skipped} omitidos y {failed} fallidos. Los errores de ECG no descartan los registros compatibles importados desde export.xml.",
|
||||
"estimateWarning": "Los totales acumulados de {count} día(s) se estimaron a partir de la mayor fuente de la exportación. Las estadísticas nativas de Apple Health siguen siendo la referencia.",
|
||||
"phase": {
|
||||
|
||||
@@ -5375,6 +5375,9 @@
|
||||
"rowsImported": "{count} lignes importées jusqu'ici",
|
||||
"doneSummary": "Terminé — {imported} enregistrements sur {read} importés, {skipped} enregistrements cliniques ignorés.",
|
||||
"doneNothing": "Rien n'a été importé de cette archive : 0 enregistrement sur {read} est arrivé.",
|
||||
"writtenByHealthLogOne": "{count} enregistrement que HealthLog a écrit lui-même dans Santé a été ignoré, car il figure déjà dans vos données.",
|
||||
"writtenByHealthLogOther": "{count} enregistrements que HealthLog a écrits lui-même dans Santé ont été ignorés, car ils figurent déjà dans vos données.",
|
||||
"writtenByHealthLogFew": "{count} enregistrements que HealthLog a écrits lui-même dans Santé ont été ignorés, car ils figurent déjà dans vos données.",
|
||||
"ecgResult": "ECG : {discovered} détectés, {imported} importés, {updated} mis à jour, {skipped} ignorés et {failed} échoués. Les erreurs d’ECG n’annulent pas les enregistrements compatibles importés depuis export.xml.",
|
||||
"estimateWarning": "Les totaux cumulés de {count} jour(s) ont été estimés à partir de la source d’export la plus élevée. Les statistiques Apple Santé natives restent la référence.",
|
||||
"phase": {
|
||||
|
||||
@@ -5375,6 +5375,9 @@
|
||||
"rowsImported": "{count} righe importate finora",
|
||||
"doneSummary": "Fatto: {imported} di {read} record importati, {skipped} record clinici saltati.",
|
||||
"doneNothing": "Da questo archivio non è stato importato nulla: sono arrivati 0 record su {read}.",
|
||||
"writtenByHealthLogOne": "{count} registrazione che HealthLog ha scritto in Salute è stata esclusa, perché è già nei tuoi dati.",
|
||||
"writtenByHealthLogOther": "{count} registrazioni che HealthLog ha scritto in Salute sono state escluse, perché sono già nei tuoi dati.",
|
||||
"writtenByHealthLogFew": "{count} registrazioni che HealthLog ha scritto in Salute sono state escluse, perché sono già nei tuoi dati.",
|
||||
"ecgResult": "ECG: {discovered} rilevati, {imported} importati, {updated} aggiornati, {skipped} ignorati e {failed} non riusciti. Gli errori ECG non eliminano i record supportati importati da export.xml.",
|
||||
"estimateWarning": "I totali cumulativi di {count} giorno/i sono stati stimati dalla fonte di esportazione più alta. Le statistiche native di Apple Salute restano autorevoli.",
|
||||
"phase": {
|
||||
|
||||
@@ -5375,6 +5375,9 @@
|
||||
"rowsImported": "지금까지 {count}개 행을 불러왔어요",
|
||||
"doneSummary": "완료 — {read}개 중 {imported}개 기록을 불러왔고, 진료 기록 {skipped}개는 건너뛰었어요.",
|
||||
"doneNothing": "그 보관 파일에서 불러온 게 없어요: {read}개 중 0개만 들어왔어요.",
|
||||
"writtenByHealthLogOne": "HealthLog가 직접 건강 앱에 기록한 {count}개 항목은 이미 데이터에 있으므로 제외했습니다.",
|
||||
"writtenByHealthLogOther": "HealthLog가 직접 건강 앱에 기록한 {count}개 항목은 이미 데이터에 있으므로 제외했습니다.",
|
||||
"writtenByHealthLogFew": "HealthLog가 직접 건강 앱에 기록한 {count}개 항목은 이미 데이터에 있으므로 제외했습니다.",
|
||||
"ecgResult": "심전도: 발견 {discovered}개, 불러옴 {imported}개, 갱신 {updated}개, 건너뜀 {skipped}개, 실패 {failed}개. 심전도가 실패해도 export.xml에서 불러온 지원 기록은 그대로 남아요.",
|
||||
"estimateWarning": "{count}일치 누적 합계는 가장 큰 내보내기 소스에서 추정했어요. Apple Health 자체 통계가 여전히 기준이에요.",
|
||||
"phase": {
|
||||
|
||||
@@ -5375,6 +5375,9 @@
|
||||
"rowsImported": "Zaimportowano dotąd {count} wierszy",
|
||||
"doneSummary": "Gotowe — zaimportowano {imported} z {read} rekordów, pominięto {skipped} rekordów klinicznych.",
|
||||
"doneNothing": "Z tego archiwum nic nie zaimportowano: dotarło 0 z {read} rekordów.",
|
||||
"writtenByHealthLogOne": "Pominięto {count} wpis, który HealthLog sam zapisał w Apple Health, ponieważ jest już w Twoich danych.",
|
||||
"writtenByHealthLogOther": "Pominięto {count} wpisów, które HealthLog sam zapisał w Apple Health, ponieważ są już w Twoich danych.",
|
||||
"writtenByHealthLogFew": "Pominięto {count} wpisy, które HealthLog sam zapisał w Apple Health, ponieważ są już w Twoich danych.",
|
||||
"ecgResult": "EKG: wykryto {discovered}, zaimportowano {imported}, zaktualizowano {updated}, pominięto {skipped}, nie powiodło się {failed}. Błędy EKG nie usuwają obsługiwanych rekordów zaimportowanych z pliku export.xml.",
|
||||
"estimateWarning": "Łączne wartości dla {count} dni oszacowano na podstawie największego źródła w eksporcie. Natywne statystyki Apple Health pozostają nadrzędne.",
|
||||
"phase": {
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* The note under a finished Apple Health import that says how many records
|
||||
* were left out because HealthLog wrote them into Apple Health itself.
|
||||
*
|
||||
* The count is often 1, and a single string rendered "1 Einträge". The note
|
||||
* reads the plural tier of the active locale, so each locale's own singular
|
||||
* (and the Polish few form) reaches the screen.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
import type { MessageBundle } from "@/lib/i18n/load-locale";
|
||||
import { locales, type Locale } from "@/lib/i18n/config";
|
||||
|
||||
import { AppleHealthOwnRecordsNote } from "../apple-health-import-card";
|
||||
|
||||
function bundle(locale: Locale): MessageBundle {
|
||||
return JSON.parse(
|
||||
readFileSync(join(process.cwd(), "messages", `${locale}.json`), "utf8"),
|
||||
) as MessageBundle;
|
||||
}
|
||||
|
||||
function render(locale: Locale, count: number): string {
|
||||
return (
|
||||
renderToStaticMarkup(
|
||||
<I18nProvider initialLocale={locale} initialMessages={bundle(locale)}>
|
||||
<AppleHealthOwnRecordsNote count={count} />
|
||||
</I18nProvider>,
|
||||
)
|
||||
// Text between the tags; a test reading its own static markup, not a
|
||||
// sanitiser for untrusted input.
|
||||
.split(/<[^>]*>/)
|
||||
.join("")
|
||||
.trim()
|
||||
);
|
||||
}
|
||||
|
||||
describe("<AppleHealthOwnRecordsNote>", () => {
|
||||
it("uses the singular for one record", () => {
|
||||
expect(render("de", 1)).toMatch(/^1 Eintrag,/);
|
||||
expect(render("en", 1)).toMatch(/^1 record that/);
|
||||
expect(render("es", 1)).toContain("1 registro que");
|
||||
expect(render("fr", 1)).toMatch(/^1 enregistrement que/);
|
||||
expect(render("it", 1)).toMatch(/^1 registrazione che/);
|
||||
expect(render("pl", 1)).toContain("1 wpis,");
|
||||
});
|
||||
|
||||
it("uses the plural for several, and the Polish few form for 2 to 4", () => {
|
||||
expect(render("de", 5)).toMatch(/^5 Einträge,/);
|
||||
expect(render("en", 5)).toMatch(/^5 records that/);
|
||||
expect(render("pl", 3)).toContain("3 wpisy,");
|
||||
expect(render("pl", 5)).toContain("5 wpisów,");
|
||||
});
|
||||
|
||||
it("renders a sentence with the count in every locale", () => {
|
||||
for (const locale of locales) {
|
||||
for (const count of [1, 2, 5]) {
|
||||
const text = render(locale, count);
|
||||
expect(text, `${locale} ${count}`).toContain(String(count));
|
||||
expect(text, `${locale} ${count}`).not.toContain("writtenByHealthLog");
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,12 @@ interface JobStatus {
|
||||
totals?: { recordsRead?: number; rowsUpserted?: number };
|
||||
clinical?: { skipped?: number };
|
||||
cumulativeEstimates?: { days?: number; rows?: number };
|
||||
/** Records HealthLog wrote into Apple Health itself, left out here. */
|
||||
writtenByHealthLog?: {
|
||||
byMarker?: number;
|
||||
byExternalId?: number;
|
||||
matchedManual?: number;
|
||||
};
|
||||
/** HK types deferred by design — read but not (yet) imported. */
|
||||
deferred?: Record<string, number>;
|
||||
/** Per-key drop counters — plain HK types are unsupported-by-design,
|
||||
@@ -247,6 +253,22 @@ export function AppleHealthSkipLines({
|
||||
);
|
||||
}
|
||||
|
||||
/** How many records the import left out because HealthLog wrote them. */
|
||||
export function AppleHealthOwnRecordsNote({ count }: { count: number }) {
|
||||
const { tCount } = useTranslations();
|
||||
return (
|
||||
<p
|
||||
data-testid="import-apple-health-own-skipped"
|
||||
className="text-muted-foreground text-sm"
|
||||
>
|
||||
{tCount(
|
||||
"settings.sections.export.import.appleHealth.writtenByHealthLog",
|
||||
count,
|
||||
)}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
export function AppleHealthImportCard() {
|
||||
const { t } = useTranslations();
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
@@ -342,6 +364,10 @@ export function AppleHealthImportCard() {
|
||||
const status = statusQuery.data ?? null;
|
||||
const isRunning = status !== null && !TERMINAL_STATES.includes(status.status);
|
||||
const isDone = status?.status === "done";
|
||||
const ownRecordsSkipped =
|
||||
(status?.result?.writtenByHealthLog?.byMarker ?? 0) +
|
||||
(status?.result?.writtenByHealthLog?.byExternalId ?? 0) +
|
||||
(status?.result?.writtenByHealthLog?.matchedManual ?? 0);
|
||||
const isFailed = status?.status === "failed";
|
||||
const busy = uploading || isRunning;
|
||||
const percent = status?.progress?.percent ?? null;
|
||||
@@ -492,6 +518,9 @@ export function AppleHealthImportCard() {
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
{isDone && ownRecordsSkipped > 0 && (
|
||||
<AppleHealthOwnRecordsNote count={ownRecordsSkipped} />
|
||||
)}
|
||||
{isDone && (status?.result?.cumulativeEstimates?.days ?? 0) > 0 && (
|
||||
<AppleHealthEstimateWarning
|
||||
days={status?.result?.cumulativeEstimates?.days ?? 0}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* The marker HealthLog's iOS app stamps on every HealthKit sample it writes
|
||||
* itself, next to `HKExternalUUID` (the HealthLog row id): a manual entry saved
|
||||
* from the app, and any Withings, import or manual row the server-to-Health
|
||||
* mirror writes. Builds from before the marker existed wrote the id alone.
|
||||
*
|
||||
* Such a sample is already in the database under its own source. When the same
|
||||
* sample comes back through an Apple Health export it is a second copy of that
|
||||
* reading, so the export import leaves it out.
|
||||
*
|
||||
* This is not the predicate the app's own sync uses. The app decides "our own
|
||||
* echo" from `HKExternalUUID` plus the sample's authoring source being the app
|
||||
* itself (`HealthKitSampleOwnership.isOwnEcho`); it keeps this marker for the
|
||||
* deletion path, where a deleted object exposes its metadata but no source. An
|
||||
* export names its source only by a display name, so the import cannot repeat
|
||||
* the source check and reads the marker instead, and for samples from before
|
||||
* the marker, `HKExternalUUID` against the account's own row ids. The key and
|
||||
* value are a stable contract with the app.
|
||||
*/
|
||||
export const HEALTHLOG_ORIGIN_METADATA_KEY = "dev.healthlog.app.origin";
|
||||
export const HEALTHLOG_ORIGIN_METADATA_VALUE = "healthlog";
|
||||
|
||||
/** Is this `<MetadataEntry>` the mark of a sample HealthLog wrote itself? */
|
||||
export function isHealthLogOriginEntry(
|
||||
key: string | undefined,
|
||||
value: string | undefined,
|
||||
): boolean {
|
||||
return (
|
||||
key === HEALTHLOG_ORIGIN_METADATA_KEY &&
|
||||
value === HEALTHLOG_ORIGIN_METADATA_VALUE
|
||||
);
|
||||
}
|
||||
@@ -271,6 +271,21 @@ function makeFakePrisma(
|
||||
}
|
||||
return returned;
|
||||
},
|
||||
// The import reads once which types hold MANUAL rows.
|
||||
groupBy: async ({
|
||||
where,
|
||||
}: {
|
||||
where: { userId: string; source: string };
|
||||
}) =>
|
||||
[
|
||||
...new Set(
|
||||
measurements
|
||||
.filter(
|
||||
(m) => m.userId === where.userId && m.source === where.source,
|
||||
)
|
||||
.map((m) => m.type),
|
||||
),
|
||||
].map((type) => ({ type })),
|
||||
findMany: async ({
|
||||
where,
|
||||
}: {
|
||||
|
||||
@@ -297,6 +297,7 @@ function makeDiscardingPrisma(sample: () => void) {
|
||||
externalId: row.externalId,
|
||||
}));
|
||||
},
|
||||
groupBy: async () => [],
|
||||
findMany: async () => [],
|
||||
findFirst: async () => null,
|
||||
update: async () => {
|
||||
|
||||
@@ -40,6 +40,11 @@ import {
|
||||
dayKeyForUserTz,
|
||||
canonicalDailyTimestamp,
|
||||
} from "@/lib/measurements/drain-per-sample-cumulative";
|
||||
import { isHealthLogOriginEntry } from "@/lib/apple-health/own-origin";
|
||||
import {
|
||||
isSameReadingAcrossSource,
|
||||
MEASURED_AT_TOLERANCE_MS,
|
||||
} from "@/lib/measurements/cross-source-merge";
|
||||
import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement";
|
||||
import {
|
||||
insertNewMeasurementRows,
|
||||
@@ -115,6 +120,19 @@ export interface ImportJobResult {
|
||||
durationMs: number;
|
||||
};
|
||||
clinical: { skipped: number };
|
||||
/**
|
||||
* Records left out because HealthLog wrote them into Apple Health itself and
|
||||
* already holds them under their own source, each counted once however often
|
||||
* the export repeats it: `byMarker` carry the app's origin marker,
|
||||
* `byExternalId` carry a HealthLog row id as `HKExternalUUID` (samples from
|
||||
* before the marker existed), `matchedManual` match a manual entry (same type
|
||||
* and value, within 2 s) without either.
|
||||
*/
|
||||
writtenByHealthLog: {
|
||||
byMarker: number;
|
||||
byExternalId: number;
|
||||
matchedManual: number;
|
||||
};
|
||||
/**
|
||||
* v1.15.0 — reproductive HealthKit samples routed into CYCLE day-logs
|
||||
* (NOT Measurement). Absent / zeroed when the account has no cycle
|
||||
@@ -163,6 +181,29 @@ interface PreparedMeasurement {
|
||||
deviceType: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A quantity `<Record>` read up to its open tag. Its contribution is committed
|
||||
* at the close tag, once its `<MetadataEntry>` children have been read.
|
||||
*/
|
||||
interface PendingRecord {
|
||||
/** Queue the record; `healthLogId` is the `HKExternalUUID` it carried. */
|
||||
commit: (healthLogId: string | null) => void;
|
||||
ownOrigin: boolean;
|
||||
healthLogId: string | null;
|
||||
/** The record's sample identity, computed only when it is left out. */
|
||||
sampleKey: () => string;
|
||||
}
|
||||
|
||||
/** Apple's export name for `HKMetadataKeyExternalUUID`. */
|
||||
const HK_EXTERNAL_UUID_METADATA_KEY = "HKExternalUUID";
|
||||
|
||||
/** The two halves of a blood pressure reading are one kind for an id match. */
|
||||
function kindOf(type: MeasurementType): string {
|
||||
return type === "BLOOD_PRESSURE_SYS" || type === "BLOOD_PRESSURE_DIA"
|
||||
? "BLOOD_PRESSURE"
|
||||
: type;
|
||||
}
|
||||
|
||||
/** Flush row written to the `Workout` table. */
|
||||
interface PreparedWorkout {
|
||||
userId: string;
|
||||
@@ -405,6 +446,8 @@ export async function streamParseExportXml(
|
||||
dayKey: string;
|
||||
rawValue: string | undefined;
|
||||
protectionUsed?: boolean;
|
||||
sampleKey: string;
|
||||
ownOrigin: boolean;
|
||||
} | null = null;
|
||||
|
||||
// Cumulative fold: type -> local day -> hashed source identity -> subtotal.
|
||||
@@ -412,6 +455,31 @@ export async function streamParseExportXml(
|
||||
MeasurementType,
|
||||
Map<string, CumulativeSourceSubtotals>
|
||||
>();
|
||||
// A record's contribution is committed at its close tag, once its child
|
||||
// `<MetadataEntry>` rows are read: one of them can mark the sample as written
|
||||
// by HealthLog itself, which has to keep it out of the import.
|
||||
let pendingRecord: PendingRecord | null = null;
|
||||
const writtenByHealthLog = { byMarker: 0, byExternalId: 0, matchedManual: 0 };
|
||||
// An export can carry the same sample twice (a correlation's records also
|
||||
// stand on their own), so a left-out sample is counted by its sample key.
|
||||
const countedOwn = new Set<string>();
|
||||
const countOwn = (
|
||||
reason: keyof typeof writtenByHealthLog,
|
||||
sampleKey: string,
|
||||
): void => {
|
||||
if (countedOwn.has(sampleKey)) return;
|
||||
countedOwn.add(sampleKey);
|
||||
writtenByHealthLog[reason] += 1;
|
||||
};
|
||||
// The `HKExternalUUID` a queued spot row carried, checked at flush.
|
||||
const healthLogIdOf = new Map<PreparedMeasurement, string>();
|
||||
// A `<Correlation>` (a blood pressure reading) wraps its records, and the
|
||||
// marker may sit on the correlation itself, before or after them. Its records
|
||||
// are held until it closes, then all kept or all left out together.
|
||||
let currentCorrelation: {
|
||||
ownOrigin: boolean;
|
||||
held: PendingRecord[];
|
||||
} | null = null;
|
||||
// Spot-row batch awaiting flush.
|
||||
const spotBatch: PreparedMeasurement[] = [];
|
||||
// Workout-row batch awaiting flush.
|
||||
@@ -468,9 +536,134 @@ export async function streamParseExportXml(
|
||||
};
|
||||
|
||||
// ── Flush helpers ──────────────────────────────────────────
|
||||
// A sample the app wrote before it stamped its origin marker still carries
|
||||
// the HealthLog row id as `HKExternalUUID`. A hit on one of this account's
|
||||
// rows (deleted ones included: the sample is ours either way) of the same
|
||||
// kind is an exact match. Blood pressure is one row per half, while the app
|
||||
// stamps both samples of a reading with one id (the systolic row's, the
|
||||
// diastolic row's in older builds), so either half matches either id.
|
||||
const withoutHealthLogIds = async (
|
||||
rows: PreparedMeasurement[],
|
||||
): Promise<PreparedMeasurement[]> => {
|
||||
const ids = new Set<string>();
|
||||
for (const row of rows) {
|
||||
const id = healthLogIdOf.get(row);
|
||||
if (id) ids.add(id);
|
||||
}
|
||||
if (ids.size === 0) return rows;
|
||||
const known = await prisma.measurement.findMany({
|
||||
where: { userId, id: { in: [...ids] } },
|
||||
select: { id: true, type: true },
|
||||
});
|
||||
if (known.length === 0) return rows;
|
||||
const typeOf = new Map(known.map((row) => [row.id, row.type]));
|
||||
return rows.filter((row) => {
|
||||
const id = healthLogIdOf.get(row);
|
||||
const rowType = id ? typeOf.get(id) : undefined;
|
||||
if (!rowType || kindOf(rowType) !== kindOf(row.type)) return true;
|
||||
countOwn("byExternalId", row.externalId);
|
||||
return false;
|
||||
});
|
||||
};
|
||||
|
||||
// A manual entry mirrored with neither the marker nor an id comes back as a
|
||||
// second copy of the reading. Leave it out when a MANUAL row of the same
|
||||
// type and value sits within 2 s. Only MANUAL: Withings and import rows are
|
||||
// written into Apple Health by the server-to-Health mirror, which has always
|
||||
// stamped the row id, so a value-and-time match against them would only add
|
||||
// false positives. Same rule and tolerance as the sync path
|
||||
// (`cross-source-merge.ts`).
|
||||
//
|
||||
// The types that hold MANUAL rows are read once per import, so a batch of a
|
||||
// type the account never typed in asks nothing, and a batch that does asks
|
||||
// one time window per type rather than one clause per row.
|
||||
let manualTypes: Promise<Set<MeasurementType>> | null = null;
|
||||
const withoutManualMirrors = async (
|
||||
rows: PreparedMeasurement[],
|
||||
): Promise<PreparedMeasurement[]> => {
|
||||
if (rows.length === 0) return rows;
|
||||
manualTypes ??= prisma.measurement
|
||||
.groupBy({
|
||||
by: ["type"],
|
||||
where: { userId, source: "MANUAL", deletedAt: null },
|
||||
})
|
||||
.then((groups) => new Set(groups.map((group) => group.type)));
|
||||
const typed = await manualTypes;
|
||||
const windows = new Map<MeasurementType, { from: number; to: number }>();
|
||||
for (const row of rows) {
|
||||
if (!typed.has(row.type)) continue;
|
||||
const at = row.measuredAt.getTime();
|
||||
const window = windows.get(row.type);
|
||||
if (!window) windows.set(row.type, { from: at, to: at });
|
||||
else {
|
||||
if (at < window.from) window.from = at;
|
||||
if (at > window.to) window.to = at;
|
||||
}
|
||||
}
|
||||
if (windows.size === 0) return rows;
|
||||
const candidates = await prisma.measurement.findMany({
|
||||
where: {
|
||||
userId,
|
||||
source: "MANUAL",
|
||||
deletedAt: null,
|
||||
OR: [...windows].map(([type, window]) => ({
|
||||
type,
|
||||
measuredAt: {
|
||||
gte: new Date(window.from - MEASURED_AT_TOLERANCE_MS),
|
||||
lte: new Date(window.to + MEASURED_AT_TOLERANCE_MS),
|
||||
},
|
||||
})),
|
||||
},
|
||||
select: { type: true, source: true, value: true, measuredAt: true },
|
||||
orderBy: { measuredAt: "asc" },
|
||||
});
|
||||
if (candidates.length === 0) return rows;
|
||||
const byType = new Map<MeasurementType, typeof candidates>();
|
||||
for (const candidate of candidates) {
|
||||
const list = byType.get(candidate.type);
|
||||
if (list) list.push(candidate);
|
||||
else byType.set(candidate.type, [candidate]);
|
||||
}
|
||||
return rows.filter((row) => {
|
||||
const list = byType.get(row.type);
|
||||
if (!list) return true;
|
||||
const at = row.measuredAt.getTime();
|
||||
// First candidate not earlier than the tolerance window.
|
||||
let lo = 0;
|
||||
let hi = list.length;
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
if (list[mid].measuredAt.getTime() < at - MEASURED_AT_TOLERANCE_MS)
|
||||
lo = mid + 1;
|
||||
else hi = mid;
|
||||
}
|
||||
for (let i = lo; i < list.length; i++) {
|
||||
if (list[i].measuredAt.getTime() > at + MEASURED_AT_TOLERANCE_MS) break;
|
||||
const mirrored = isSameReadingAcrossSource(
|
||||
{
|
||||
type: row.type,
|
||||
source: "APPLE_HEALTH",
|
||||
value: row.value,
|
||||
measuredAt: row.measuredAt,
|
||||
},
|
||||
list[i],
|
||||
);
|
||||
if (mirrored) {
|
||||
countOwn("matchedManual", row.externalId);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
});
|
||||
};
|
||||
|
||||
const flushSpotBatch = async (): Promise<void> => {
|
||||
if (spotBatch.length === 0) return;
|
||||
const chunk = spotBatch.splice(0, spotBatch.length);
|
||||
const incoming = spotBatch.splice(0, spotBatch.length);
|
||||
const chunk = await withoutManualMirrors(
|
||||
await withoutHealthLogIds(incoming),
|
||||
);
|
||||
for (const row of incoming) healthLogIdOf.delete(row);
|
||||
for (const row of chunk) widenSpan(row.measuredAt);
|
||||
const insertedArrivals: Array<{
|
||||
id: string;
|
||||
@@ -847,7 +1040,18 @@ export async function streamParseExportXml(
|
||||
Date.parse(attrs.endDate ?? attrs.startDate ?? ""),
|
||||
)
|
||||
? null
|
||||
: { hkType, dayKey, rawValue: attrs.value };
|
||||
: {
|
||||
hkType,
|
||||
dayKey,
|
||||
rawValue: attrs.value,
|
||||
sampleKey: hashSampleKey(
|
||||
hkType,
|
||||
attrs.value ?? "",
|
||||
attrs.startDate,
|
||||
attrs.endDate,
|
||||
),
|
||||
ownOrigin: false,
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -907,46 +1111,61 @@ export async function streamParseExportXml(
|
||||
const stat = bumpStat(mapped.type);
|
||||
stat.read += 1;
|
||||
|
||||
if (CUMULATIVE_HK_TYPES.has(mapped.type)) {
|
||||
const dayKey = dayKeyForUserTz(mapped.takenAt, userTimezone);
|
||||
const sourceHash = hashCumulativeSourceIdentity(
|
||||
attrs.sourceName,
|
||||
attrs.device,
|
||||
);
|
||||
let byDay = cumulativeBucket.get(mapped.type);
|
||||
if (!byDay) {
|
||||
byDay = new Map();
|
||||
cumulativeBucket.set(mapped.type, byDay);
|
||||
}
|
||||
let bySource = byDay.get(dayKey);
|
||||
if (!bySource) {
|
||||
bySource = new Map();
|
||||
byDay.set(dayKey, bySource);
|
||||
}
|
||||
bySource.set(
|
||||
sourceHash,
|
||||
(bySource.get(sourceHash) ?? 0) + mapped.value,
|
||||
);
|
||||
} else {
|
||||
// Spot row: derive a stable externalId, queue for flush.
|
||||
const externalId = hashSampleKey(
|
||||
hkType,
|
||||
attrs.value ?? "",
|
||||
attrs.startDate,
|
||||
attrs.endDate,
|
||||
);
|
||||
spotBatch.push({
|
||||
userId,
|
||||
type: mapped.type,
|
||||
value: mapped.value,
|
||||
unit: mapped.unit,
|
||||
measuredAt: mapped.takenAt,
|
||||
externalId,
|
||||
externalSourceVersion: attrs.sourceVersion ?? null,
|
||||
sleepStage: mapped.sleepStage ?? null,
|
||||
deviceType: null,
|
||||
});
|
||||
}
|
||||
pendingRecord = {
|
||||
ownOrigin: false,
|
||||
healthLogId: null,
|
||||
sampleKey: () =>
|
||||
hashSampleKey(
|
||||
hkType,
|
||||
attrs.value ?? "",
|
||||
attrs.startDate,
|
||||
attrs.endDate,
|
||||
),
|
||||
commit: (healthLogId) => {
|
||||
if (CUMULATIVE_HK_TYPES.has(mapped.type)) {
|
||||
const dayKey = dayKeyForUserTz(mapped.takenAt, userTimezone);
|
||||
const sourceHash = hashCumulativeSourceIdentity(
|
||||
attrs.sourceName,
|
||||
attrs.device,
|
||||
);
|
||||
let byDay = cumulativeBucket.get(mapped.type);
|
||||
if (!byDay) {
|
||||
byDay = new Map();
|
||||
cumulativeBucket.set(mapped.type, byDay);
|
||||
}
|
||||
let bySource = byDay.get(dayKey);
|
||||
if (!bySource) {
|
||||
bySource = new Map();
|
||||
byDay.set(dayKey, bySource);
|
||||
}
|
||||
bySource.set(
|
||||
sourceHash,
|
||||
(bySource.get(sourceHash) ?? 0) + mapped.value,
|
||||
);
|
||||
} else {
|
||||
// Spot row: derive a stable externalId, queue for flush.
|
||||
const externalId = hashSampleKey(
|
||||
hkType,
|
||||
attrs.value ?? "",
|
||||
attrs.startDate,
|
||||
attrs.endDate,
|
||||
);
|
||||
const row: PreparedMeasurement = {
|
||||
userId,
|
||||
type: mapped.type,
|
||||
value: mapped.value,
|
||||
unit: mapped.unit,
|
||||
measuredAt: mapped.takenAt,
|
||||
externalId,
|
||||
externalSourceVersion: attrs.sourceVersion ?? null,
|
||||
sleepStage: mapped.sleepStage ?? null,
|
||||
deviceType: null,
|
||||
};
|
||||
spotBatch.push(row);
|
||||
if (healthLogId) healthLogIdOf.set(row, healthLogId);
|
||||
}
|
||||
},
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1024,7 +1243,21 @@ export async function streamParseExportXml(
|
||||
return;
|
||||
}
|
||||
|
||||
if (name === "Correlation") {
|
||||
currentCorrelation = { ownOrigin: false, held: [] };
|
||||
return;
|
||||
}
|
||||
|
||||
if (name === "MetadataEntry") {
|
||||
if (isHealthLogOriginEntry(attrs.key, attrs.value)) {
|
||||
// On a record, or on the correlation that wraps it.
|
||||
if (pendingRecord) pendingRecord.ownOrigin = true;
|
||||
else if (currentCycleRecord) currentCycleRecord.ownOrigin = true;
|
||||
else if (currentCorrelation) currentCorrelation.ownOrigin = true;
|
||||
}
|
||||
if (pendingRecord && attrs.key === HK_EXTERNAL_UUID_METADATA_KEY) {
|
||||
pendingRecord.healthLogId = attrs.value || null;
|
||||
}
|
||||
// Attach the SexualActivity protection flag to the open cycle record.
|
||||
// Apple writes `HKMetadataKeySexualActivityProtectionUsed` with a
|
||||
// `"0"`/`"1"` (or `"true"`/`"false"`) value.
|
||||
@@ -1051,16 +1284,15 @@ export async function streamParseExportXml(
|
||||
|
||||
if (
|
||||
name === "Me" ||
|
||||
name === "Correlation" ||
|
||||
name === "ActivitySummary" ||
|
||||
name === "WorkoutEvent" ||
|
||||
name === "WorkoutRoute" ||
|
||||
name === "FileReference" ||
|
||||
name === "HealthData"
|
||||
) {
|
||||
// Known elements we intentionally ignore at the open-tag stage.
|
||||
// Correlation envelopes flatten naturally because their child
|
||||
// `<Record>` elements still fire their own `onopentag`.
|
||||
// Known elements we intentionally ignore at the open-tag stage. A
|
||||
// `<Correlation>` is handled above: its child `<Record>` elements fire
|
||||
// their own `onopentag`, and are held until it closes.
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1070,6 +1302,24 @@ export async function streamParseExportXml(
|
||||
};
|
||||
|
||||
parser.onclosetag = async (tagName) => {
|
||||
if (tagName === "Record" && pendingRecord) {
|
||||
const rec = pendingRecord;
|
||||
pendingRecord = null;
|
||||
if (currentCorrelation) currentCorrelation.held.push(rec);
|
||||
else if (rec.ownOrigin) countOwn("byMarker", rec.sampleKey());
|
||||
else rec.commit(rec.healthLogId);
|
||||
}
|
||||
if (tagName === "Correlation" && currentCorrelation) {
|
||||
const correlation = currentCorrelation;
|
||||
currentCorrelation = null;
|
||||
for (const rec of correlation.held) {
|
||||
if (correlation.ownOrigin || rec.ownOrigin) {
|
||||
countOwn("byMarker", rec.sampleKey());
|
||||
} else {
|
||||
rec.commit(rec.healthLogId);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (tagName === "Workout" && currentWorkout) {
|
||||
workoutBatch.push(currentWorkout);
|
||||
currentWorkout = null;
|
||||
@@ -1077,6 +1327,11 @@ export async function streamParseExportXml(
|
||||
if (tagName === "Record" && currentCycleRecord) {
|
||||
const rec = currentCycleRecord;
|
||||
currentCycleRecord = null;
|
||||
if (rec.ownOrigin) {
|
||||
// A day HealthLog mirrored into Apple Health from its own day-log.
|
||||
countOwn("byMarker", rec.sampleKey);
|
||||
return;
|
||||
}
|
||||
const consumed = cycleAccumulator.consume(
|
||||
rec.hkType,
|
||||
rec.dayKey,
|
||||
@@ -1212,6 +1467,7 @@ export async function streamParseExportXml(
|
||||
perType,
|
||||
workouts,
|
||||
clinical,
|
||||
writtenByHealthLog,
|
||||
cycle,
|
||||
deferred,
|
||||
unknown,
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
SYNTHETIC FIXTURE. Not an export from a real person or device.
|
||||
|
||||
Written by hand to the shape Apple's export.xml documents in its own DTD
|
||||
(abridged below): a <Record> may hold <MetadataEntry key= value=> children.
|
||||
Every value, date and id is invented. What it cannot show is whether Apple's
|
||||
exporter writes a custom metadata key such as dev.healthlog.app.origin
|
||||
verbatim; that has to be checked against a real export (see #1109).
|
||||
|
||||
Records, in order:
|
||||
1. weight 80.1 carries the HealthLog origin marker -> left out
|
||||
2. weight 80.2 user-entered, no marker -> imported
|
||||
3. weight 80.3 marker among other metadata entries -> left out
|
||||
4. weight 80.4 no metadata at all -> imported
|
||||
5. weight 80.5 origin key with another value -> imported
|
||||
-->
|
||||
<!DOCTYPE HealthData [
|
||||
<!ELEMENT HealthData (ExportDate, Me, Record*)>
|
||||
<!ATTLIST HealthData locale CDATA #REQUIRED>
|
||||
<!ELEMENT ExportDate EMPTY>
|
||||
<!ATTLIST ExportDate value CDATA #REQUIRED>
|
||||
<!ELEMENT Me EMPTY>
|
||||
<!ELEMENT Record (MetadataEntry*)>
|
||||
<!ATTLIST Record
|
||||
type CDATA #REQUIRED
|
||||
unit CDATA #IMPLIED
|
||||
value CDATA #IMPLIED
|
||||
sourceName CDATA #REQUIRED
|
||||
sourceVersion CDATA #IMPLIED
|
||||
device CDATA #IMPLIED
|
||||
creationDate CDATA #IMPLIED
|
||||
startDate CDATA #REQUIRED
|
||||
endDate CDATA #REQUIRED>
|
||||
<!ELEMENT MetadataEntry EMPTY>
|
||||
<!ATTLIST MetadataEntry key CDATA #REQUIRED value CDATA #REQUIRED>
|
||||
]>
|
||||
<HealthData locale="en_US">
|
||||
<ExportDate value="2026-05-20 12:00:00 +0200"/>
|
||||
<Me/>
|
||||
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 08:00:02 +0200" startDate="2026-05-14 08:00:00 +0200" endDate="2026-05-14 08:00:00 +0200" value="80.1">
|
||||
<MetadataEntry key="dev.healthlog.app.origin" value="healthlog"/>
|
||||
<MetadataEntry key="HKExternalUUID" value="00000000-0000-0000-0000-000000000001"/>
|
||||
</Record>
|
||||
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 09:00:02 +0200" startDate="2026-05-14 09:00:00 +0200" endDate="2026-05-14 09:00:00 +0200" value="80.2">
|
||||
<MetadataEntry key="HKWasUserEntered" value="1"/>
|
||||
</Record>
|
||||
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 10:00:02 +0200" startDate="2026-05-14 10:00:00 +0200" endDate="2026-05-14 10:00:00 +0200" value="80.3">
|
||||
<MetadataEntry key="HKWasUserEntered" value="1"/>
|
||||
<MetadataEntry key="dev.healthlog.app.origin" value="healthlog"/>
|
||||
</Record>
|
||||
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 11:00:02 +0200" startDate="2026-05-14 11:00:00 +0200" endDate="2026-05-14 11:00:00 +0200" value="80.4"/>
|
||||
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 12:00:02 +0200" startDate="2026-05-14 12:00:00 +0200" endDate="2026-05-14 12:00:00 +0200" value="80.5">
|
||||
<MetadataEntry key="dev.healthlog.app.origin" value="some-other-app"/>
|
||||
</Record>
|
||||
</HealthData>
|
||||
@@ -0,0 +1,572 @@
|
||||
/**
|
||||
* The Apple Health export must not bring back what HealthLog wrote into Apple
|
||||
* Health itself.
|
||||
*
|
||||
* The iOS app writes manual entries into Apple Health, and mirrors Withings and
|
||||
* import rows into it, each stamped with `dev.healthlog.app.origin = healthlog`.
|
||||
* The server already holds those readings under their own source, so the same
|
||||
* sample arriving through an export is a second copy: the series shows it
|
||||
* twice and the rollups count it twice.
|
||||
*
|
||||
* Asserted against a real Postgres:
|
||||
* - a record carrying the marker is left out, and counted;
|
||||
* - a record without it lands as before;
|
||||
* - the marker is read from a child `<MetadataEntry>` that follows the
|
||||
* record's open tag, so the record is committed at its close tag;
|
||||
* - a sample from before the marker carries the HealthLog row id as
|
||||
* `HKExternalUUID`; a hit on one of the account's rows of the same kind
|
||||
* leaves it out, either id of a blood pressure pair covering both halves;
|
||||
* - a manual entry mirrored with neither (a MANUAL row with the same type and
|
||||
* value within 2 s) is left out and counted separately;
|
||||
* - a MANUAL row with another value, or 3 s away, is not a match;
|
||||
* - a Withings row of the same value and time is not a match either (those
|
||||
* are mirrored with their row id, so a match would only add false
|
||||
* positives);
|
||||
* - the value match asks nothing for a type the account has no MANUAL rows
|
||||
* of, and each sample left out is counted once.
|
||||
*/
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import { streamParseExportXml } from "@/lib/measurements/import-apple-health-export";
|
||||
|
||||
const prisma = getPrismaClient();
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(prisma);
|
||||
});
|
||||
|
||||
function writeXml(records: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "healthlog-import-own-"));
|
||||
const xmlPath = join(dir, "export.xml");
|
||||
writeFileSync(
|
||||
xmlPath,
|
||||
`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
|
||||
<HealthData locale="en_US">
|
||||
${records}
|
||||
</HealthData>`,
|
||||
);
|
||||
return xmlPath;
|
||||
}
|
||||
|
||||
const weight = (value: number, end: string, children = "") =>
|
||||
` <Record type="HKQuantityTypeIdentifierBodyMass" sourceName="Health" unit="kg" startDate="${end}" endDate="${end}" value="${value}">${children}</Record>`;
|
||||
const OWN = `<MetadataEntry key="dev.healthlog.app.origin" value="healthlog"/>`;
|
||||
const OTHER = `<MetadataEntry key="HKWasUserEntered" value="1"/>`;
|
||||
|
||||
async function run(records: string) {
|
||||
const user = await prisma.user.create({
|
||||
data: { username: "own-samples", email: "own@example.test", role: "USER" },
|
||||
});
|
||||
const result = await streamParseExportXml({
|
||||
xmlPath: writeXml(records),
|
||||
userId: user.id,
|
||||
userTimezone: "Europe/Berlin",
|
||||
prisma,
|
||||
});
|
||||
const rows = await prisma.measurement.findMany({
|
||||
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
|
||||
orderBy: { measuredAt: "asc" },
|
||||
});
|
||||
return { user, result, rows };
|
||||
}
|
||||
|
||||
describe("a record HealthLog wrote itself (the origin marker)", () => {
|
||||
it("is left out and counted; an ordinary record still lands", async () => {
|
||||
const { result, rows } = await run(
|
||||
[
|
||||
weight(80.1, "2026-05-14 08:00:00 +0200", `\n ${OWN}\n `),
|
||||
weight(80.2, "2026-05-14 09:00:00 +0200", `\n ${OTHER}\n `),
|
||||
weight(80.3, "2026-05-14 10:00:00 +0200"),
|
||||
].join("\n"),
|
||||
);
|
||||
expect(rows.map((r) => r.value)).toEqual([80.2, 80.3]);
|
||||
expect(result.writtenByHealthLog).toEqual({
|
||||
byMarker: 1,
|
||||
byExternalId: 0,
|
||||
matchedManual: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("finds the marker among other metadata entries", async () => {
|
||||
const { result, rows } = await run(
|
||||
weight(80.1, "2026-05-14 08:00:00 +0200", `\n ${OTHER}\n ${OWN}\n`),
|
||||
);
|
||||
expect(rows).toHaveLength(0);
|
||||
expect(result.writtenByHealthLog.byMarker).toBe(1);
|
||||
});
|
||||
|
||||
it("does not let one record's marker spill onto the next", async () => {
|
||||
const { rows } = await run(
|
||||
[
|
||||
weight(80.1, "2026-05-14 08:00:00 +0200", `\n ${OWN}\n`),
|
||||
weight(80.2, "2026-05-14 09:00:00 +0200"),
|
||||
].join("\n"),
|
||||
);
|
||||
expect(rows.map((r) => r.value)).toEqual([80.2]);
|
||||
});
|
||||
|
||||
it("needs the exact value: another origin does not count", async () => {
|
||||
const { rows } = await run(
|
||||
weight(
|
||||
80.1,
|
||||
"2026-05-14 08:00:00 +0200",
|
||||
`<MetadataEntry key="dev.healthlog.app.origin" value="other"/>`,
|
||||
),
|
||||
);
|
||||
expect(rows).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
const bpRecord = (
|
||||
type: "Systolic" | "Diastolic",
|
||||
value: number,
|
||||
at: string,
|
||||
children = "",
|
||||
) =>
|
||||
` <Record type="HKQuantityTypeIdentifierBloodPressure${type}" sourceName="Health" unit="mmHg" startDate="${at}" endDate="${at}" value="${value}">${children}</Record>`;
|
||||
const correlation = (inner: string, children = "") =>
|
||||
` <Correlation type="HKCorrelationTypeIdentifierBloodPressure" sourceName="Health" startDate="2026-05-14 08:00:00 +0200" endDate="2026-05-14 08:00:00 +0200">${children}\n${inner}\n </Correlation>`;
|
||||
|
||||
describe("blood pressure correlations and self-closing records", () => {
|
||||
const AT = "2026-05-14 08:00:00 +0200";
|
||||
async function runBp(records: string) {
|
||||
const user = await prisma.user.create({
|
||||
data: { username: "own-bp", email: "bp@example.test", role: "USER" },
|
||||
});
|
||||
const result = await streamParseExportXml({
|
||||
xmlPath: writeXml(records),
|
||||
userId: user.id,
|
||||
userTimezone: "Europe/Berlin",
|
||||
prisma,
|
||||
});
|
||||
const rows = await prisma.measurement.findMany({
|
||||
where: {
|
||||
userId: user.id,
|
||||
type: { in: ["BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA"] },
|
||||
},
|
||||
orderBy: [{ type: "asc" }],
|
||||
});
|
||||
return { result, values: rows.map((r) => r.value) };
|
||||
}
|
||||
|
||||
it("imports an unmarked correlation's two records", async () => {
|
||||
const { values, result } = await runBp(
|
||||
correlation(
|
||||
[bpRecord("Systolic", 121, AT), bpRecord("Diastolic", 79, AT)].join(
|
||||
"\n",
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(values).toEqual([121, 79]);
|
||||
expect(result.writtenByHealthLog.byMarker).toBe(0);
|
||||
});
|
||||
|
||||
it("leaves out both records when the marker is on the correlation, before them", async () => {
|
||||
const { values, result } = await runBp(
|
||||
correlation(
|
||||
[bpRecord("Systolic", 121, AT), bpRecord("Diastolic", 79, AT)].join(
|
||||
"\n",
|
||||
),
|
||||
`\n ${OWN}`,
|
||||
),
|
||||
);
|
||||
expect(values).toEqual([]);
|
||||
expect(result.writtenByHealthLog.byMarker).toBe(2);
|
||||
});
|
||||
|
||||
it("leaves out both records when the marker is on the correlation, after them", async () => {
|
||||
const records = [
|
||||
bpRecord("Systolic", 121, AT),
|
||||
bpRecord("Diastolic", 79, AT),
|
||||
].join("\n");
|
||||
const { values } = await runBp(
|
||||
` <Correlation type="HKCorrelationTypeIdentifierBloodPressure" sourceName="Health" startDate="${AT}" endDate="${AT}">\n${records}\n ${OWN}\n </Correlation>`,
|
||||
);
|
||||
expect(values).toEqual([]);
|
||||
});
|
||||
|
||||
it("leaves out only the record that carries the marker", async () => {
|
||||
const { values } = await runBp(
|
||||
correlation(
|
||||
[
|
||||
bpRecord("Systolic", 121, AT, `\n ${OWN}\n `),
|
||||
bpRecord("Diastolic", 79, AT),
|
||||
].join("\n"),
|
||||
),
|
||||
);
|
||||
expect(values).toEqual([79]);
|
||||
});
|
||||
|
||||
it("does not let a correlation's marker spill onto the next record", async () => {
|
||||
const { values } = await runBp(
|
||||
[
|
||||
correlation(
|
||||
[bpRecord("Systolic", 121, AT), bpRecord("Diastolic", 79, AT)].join(
|
||||
"\n",
|
||||
),
|
||||
`\n ${OWN}`,
|
||||
),
|
||||
bpRecord("Systolic", 130, "2026-05-14 09:00:00 +0200"),
|
||||
].join("\n"),
|
||||
);
|
||||
expect(values).toEqual([130]);
|
||||
});
|
||||
|
||||
it("imports a self-closing record, with no children to wait for", async () => {
|
||||
const { rows } = await run(
|
||||
` <Record type="HKQuantityTypeIdentifierBodyMass" sourceName="Health" unit="kg" startDate="${AT}" endDate="${AT}" value="80.9"/>`,
|
||||
);
|
||||
expect(rows.map((r) => r.value)).toEqual([80.9]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the synthetic export fixture", () => {
|
||||
// `export-own-origin.synthetic.xml` is invented, in the shape Apple's DTD
|
||||
// documents. It shows the parser reads a file laid out like an export; it
|
||||
// cannot show that Apple writes a custom metadata key verbatim.
|
||||
it("leaves out the two marked records and imports the other three", async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { username: "own-fixture", email: "fx@example.test", role: "USER" },
|
||||
});
|
||||
const result = await streamParseExportXml({
|
||||
xmlPath: join(
|
||||
process.cwd(),
|
||||
"tests/fixtures/apple-health/export-own-origin.synthetic.xml",
|
||||
),
|
||||
userId: user.id,
|
||||
userTimezone: "Europe/Berlin",
|
||||
prisma,
|
||||
});
|
||||
const rows = await prisma.measurement.findMany({
|
||||
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
|
||||
orderBy: { measuredAt: "asc" },
|
||||
});
|
||||
expect(rows.map((r) => r.value)).toEqual([80.2, 80.4, 80.5]);
|
||||
expect(result.writtenByHealthLog).toEqual({
|
||||
byMarker: 2,
|
||||
byExternalId: 0,
|
||||
matchedManual: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("a manual entry mirrored before the marker existed", () => {
|
||||
async function runAgainst(
|
||||
manual: { value: number; at: string; source?: "MANUAL" | "WITHINGS" },
|
||||
record: { value: number; at: string },
|
||||
) {
|
||||
const user = await prisma.user.create({
|
||||
data: { username: "own-fallback", email: "f@example.test", role: "USER" },
|
||||
});
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId: user.id,
|
||||
type: "WEIGHT",
|
||||
unit: "kg",
|
||||
source: manual.source ?? "MANUAL",
|
||||
value: manual.value,
|
||||
measuredAt: new Date(manual.at),
|
||||
},
|
||||
});
|
||||
const result = await streamParseExportXml({
|
||||
xmlPath: writeXml(weight(record.value, "2026-05-14 08:00:00 +0200")),
|
||||
userId: user.id,
|
||||
userTimezone: "Europe/Berlin",
|
||||
prisma,
|
||||
});
|
||||
const rows = await prisma.measurement.findMany({
|
||||
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
|
||||
});
|
||||
return { result, rows };
|
||||
}
|
||||
|
||||
it("is left out when a MANUAL row has the same value within 2 s", async () => {
|
||||
const { result, rows } = await runAgainst(
|
||||
{ value: 80.4, at: "2026-05-14T06:00:01.500Z" },
|
||||
{ value: 80.4, at: "" },
|
||||
);
|
||||
expect(rows).toHaveLength(0);
|
||||
expect(result.writtenByHealthLog).toEqual({
|
||||
byMarker: 0,
|
||||
byExternalId: 0,
|
||||
matchedManual: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("is kept when the value differs", async () => {
|
||||
const { result, rows } = await runAgainst(
|
||||
{ value: 80.9, at: "2026-05-14T06:00:00.000Z" },
|
||||
{ value: 80.4, at: "" },
|
||||
);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(result.writtenByHealthLog.matchedManual).toBe(0);
|
||||
});
|
||||
|
||||
it("is kept when the MANUAL row is 3 s away", async () => {
|
||||
const { rows } = await runAgainst(
|
||||
{ value: 80.4, at: "2026-05-14T06:00:03.000Z" },
|
||||
{ value: 80.4, at: "" },
|
||||
);
|
||||
expect(rows).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("is kept against a Withings row of the same value and time", async () => {
|
||||
const { result, rows } = await runAgainst(
|
||||
{ value: 80.4, at: "2026-05-14T06:00:00.000Z", source: "WITHINGS" },
|
||||
{ value: 80.4, at: "" },
|
||||
);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(result.writtenByHealthLog.matchedManual).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Count every `measurement.findMany` the import issues against MANUAL rows,
|
||||
* by wrapping the client the import is handed.
|
||||
*/
|
||||
function countingManualQueries() {
|
||||
const manualQueries: Array<{ where?: Record<string, unknown> }> = [];
|
||||
const measurement = new Proxy(prisma.measurement, {
|
||||
get(target, prop, receiver) {
|
||||
const value = Reflect.get(target, prop, receiver);
|
||||
if (prop !== "findMany" || typeof value !== "function") return value;
|
||||
return (args: { where?: Record<string, unknown> }) => {
|
||||
if (args?.where?.source === "MANUAL") manualQueries.push(args);
|
||||
return value.call(target, args);
|
||||
};
|
||||
},
|
||||
});
|
||||
const client = new Proxy(prisma, {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "measurement") return measurement;
|
||||
const value = Reflect.get(target, prop, receiver);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
}) as typeof prisma;
|
||||
return { client, manualQueries };
|
||||
}
|
||||
|
||||
describe("the manual-mirror lookup stays off the hot path", () => {
|
||||
const AT = (i: number) =>
|
||||
`2026-05-14 ${String(8 + Math.floor(i / 60)).padStart(2, "0")}:${String(i % 60).padStart(2, "0")}:00 +0200`;
|
||||
const heartRate = (value: number, at: string) =>
|
||||
` <Record type="HKQuantityTypeIdentifierHeartRate" sourceName="Health" unit="count/min" startDate="${at}" endDate="${at}" value="${value}"/>`;
|
||||
|
||||
async function runCounting(records: string, manualType?: "WEIGHT") {
|
||||
const user = await prisma.user.create({
|
||||
data: { username: "own-hot", email: "hot@example.test", role: "USER" },
|
||||
});
|
||||
if (manualType) {
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId: user.id,
|
||||
type: manualType,
|
||||
unit: "kg",
|
||||
source: "MANUAL",
|
||||
value: 80.4,
|
||||
measuredAt: new Date("2026-05-14T06:00:00.000Z"),
|
||||
},
|
||||
});
|
||||
}
|
||||
const { client, manualQueries } = countingManualQueries();
|
||||
const result = await streamParseExportXml({
|
||||
xmlPath: writeXml(records),
|
||||
userId: user.id,
|
||||
userTimezone: "Europe/Berlin",
|
||||
prisma: client,
|
||||
spotBatchSize: 10,
|
||||
});
|
||||
return { result, manualQueries, user };
|
||||
}
|
||||
|
||||
it("asks nothing per batch when the account has no MANUAL rows", async () => {
|
||||
const records = Array.from({ length: 40 }, (_, i) =>
|
||||
heartRate(60 + (i % 20), AT(i)),
|
||||
).join("\n");
|
||||
const { manualQueries, result } = await runCounting(records);
|
||||
expect(manualQueries).toHaveLength(0);
|
||||
expect(result.writtenByHealthLog.matchedManual).toBe(0);
|
||||
});
|
||||
|
||||
it("asks nothing for a type without MANUAL rows, and one bounded window per batch for one with them", async () => {
|
||||
const records = [
|
||||
...Array.from({ length: 40 }, (_, i) => heartRate(60 + (i % 20), AT(i))),
|
||||
weight(80.4, "2026-05-14 08:00:00 +0200"),
|
||||
weight(80.6, "2026-05-14 09:00:00 +0200"),
|
||||
].join("\n");
|
||||
const { manualQueries, result, user } = await runCounting(
|
||||
records,
|
||||
"WEIGHT",
|
||||
);
|
||||
// The heart rates ask nothing; the weights ask once, with one window for
|
||||
// their type, not one clause per row.
|
||||
expect(manualQueries).toHaveLength(1);
|
||||
const or = manualQueries[0].where?.OR as Array<{ type: string }>;
|
||||
expect(or).toHaveLength(1);
|
||||
expect(or[0].type).toBe("WEIGHT");
|
||||
// The mirror is still caught; the other weight still lands.
|
||||
expect(result.writtenByHealthLog.matchedManual).toBe(1);
|
||||
const weights = await prisma.measurement.findMany({
|
||||
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
|
||||
});
|
||||
expect(weights.map((w) => w.value)).toEqual([80.6]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a sample whose HKExternalUUID is a HealthLog row id", () => {
|
||||
const AT = "2026-05-14 08:00:00 +0200";
|
||||
const ext = (id: string) =>
|
||||
`<MetadataEntry key="HKExternalUUID" value="${id}"/>`;
|
||||
|
||||
async function runWith(
|
||||
seed: (userId: string) => Promise<Record<string, string>>,
|
||||
records: (ids: Record<string, string>) => string,
|
||||
) {
|
||||
const user = await prisma.user.create({
|
||||
data: { username: "own-ext", email: "ext@example.test", role: "USER" },
|
||||
});
|
||||
const ids = await seed(user.id);
|
||||
const result = await streamParseExportXml({
|
||||
xmlPath: writeXml(records(ids)),
|
||||
userId: user.id,
|
||||
userTimezone: "Europe/Berlin",
|
||||
prisma,
|
||||
});
|
||||
const rows = await prisma.measurement.findMany({
|
||||
where: { userId: user.id, source: "APPLE_HEALTH" },
|
||||
orderBy: [{ type: "asc" }],
|
||||
});
|
||||
return { result, rows };
|
||||
}
|
||||
|
||||
const seedRow = async (
|
||||
userId: string,
|
||||
type: "WEIGHT" | "BLOOD_PRESSURE_SYS" | "BLOOD_PRESSURE_DIA",
|
||||
value: number,
|
||||
source: "MANUAL" | "WITHINGS" = "MANUAL",
|
||||
) =>
|
||||
(
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId,
|
||||
type,
|
||||
value,
|
||||
unit: type === "WEIGHT" ? "kg" : "mmHg",
|
||||
source,
|
||||
// Ten minutes off the sample: only the id can tie them together.
|
||||
measuredAt: new Date("2026-05-14T06:10:00.000Z"),
|
||||
},
|
||||
})
|
||||
).id;
|
||||
|
||||
it("is left out without the marker, for a Withings row the value match never covers", async () => {
|
||||
const { result, rows } = await runWith(
|
||||
async (userId) => ({
|
||||
w: await seedRow(userId, "WEIGHT", 81, "WITHINGS"),
|
||||
}),
|
||||
(ids) => weight(80.4, AT, ext(ids.w)),
|
||||
);
|
||||
expect(rows).toHaveLength(0);
|
||||
expect(result.writtenByHealthLog).toEqual({
|
||||
byMarker: 0,
|
||||
byExternalId: 1,
|
||||
matchedManual: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves out both halves of a pressure pair stamped with either half's id", async () => {
|
||||
const { result, rows } = await runWith(
|
||||
async (userId) => ({
|
||||
sys: await seedRow(userId, "BLOOD_PRESSURE_SYS", 121),
|
||||
dia: await seedRow(userId, "BLOOD_PRESSURE_DIA", 79),
|
||||
}),
|
||||
(ids) =>
|
||||
[
|
||||
// Current builds stamp both samples with the systolic id ...
|
||||
bpRecord("Systolic", 121, AT, ext(ids.sys)),
|
||||
bpRecord("Diastolic", 79, AT, ext(ids.sys)),
|
||||
// ... older builds stamped both with the diastolic id.
|
||||
bpRecord("Systolic", 122, "2026-05-14 09:00:00 +0200", ext(ids.dia)),
|
||||
bpRecord("Diastolic", 80, "2026-05-14 09:00:00 +0200", ext(ids.dia)),
|
||||
].join("\n"),
|
||||
);
|
||||
expect(rows).toHaveLength(0);
|
||||
expect(result.writtenByHealthLog.byExternalId).toBe(4);
|
||||
});
|
||||
|
||||
it("is imported when the id is unknown, belongs to another account, or names another kind", async () => {
|
||||
const other = await prisma.user.create({
|
||||
data: { username: "own-ext-2", email: "ext2@example.test", role: "USER" },
|
||||
});
|
||||
const foreignId = await seedRow(other.id, "WEIGHT", 80.4);
|
||||
const { result, rows } = await runWith(
|
||||
async (userId) => ({
|
||||
sys: await seedRow(userId, "BLOOD_PRESSURE_SYS", 121),
|
||||
}),
|
||||
(ids) =>
|
||||
[
|
||||
weight(80.1, AT, ext("00000000-0000-4000-8000-000000000001")),
|
||||
weight(80.2, "2026-05-14 09:00:00 +0200", ext(foreignId)),
|
||||
weight(80.3, "2026-05-14 10:00:00 +0200", ext(ids.sys)),
|
||||
].join("\n"),
|
||||
);
|
||||
expect(rows.map((r) => r.value)).toEqual([80.1, 80.2, 80.3]);
|
||||
expect(result.writtenByHealthLog.byExternalId).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("each sample left out is counted once", () => {
|
||||
it("does not count a marked record twice when it appears top level and inside its correlation", async () => {
|
||||
const AT = "2026-05-14 08:00:00 +0200";
|
||||
const sys = bpRecord("Systolic", 121, AT, `\n ${OWN}\n `);
|
||||
const dia = bpRecord("Diastolic", 79, AT, `\n ${OWN}\n `);
|
||||
const user = await prisma.user.create({
|
||||
data: { username: "own-once", email: "once@example.test", role: "USER" },
|
||||
});
|
||||
const result = await streamParseExportXml({
|
||||
xmlPath: writeXml(
|
||||
[sys, dia, correlation([sys, dia].join("\n"))].join("\n"),
|
||||
),
|
||||
userId: user.id,
|
||||
userTimezone: "Europe/Berlin",
|
||||
prisma,
|
||||
});
|
||||
expect(result.writtenByHealthLog.byMarker).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a cycle sample HealthLog wrote itself", () => {
|
||||
it("is left out of the day-log import", async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: "own-cycle",
|
||||
email: "cycle@example.test",
|
||||
role: "USER",
|
||||
gender: "FEMALE",
|
||||
timezone: "Europe/Berlin",
|
||||
},
|
||||
});
|
||||
const flow = (day: string, children = "") =>
|
||||
` <Record type="HKCategoryTypeIdentifierMenstrualFlow" sourceName="Health" value="HKCategoryValueMenstrualFlowMedium" startDate="${day} 08:00:00 +0000" endDate="${day} 08:00:00 +0000">${children}</Record>`;
|
||||
const result = await streamParseExportXml({
|
||||
xmlPath: writeXml(
|
||||
[flow("2026-03-02", OWN), flow("2026-03-03")].join("\n"),
|
||||
),
|
||||
userId: user.id,
|
||||
userTimezone: "Europe/Berlin",
|
||||
prisma,
|
||||
});
|
||||
const days = await prisma.cycleDayLog.findMany({
|
||||
where: { userId: user.id },
|
||||
orderBy: { date: "asc" },
|
||||
});
|
||||
expect(days.map((d) => d.date)).toEqual(["2026-03-03"]);
|
||||
expect(result.writtenByHealthLog.byMarker).toBe(1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user