mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 15:42:24 +00:00
buildFullBackupPayload now asks every domain for its slice at once and lays the slices out in the file's key order (backup-export-sections.ts), merging the manifest where it first appears. The measurement, medication, mood, instance-settings and nutrient readers and serialisers move out of the payload builder into their domain modules; the records section keeps its composition and manifest, while labs, illness, allergies, family history, workouts and documents each export from their own module. The measurement restore half moves to measurements-restore.ts so the export graph does not pull in the restore's dependencies. The file, its counts and their order are unchanged; the golden copies still match. The measurement completeness guard now reads the domain module and also checks that every carried column is written back.
420 lines
17 KiB
TypeScript
420 lines
17 KiB
TypeScript
/**
|
|
* The restore half of the measurements domain. The export half, with the
|
|
* select every column is carried by, lives in `measurements-backup.ts`.
|
|
*
|
|
* The restore reads the readings twice. The first read (`readStreamedBackup`,
|
|
* before anything is deleted) checks each one and notes what the second read
|
|
* has to know about the file as a whole: which `stats:` aggregates are live,
|
|
* and which nights the retired heart-rate variability filing shares with an
|
|
* RMSSD reading (`createMeasurementRestorePrepass`). The second read, inside
|
|
* the transaction, writes them a batch at a time (`restoreMeasurements`), so
|
|
* the restore never holds more than one batch.
|
|
*/
|
|
import type { Prisma } from "@/generated/prisma/client";
|
|
import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client";
|
|
import { encryptNote } from "@/lib/crypto/note-cipher";
|
|
import {
|
|
isLegacyRmssdRow,
|
|
LEGACY_RMSSD_SOURCES,
|
|
} from "@/lib/export/legacy-rmssd";
|
|
import {
|
|
insertMeasurementRows,
|
|
type MeasurementInsertRow,
|
|
} from "@/lib/export/measurement-bulk-insert";
|
|
import {
|
|
decodeEncryptedBytes,
|
|
type MeasurementRestorePrepass,
|
|
type RestoreContext,
|
|
} from "@/lib/export/restore-context";
|
|
import { resolveUserTimezone } from "@/lib/measurements/consolidation-base";
|
|
import {
|
|
FOLDED_TYPES,
|
|
compactionTombstoneCoverId,
|
|
} from "@/lib/measurements/folded-window";
|
|
import type { BackupMeasurement } from "@/lib/validations/backup";
|
|
|
|
/** Both unique identities of a row, as it would be filed under `HRV_RMSSD`. */
|
|
function rmssdTwinKeysOf(row: BackupMeasurement): string[] {
|
|
const keys = [
|
|
`at|${row.source}|${new Date(row.measuredAt).toISOString()}|${row.sleepStage ?? ""}`,
|
|
];
|
|
if (row.externalId) keys.push(`ext|${row.source}|${row.externalId}`);
|
|
return keys;
|
|
}
|
|
|
|
const foldStatsKey = (source: string, type: string, externalId: string) =>
|
|
`${source}|${type}|${externalId}`;
|
|
|
|
/**
|
|
* The notes the measurement pass takes from the first read of the file.
|
|
*
|
|
* The live `stats:` ids of the folded types, so the measurement pass can tell
|
|
* a compaction tombstone from a person's deletion without a third read of the
|
|
* file. Keyed by source as well: Health Connect folds into the same `stats:`
|
|
* ids as Apple Health, and an Apple Health hour must not make a Health
|
|
* Connect deletion look folded.
|
|
*
|
|
* Migration 0420 refiled Fitbit and Google Health HRV as `HRV_RMSSD`: both
|
|
* providers report a nightly RMSSD, which their mappers had written as the
|
|
* SDNN series. A file written before that release still carries those rows
|
|
* as `HEART_RATE_VARIABILITY`, and restoring it verbatim would undo the
|
|
* migration for the account. The restore applies the migration's rule:
|
|
*
|
|
* - a legacy row moves to RMSSD, unless a LIVE RMSSD row of its source
|
|
* holds the same night under either unique identity (external id, or
|
|
* measured-at and sleep stage). Then the twin holds the night and the
|
|
* legacy row is not written under either type: as RMSSD it would break
|
|
* the unique key, as SDNN it would put RMSSD back into the SDNN series;
|
|
* - a legacy row whose only twin is a tombstone wins: the tombstone is the
|
|
* row left out, since it shares the key and holds nothing;
|
|
* - a legacy tombstone with any RMSSD twin is left out.
|
|
*
|
|
* The keys are gathered on the first read.
|
|
*/
|
|
export function createMeasurementRestorePrepass(): MeasurementRestorePrepass {
|
|
const liveFoldStatsIds = new Set<string>();
|
|
const foldedTypes = new Set<string>(FOLDED_TYPES);
|
|
const foldSources = new Set<string>(["APPLE_HEALTH", "HEALTH_CONNECT"]);
|
|
const noteLiveFoldRow = (row: BackupMeasurement) => {
|
|
if (
|
|
row.deletedAt == null &&
|
|
row.source !== undefined &&
|
|
foldSources.has(row.source) &&
|
|
foldedTypes.has(row.type) &&
|
|
row.externalId?.startsWith("stats:")
|
|
) {
|
|
liveFoldStatsIds.add(foldStatsKey(row.source, row.type, row.externalId));
|
|
}
|
|
};
|
|
const rmssdLiveKeys = new Set<string>();
|
|
const rmssdAnyKeys = new Set<string>();
|
|
const legacyLiveKeys = new Set<string>();
|
|
const noteRmssdTwin = (row: BackupMeasurement) => {
|
|
if (isLegacyRmssdRow(row)) {
|
|
if (row.deletedAt == null) {
|
|
for (const key of rmssdTwinKeysOf(row)) legacyLiveKeys.add(key);
|
|
}
|
|
return;
|
|
}
|
|
if (
|
|
row.type === "HRV_RMSSD" &&
|
|
row.source !== undefined &&
|
|
LEGACY_RMSSD_SOURCES.has(row.source)
|
|
) {
|
|
for (const key of rmssdTwinKeysOf(row)) {
|
|
rmssdAnyKeys.add(key);
|
|
if (row.deletedAt == null) rmssdLiveKeys.add(key);
|
|
}
|
|
}
|
|
};
|
|
const anyKeyIn = (row: BackupMeasurement, keys: ReadonlySet<string>) =>
|
|
rmssdTwinKeysOf(row).some((key) => keys.has(key));
|
|
const isShadowedLegacyRmssd = (row: BackupMeasurement): boolean => {
|
|
if (isLegacyRmssdRow(row)) {
|
|
return anyKeyIn(
|
|
row,
|
|
row.deletedAt == null ? rmssdLiveKeys : rmssdAnyKeys,
|
|
);
|
|
}
|
|
// An RMSSD tombstone on the night a live legacy row is about to take.
|
|
return (
|
|
row.type === "HRV_RMSSD" &&
|
|
row.deletedAt != null &&
|
|
row.source !== undefined &&
|
|
LEGACY_RMSSD_SOURCES.has(row.source) &&
|
|
anyKeyIn(row, legacyLiveKeys) &&
|
|
!anyKeyIn(row, rmssdLiveKeys)
|
|
);
|
|
};
|
|
return {
|
|
note(row) {
|
|
noteLiveFoldRow(row);
|
|
noteRmssdTwin(row);
|
|
},
|
|
liveFoldStatsIds,
|
|
isShadowedLegacyRmssd,
|
|
};
|
|
}
|
|
|
|
export async function clearMeasurements(ctx: RestoreContext): Promise<void> {
|
|
const measurements = await ctx.tx.measurement.deleteMany({
|
|
where: { userId: ctx.ownerId },
|
|
});
|
|
ctx.cleared.measurements = measurements.count;
|
|
}
|
|
|
|
/**
|
|
* The persisted measurement rollup partition, wiped for the same reason as
|
|
* the mood one: kept, it would hold the previous owner's daily means after
|
|
* the underlying measurements were replaced, and the chart's `source=rollup`
|
|
* fast-path could surface a stale 30-day mean built from rows that no longer
|
|
* exist. The rebuild after the commit mints fresh rows from the restored set.
|
|
*/
|
|
export async function clearMeasurementRollups(
|
|
ctx: RestoreContext,
|
|
): Promise<void> {
|
|
await ctx.tx.measurementRollup.deleteMany({ where: { userId: ctx.ownerId } });
|
|
}
|
|
|
|
function toRestoredMeasurementData(measurement: BackupMeasurement) {
|
|
return {
|
|
type: isLegacyRmssdRow(measurement)
|
|
? ("HRV_RMSSD" as const)
|
|
: measurement.type,
|
|
value: measurement.value,
|
|
valueMin: measurement.valueMin ?? null,
|
|
valueMax: measurement.valueMax ?? null,
|
|
unit: measurement.unit,
|
|
source: measurement.source ?? "MANUAL",
|
|
measuredAt: new Date(measurement.measuredAt),
|
|
notes: null,
|
|
notesEncrypted:
|
|
measurement.notesEncrypted == null
|
|
? encryptNote(measurement.notes ?? null)
|
|
: decodeEncryptedBytes(measurement.notesEncrypted),
|
|
externalId: measurement.externalId ?? null,
|
|
externalSourceVersion: measurement.externalSourceVersion ?? null,
|
|
// Authority of an Apple Health `stats:` aggregate. A backup
|
|
// written before this field rode the payload carries nothing, and
|
|
// restoring NULL would flatten the authority ladder: an export.xml
|
|
// source-day estimate could then overwrite a native HealthKit
|
|
// statistic, which is the ambiguity migration 0263 already
|
|
// repaired once. Such a row is exactly the case LEGACY_UNKNOWN
|
|
// exists for, so it comes back repairable instead of ambiguous.
|
|
//
|
|
// The gate is scoped exactly as 0263 scoped its backfill and as
|
|
// the cumulative drain scopes its own LEGACY_UNKNOWN writes:
|
|
// Apple Health rows only. Other providers mint `stats:` ids for
|
|
// their cumulative dailies too (Fitbit, Google Health) and have
|
|
// never carried provenance — marking those would invent a state
|
|
// they were never in. Ordinary point rows keep NULL honestly.
|
|
//
|
|
// Only an ABSENT field is the old-backup case. A current backup
|
|
// writes the field for every row, and an explicit null is the row's
|
|
// real state: the nightly daily-mean fold writes Apple Health
|
|
// `stats:` rows with no provenance. Treating that null as absent
|
|
// turned every such row into LEGACY_UNKNOWN on restore (#1031).
|
|
aggregationProvenance: (measurement.aggregationProvenance !== undefined
|
|
? measurement.aggregationProvenance
|
|
: measurement.source === "APPLE_HEALTH" &&
|
|
measurement.externalId?.startsWith("stats:")
|
|
? "LEGACY_UNKNOWN"
|
|
: null) as never,
|
|
glucoseContext: (measurement.glucoseContext ?? null) as never,
|
|
sleepStage: (measurement.sleepStage ?? null) as never,
|
|
rhythmClassification: (measurement.rhythmClassification ?? null) as never,
|
|
deviceType: measurement.deviceType ?? null,
|
|
syncVersion: measurement.syncVersion ?? 1,
|
|
deletedAt: measurement.deletedAt ? new Date(measurement.deletedAt) : null,
|
|
...(measurement.createdAt
|
|
? { createdAt: new Date(measurement.createdAt) }
|
|
: {}),
|
|
...(measurement.updatedAt
|
|
? { updatedAt: new Date(measurement.updatedAt) }
|
|
: {}),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Which deleted readings the file carries are not written back.
|
|
*
|
|
* Deleted readings the file carries are restored as deleted rows
|
|
* (tombstones), with one exception: a tombstone deleted longer ago than
|
|
* `TOMBSTONE_RETENTION_DAYS` is not written at all.
|
|
*
|
|
* Such a row is exactly what the nightly tombstone purge
|
|
* (`measurement-tombstone-cleanup.ts`) hard-deletes, so writing it costs the
|
|
* restore a row in thirteen indexes for a row that is gone by the next
|
|
* morning. Nothing reads it in between: the delta feed answers
|
|
* `cursorExpired` for every cursor older than the same horizon, so no client
|
|
* that would still be served this tombstone exists; the step-consolidation
|
|
* repair and the hourly rebuild only look inside the horizon; and the undo of
|
|
* a delete is offered for what the purge has not removed yet, which this row
|
|
* would not survive either.
|
|
*
|
|
* Younger tombstones are written. They still have readers: a paired client
|
|
* that has not yet drained that deletion from the delta feed, the undo of a
|
|
* recent delete, the consolidation repairs that rebuild from folded samples,
|
|
* and the unique `(user, type, source, externalId)` key that stops a device
|
|
* from uploading a folded sample again. The backup file itself is unchanged
|
|
* and keeps every row.
|
|
*
|
|
* Nor is a compaction tombstone written, whatever its age: a raw sample the
|
|
* folds soft-deleted before v1.42, whose whole hour or day the fold had
|
|
* reached when it was deleted and which a live `stats:` row of the same
|
|
* source in the same file covers. The folds delete those outright now and the
|
|
* backlog purge removes the old ones, so writing one back would only hand the
|
|
* purge the same row again; the `folded_window` ingest guard is what keeps
|
|
* the sample from being uploaded again. Classified by the rule the purge and
|
|
* the sync feed use (`folded-window.ts`), against the `stats:` ids gathered
|
|
* on the file's first read and the zone the account will have after the
|
|
* restore. Counted with the expired ones.
|
|
*/
|
|
async function expiredTombstoneFilter(
|
|
ctx: RestoreContext,
|
|
): Promise<(measurement: BackupMeasurement) => boolean> {
|
|
const { tx, ownerId, payload } = ctx;
|
|
const tombstoneHorizonMs = Date.now() - TOMBSTONE_RETENTION_DAYS * 86_400_000;
|
|
const foldTz = resolveUserTimezone(
|
|
payload.accountSettings?.timezone ??
|
|
(
|
|
await tx.user.findUnique({
|
|
where: { id: ownerId },
|
|
select: { timezone: true },
|
|
})
|
|
)?.timezone ??
|
|
null,
|
|
);
|
|
// Only once the fold repair has been through the means this file holds.
|
|
// The repair recomputes a mean an older release folded from part of its
|
|
// day, and these tombstones are what it reads. A file exported after the
|
|
// account's repair finished carries repaired means, so its compaction
|
|
// tombstones can go. Any other file is written back whole, the account's
|
|
// repair marker is removed, and the repair runs again once the transaction
|
|
// commits; the purge then removes the tombstones as usual.
|
|
const foldRepair = await tx.measurementFoldRepair.findUnique({
|
|
where: { userId: ownerId },
|
|
select: { completedAt: true },
|
|
});
|
|
const dropCompactionTombstones =
|
|
foldRepair !== null &&
|
|
new Date(payload.exportedAt).getTime() > foldRepair.completedAt.getTime();
|
|
if (!dropCompactionTombstones) {
|
|
await tx.measurementFoldRepair.deleteMany({
|
|
where: { userId: ownerId },
|
|
});
|
|
ctx.outputs.foldRepairReset = true;
|
|
}
|
|
const { liveFoldStatsIds } = ctx.measurementPrepass;
|
|
const isCompactionTombstone = (measurement: BackupMeasurement) => {
|
|
if (!dropCompactionTombstones) return false;
|
|
if (measurement.deletedAt == null) return false;
|
|
const source = measurement.source ?? "MANUAL";
|
|
const covering = compactionTombstoneCoverId(
|
|
{
|
|
type: measurement.type,
|
|
source,
|
|
externalId: measurement.externalId ?? null,
|
|
measuredAt: new Date(measurement.measuredAt),
|
|
deletedAt: new Date(measurement.deletedAt),
|
|
},
|
|
foldTz,
|
|
);
|
|
return (
|
|
covering !== null &&
|
|
liveFoldStatsIds.has(foldStatsKey(source, measurement.type, covering))
|
|
);
|
|
};
|
|
return (measurement: BackupMeasurement) =>
|
|
measurement.deletedAt != null &&
|
|
(new Date(measurement.deletedAt).getTime() < tombstoneHorizonMs ||
|
|
isCompactionTombstone(measurement));
|
|
}
|
|
|
|
/**
|
|
* Two later sections point INTO the measurements (a record's
|
|
* `sourceMeasurementId`, an ECG strip's `measurementId`) and have to know
|
|
* which of those ids this transaction actually wrote. They used to be handed
|
|
* every written id; they only ever ask about the ones they reference, so only
|
|
* those are collected.
|
|
*/
|
|
function referencedMeasurementIdsOf(
|
|
payload: RestoreContext["payload"],
|
|
): Set<string> {
|
|
const referenced = new Set<string>();
|
|
for (const record of payload.personalRecords) {
|
|
if (record.sourceMeasurementId) {
|
|
referenced.add(record.sourceMeasurementId);
|
|
}
|
|
}
|
|
for (const recording of payload.ecgRecordings) {
|
|
if (recording.measurementId) {
|
|
referenced.add(recording.measurementId);
|
|
}
|
|
}
|
|
return referenced;
|
|
}
|
|
|
|
/**
|
|
* v1 payloads did not require stable ids. Their historical natural-key
|
|
* reconciliation is preserved without routing canonical v2 rows through it.
|
|
*/
|
|
async function reconcileWithoutId(
|
|
tx: Prisma.TransactionClient,
|
|
ownerId: string,
|
|
measurement: BackupMeasurement,
|
|
): Promise<void> {
|
|
const restoredData = toRestoredMeasurementData(measurement);
|
|
const existing = await tx.measurement.findFirst({
|
|
where: {
|
|
userId: ownerId,
|
|
type: measurement.type,
|
|
source: restoredData.source,
|
|
measuredAt: restoredData.measuredAt,
|
|
sleepStage: restoredData.sleepStage,
|
|
},
|
|
select: { id: true },
|
|
});
|
|
if (existing) {
|
|
await tx.measurement.update({
|
|
where: { id: existing.id, userId: ownerId },
|
|
data: restoredData,
|
|
});
|
|
} else {
|
|
await tx.measurement.create({
|
|
data: { userId: ownerId, ...restoredData },
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The measurements, a batch at a time from a second read of the file, so the
|
|
* restore never holds more than one batch of them.
|
|
*/
|
|
export async function restoreMeasurements(ctx: RestoreContext): Promise<void> {
|
|
const { tx, ownerId, payload, streamed } = ctx;
|
|
const referencedMeasurementIds = referencedMeasurementIdsOf(payload);
|
|
const writtenReferencedIds = ctx.links.writtenReferencedMeasurementIds;
|
|
const measurementBatchSize = 1_000;
|
|
const isExpiredTombstone = await expiredTombstoneFilter(ctx);
|
|
const { isShadowedLegacyRmssd } = ctx.measurementPrepass;
|
|
ctx.reportMeasurements(0);
|
|
await streamed.forEachMeasurementBatch(
|
|
measurementBatchSize,
|
|
async (fileBatch) => {
|
|
const kept = fileBatch.filter(
|
|
(measurement) => !isExpiredTombstone(measurement),
|
|
);
|
|
ctx.outputs.expiredTombstonesSkipped += fileBatch.length - kept.length;
|
|
const batch = kept.filter(
|
|
(measurement) => !isShadowedLegacyRmssd(measurement),
|
|
);
|
|
const stableRows = batch.flatMap((measurement): MeasurementInsertRow[] =>
|
|
measurement.id
|
|
? [
|
|
{
|
|
id: measurement.id,
|
|
userId: ownerId,
|
|
...toRestoredMeasurementData(measurement),
|
|
},
|
|
]
|
|
: [],
|
|
);
|
|
// One statement per batch, not `createMany`: see
|
|
// `insertMeasurementRows` for what the latter cost here.
|
|
await insertMeasurementRows(tx, stableRows);
|
|
// Counted through the file, so the bar ends at the file's total.
|
|
ctx.reportMeasurements(fileBatch.length);
|
|
for (const row of stableRows) {
|
|
if (referencedMeasurementIds.has(row.id)) {
|
|
writtenReferencedIds.add(row.id);
|
|
}
|
|
}
|
|
for (const measurement of batch) {
|
|
if (measurement.id) continue;
|
|
await reconcileWithoutId(tx, ownerId, measurement);
|
|
}
|
|
},
|
|
);
|
|
}
|