Files
HealthLog/src/lib/measurements/consolidation-base.ts
T
Marc-André Bombeck 1375d20a9d chore(release): v1.28.31 — heart-rate history keeps its daily shape
Long-term retention for the dense intra-day tier (PULSE, HRV, SpO2) folds
to per-local-hour means instead of a single daily mean: the last 14 days
keep every raw sample, older days keep up to 24 points per metric — the
day's shape survives permanently at ~9k rows/metric/year. Hourly rows key
as stats:<HK>:<YYYY-MM-DD>T<HH> anchored at local HH:30, adopt-in-place
against both unique indexes with the P2002 retry outside the transaction.
The pre-fold DAY-rollup recompute extends to HRV (mean fidelity, not only
min/max) and the post-fold recompute is gone for the whole tier. Derived
resting rows for proxy users still mint from the day's raw samples.

A one-shot rebuild reconstructs the hourly shape for every already-folded
day from the soft-deleted raw samples and retires the daily row in the
same transaction — at no instant are both grains live, so no
AVG-over-live-rows reader can double-count. Boot discovery pairs live
daily-grain rows with surviving tombstones (parameter-bound SQL, pinned
against real Postgres) and converges to zero once converted; days whose
tombstones were pruned keep their daily mean. PULSE/SpO2 DAY buckets keep
the true pre-fold stats; HRV DAY buckets recompute from the hourly rows
(strictly better than the collapsed fold-time bucket); WEEK/MONTH/YEAR
respans enqueue whole-bucket recomputes.
2026-07-12 12:07:41 +02:00

474 lines
16 KiB
TypeScript

/**
* Shared base for the three per-user-day consolidation drains:
* - `drain-per-sample-cumulative.ts` (SUM, hard-delete)
* - `consolidate-daily-mean.ts` (MEAN, soft-delete)
* - `consolidate-legacy-steps.ts` (SUM, soft-delete, existing-total skip)
*
* All three walk the same skeleton: load the user set, resolve each
* user's timezone, scan live source rows for a type set inside an
* optional grace window, group the rows into per-day buckets in the
* user's timezone, reduce each day's values, then mint one canonical
* `stats:<HKIdentifier>:<YYYY-MM-DD>` daily row and drop the source rows.
*
* The drains diverge on the reducer (sum vs mean), the delete strategy
* (hard vs soft), the source-scope filter, the canonical-unit resolution,
* the mint `source`, and — for the legacy-step pass only — an
* existing-total skip plus a P2002 conflict step-over. This module owns
* the common machinery and parameterises the divergent parts through the
* `runConsolidation` driver below; each drain stays a thin caller that
* supplies its reducer/types and keeps its own public summary shape.
*
* The timezone day-math helpers (`dayKeyForUserTz`,
* `canonicalDailyTimestamp`, `localStartOfDay`, `localDayWindow`), the
* `PerSampleRow` shape, and `CONSOLIDATION_GRACE_CUTOFF_HOURS` all live
* in the dependency-free leaf `consolidation-tz.ts`. Both this module
* and `drain-per-sample-cumulative.ts` import them from there, so there
* is no value-level cycle between the two — a cycle would otherwise trip
* a module-init TDZ in the production bundle.
*/
import type {
MeasurementType,
PrismaClient,
Prisma,
} from "@/generated/prisma/client";
import {
CONSOLIDATION_GRACE_CUTOFF_HOURS,
canonicalDailyTimestamp,
dayKeyForUserTz,
type PerSampleRow,
} from "./consolidation-tz";
// Re-export the shared grace constant so the established
// `consolidation-base` import surface stays unchanged for callers.
export { CONSOLIDATION_GRACE_CUTOFF_HOURS };
/** Fallback timezone for users with no `User.timezone` set. */
const DEFAULT_TIMEZONE = "Europe/Berlin";
/**
* Options shared by every drain. Each drain re-exports its own alias of
* this shape so the call sites and tests keep their established names.
*/
export interface ConsolidationOptions {
/** Limit the pass to a single user. Default = every user. */
userId?: string;
/** Preview-only mode — no DB writes. */
dryRun?: boolean;
/** Logger sink — defaults to `console.log`. */
log?: (line: string) => void;
}
/**
* Resolve a user's effective timezone, falling back to `Europe/Berlin`
* for an empty / null value.
*/
export function resolveUserTimezone(timezone: string | null): string {
return timezone && timezone.length > 0 ? timezone : DEFAULT_TIMEZONE;
}
/**
* Compute the cutoff instant for a grace window. Returns `null` when no
* positive `cutoffHours` is supplied (drain everything the caller points
* at), matching the one-shot / CLI default the drains carry.
*/
export function resolveCutoffInstant(
cutoffHours: number | undefined,
): Date | null {
return typeof cutoffHours === "number" && cutoffHours > 0
? new Date(Date.now() - cutoffHours * 60 * 60 * 1000)
: null;
}
/**
* Load the `{ id, timezone }` set the pass walks — a single user when
* `userId` is set, every user otherwise. Identical query across all
* three drains.
*/
export async function loadConsolidationUsers(
prismaClient: PrismaClient,
userId: string | undefined,
): Promise<Array<{ id: string; timezone: string | null }>> {
return userId
? prismaClient.user.findMany({
where: { id: userId },
select: { id: true, timezone: true },
})
: prismaClient.user.findMany({
select: { id: true, timezone: true },
});
}
/**
* Group an array of per-sample rows into per-day buckets keyed by the
* user's timezone. Rows already in the daily-stats shape (externalId
* starting with `statsPrefix`) are skipped — re-running on a
* previously-collapsed bucket is a no-op. This is the single bucketing
* implementation behind `bucketRowsByUserDay`, `bucketMeanRows`, and
* `bucketLegacyStepRows`; the only thing that varied between them was the
* prefix they skip on.
*/
export function bucketRowsByDay(
rows: readonly PerSampleRow[],
tz: string,
statsPrefix: string,
): Map<string, PerSampleRow[]> {
const byDay = new Map<string, PerSampleRow[]>();
for (const row of rows) {
if (row.externalId !== null && row.externalId.startsWith(statsPrefix)) {
continue;
}
const key = dayKeyForUserTz(row.measuredAt, tz);
const slot = byDay.get(key) ?? [];
slot.push(row);
byDay.set(key, slot);
}
return byDay;
}
/** Outcome of writing a single per-day bucket. */
export type DayWriteOutcome =
{ kind: "written"; sourceRowsRemoved: number } | { kind: "skipped-conflict" };
/**
* Context a per-day write strategy receives. The strategy owns its own
* `$transaction` so the mint + delete commit atomically — the original
* drains each wrapped the pair in one transaction and that is preserved.
*/
export interface DayWriteContext {
prismaClient: PrismaClient;
userId: string;
type: MeasurementType;
externalId: string;
canonicalTimestamp: Date;
reducedValue: number;
dayRows: PerSampleRow[];
sourceRowIds: string[];
/**
* The user's resolved IANA timezone and the bucket's calendar-day key.
* A drain whose write grain is finer than the day (the dense-tier hourly
* fold) needs both to derive per-hour sub-buckets; the day-grain drains
* ignore them.
*/
tz: string;
dateKey: string;
}
/** Inputs to one per-user-type-day consolidation pass. */
export interface ConsolidationParams<TType extends MeasurementType> {
prismaClient: PrismaClient;
options: ConsolidationOptions & { cutoffHours?: number };
/** Types the pass scans. Accepts an array or a `ReadonlySet`. */
types: Iterable<TType>;
/** Maps a type to its canonical HealthKit identifier, or `null` to skip. */
hkIdentifierForType: (type: TType) => string | null;
/** Builds the daily `stats:<HKIdentifier>:<dateKey>` externalId. */
dailyStatsExternalId: (hkIdentifier: string, dateKey: string) => string;
/** Prefix marking an already-collapsed daily-stats row. */
statsPrefix: string;
/** Reduce a non-empty per-day bucket to its canonical value. */
reduce: (rows: readonly PerSampleRow[]) => number;
/**
* Build the per-(user, type) Prisma `where` for the scan, given the
* grace cutoff (or `null`). Owns the source-scope + soft-delete-filter
* differences between drains.
*/
buildScanWhere: (input: {
userId: string;
type: TType;
cutoffAt: Date | null;
statsPrefix: string;
}) => Prisma.MeasurementWhereInput;
/**
* Columns the scan selects. Mean needs `unit`; the others don't.
* Defaults to the shared set when omitted.
*/
scanSelect?: Prisma.MeasurementSelect;
/**
* Per-day callback invoked BEFORE the write transaction (dry-run
* included). Returns `false` to record the bucket but skip the mint
* (legacy-step existing-total case). Optional; defaults to always-write.
*/
onBucket?: (input: {
prismaClient: PrismaClient;
userId: string;
type: TType;
dateKey: string;
dayRows: PerSampleRow[];
reducedValue: number;
canonicalTimestamp: Date;
externalId: string;
}) => Promise<boolean> | boolean;
/**
* Write one per-day bucket inside the supplied transaction (mint +
* delete/soft-delete). Owns the hard-vs-soft delete + mint-source
* differences. `shouldMint` carries the `onBucket` decision.
*/
writeDay: (
ctx: DayWriteContext & { shouldMint: boolean },
) => Promise<DayWriteOutcome>;
/**
* Accumulate a written / previewed bucket into the drain's own summary.
* Called once per day the pass acts on. `outcome` is `null` on dry-run.
*/
recordBucket: (input: {
userId: string;
type: TType;
dateKey: string;
dayRows: PerSampleRow[];
reducedValue: number;
canonicalTimestamp: Date;
externalId: string;
shouldMint: boolean;
outcome: DayWriteOutcome | null;
}) => void;
/** Per-user log line, START — optional. */
onUserStart?: (input: {
userId: string;
tz: string;
dryRun: boolean;
}) => void;
/**
* Fired once per (user, type) immediately after the rows are bucketed,
* with the raw scanned-row count and per-day bucket count. Only invoked
* for types that yielded at least one source row. Lets a single-type
* drain (legacy steps) reproduce its scan-time log line. Optional.
*/
onScan?: (input: {
userId: string;
type: TType;
tz: string;
rowCount: number;
dayCount: number;
dryRun: boolean;
}) => void;
/** Per-user log line, COMPLETE — optional. */
onUserComplete?: (input: {
userId: string;
tz: string;
dryRun: boolean;
}) => void;
/**
* Per-day failure boundary — optional. When supplied, an error thrown
* while reducing / writing / recording ONE day bucket is reported here
* and the walk continues with the next bucket, so a single poisoned day
* (e.g. a unique-index collision on the mint) can no longer abort the
* whole global pass and strand every later user / type / day. When
* omitted, errors propagate exactly as before — drains whose `writeDay`
* classifies its own errors (and relies on the rethrow reaching pg-boss
* for a retry) keep their established semantics.
*/
onBucketError?: (input: {
userId: string;
type: TType;
dateKey: string;
error: unknown;
}) => void;
}
const DEFAULT_SCAN_SELECT: Prisma.MeasurementSelect = {
id: true,
type: true,
value: true,
measuredAt: true,
externalId: true,
};
/**
* Page size for the keyset-paginated source-row scan. Heavy multi-year
* tenants previously materialised the entire per-(user, type) history in a
* single unbounded `findMany` result set — the dominant Prisma intermediate
* cost and the root of the consolidation boot-storm on large accounts.
* Scanning in bounded keyset pages caps the per-query result set while still
* accumulating the same full row set.
*/
const CONSOLIDATION_SCAN_PAGE_SIZE = 5000;
/**
* Keyset-paginate the per-(user, type) source-row scan on `(measuredAt, id)`
* ascending, accumulating every page into one array. Output is identical to a
* single unbounded `findMany` — same `where`, same `select`, same ascending
* order — but the Prisma client never holds more than one page of rows at a
* time, so a multi-year tenant no longer materialises its whole history in one
* result set at worker boot.
*
* `(measuredAt, id)` is a stable keyset: `measuredAt` is the primary order and
* `id` breaks ties, so each page resumes exactly after the last row with no
* gaps or duplicates. The cursor clause is AND-combined with the caller's
* `baseWhere` so the scope / soft-delete / grace filters always still apply.
*/
export async function scanSourceRowsPaged(
prismaClient: PrismaClient,
baseWhere: Prisma.MeasurementWhereInput,
scanSelect: Prisma.MeasurementSelect,
pageSize: number = CONSOLIDATION_SCAN_PAGE_SIZE,
): Promise<PerSampleRow[]> {
const accumulated: PerSampleRow[] = [];
let cursor: { measuredAt: Date; id: string } | null = null;
for (;;) {
const where: Prisma.MeasurementWhereInput = cursor
? {
AND: [
baseWhere,
{
OR: [
{ measuredAt: { gt: cursor.measuredAt } },
{ measuredAt: cursor.measuredAt, id: { gt: cursor.id } },
],
},
],
}
: baseWhere;
const page = (await prismaClient.measurement.findMany({
where,
select: scanSelect,
orderBy: [{ measuredAt: "asc" }, { id: "asc" }],
take: pageSize,
})) as PerSampleRow[];
if (page.length === 0) break;
for (const row of page) accumulated.push(row);
// A short page is the last page — no further rows can satisfy the keyset.
if (page.length < pageSize) break;
const last = page[page.length - 1]!;
cursor = { measuredAt: last.measuredAt, id: last.id };
}
return accumulated;
}
/**
* Drive one consolidation pass. Walks `users → types → days`, scans live
* source rows inside the grace window, buckets them, reduces each day,
* and delegates the per-day mint + delete to `writeDay`. Returns the
* number of users scanned (so the caller can seed its summary totals)
* plus the number of day buckets absorbed by `onBucketError` (0 when the
* boundary is not supplied).
*
* Idempotency, the grace-window cutoff, and the "skip already-collapsed
* rows" predicate are all owned here; the divergent reducer / delete /
* scope / summary concerns are supplied by the params.
*/
export async function runConsolidation<TType extends MeasurementType>(
params: ConsolidationParams<TType>,
): Promise<{ usersScanned: number; dryRun: boolean; daysFailed: number }> {
const { prismaClient, options } = params;
const dryRun = options.dryRun ?? false;
const cutoffAt = resolveCutoffInstant(options.cutoffHours);
const scanSelect = params.scanSelect ?? DEFAULT_SCAN_SELECT;
let daysFailed = 0;
const users = await loadConsolidationUsers(prismaClient, options.userId);
for (const user of users) {
const tz = resolveUserTimezone(user.timezone);
params.onUserStart?.({ userId: user.id, tz, dryRun });
for (const type of params.types) {
const hkIdentifier = params.hkIdentifierForType(type);
if (!hkIdentifier) continue;
// Keyset-paginate the scan instead of loading the whole per-(user, type)
// history in one result set; the accumulated array is byte-identical to a
// single unbounded `findMany`, but a multi-year tenant no longer spikes
// worker memory at boot. See `scanSourceRowsPaged`.
const sourceRows = await scanSourceRowsPaged(
prismaClient,
params.buildScanWhere({
userId: user.id,
type,
cutoffAt,
statsPrefix: params.statsPrefix,
}),
scanSelect,
);
if (sourceRows.length === 0) continue;
const byDay = bucketRowsByDay(sourceRows, tz, params.statsPrefix);
params.onScan?.({
userId: user.id,
type,
tz,
rowCount: sourceRows.length,
dayCount: byDay.size,
dryRun,
});
for (const [dateKey, dayRows] of byDay) {
if (dayRows.length === 0) continue;
try {
const reducedValue = params.reduce(dayRows);
const canonicalTimestamp = canonicalDailyTimestamp(dateKey, tz);
const externalId = params.dailyStatsExternalId(hkIdentifier, dateKey);
const shouldMint = params.onBucket
? await params.onBucket({
prismaClient,
userId: user.id,
type,
dateKey,
dayRows,
reducedValue,
canonicalTimestamp,
externalId,
})
: true;
let outcome: DayWriteOutcome | null = null;
if (!dryRun) {
const sourceRowIds = dayRows.map((r) => r.id);
outcome = await params.writeDay({
prismaClient,
userId: user.id,
type,
externalId,
canonicalTimestamp,
reducedValue,
dayRows,
sourceRowIds,
tz,
dateKey,
shouldMint,
});
}
params.recordBucket({
userId: user.id,
type,
dateKey,
dayRows,
reducedValue,
canonicalTimestamp,
externalId,
shouldMint,
outcome,
});
} catch (err) {
// No boundary supplied → preserve the historical abort-the-run
// behaviour for the drains that classify errors in `writeDay`.
if (!params.onBucketError) throw err;
daysFailed += 1;
params.onBucketError({
userId: user.id,
type,
dateKey,
error: err,
});
}
}
}
params.onUserComplete?.({ userId: user.id, tz, dryRun });
}
return { usersScanned: users.length, dryRun, daysFailed };
}