mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-15 23:12:04 +00:00
Long-term retention for the dense intra-day tier (PULSE, HRV, SpO2) folds to per-local-hour means instead of a single daily mean: the last 14 days keep every raw sample, older days keep up to 24 points per metric — the day's shape survives permanently at ~9k rows/metric/year. Hourly rows key as stats:<HK>:<YYYY-MM-DD>T<HH> anchored at local HH:30, adopt-in-place against both unique indexes with the P2002 retry outside the transaction. The pre-fold DAY-rollup recompute extends to HRV (mean fidelity, not only min/max) and the post-fold recompute is gone for the whole tier. Derived resting rows for proxy users still mint from the day's raw samples. A one-shot rebuild reconstructs the hourly shape for every already-folded day from the soft-deleted raw samples and retires the daily row in the same transaction — at no instant are both grains live, so no AVG-over-live-rows reader can double-count. Boot discovery pairs live daily-grain rows with surviving tombstones (parameter-bound SQL, pinned against real Postgres) and converges to zero once converted; days whose tombstones were pruned keep their daily mean. PULSE/SpO2 DAY buckets keep the true pre-fold stats; HRV DAY buckets recompute from the hourly rows (strictly better than the collapsed fold-time bucket); WEEK/MONTH/YEAR respans enqueue whole-bucket recomputes.
474 lines
16 KiB
TypeScript
474 lines
16 KiB
TypeScript
/**
|
|
* Shared base for the three per-user-day consolidation drains:
|
|
* - `drain-per-sample-cumulative.ts` (SUM, hard-delete)
|
|
* - `consolidate-daily-mean.ts` (MEAN, soft-delete)
|
|
* - `consolidate-legacy-steps.ts` (SUM, soft-delete, existing-total skip)
|
|
*
|
|
* All three walk the same skeleton: load the user set, resolve each
|
|
* user's timezone, scan live source rows for a type set inside an
|
|
* optional grace window, group the rows into per-day buckets in the
|
|
* user's timezone, reduce each day's values, then mint one canonical
|
|
* `stats:<HKIdentifier>:<YYYY-MM-DD>` daily row and drop the source rows.
|
|
*
|
|
* The drains diverge on the reducer (sum vs mean), the delete strategy
|
|
* (hard vs soft), the source-scope filter, the canonical-unit resolution,
|
|
* the mint `source`, and — for the legacy-step pass only — an
|
|
* existing-total skip plus a P2002 conflict step-over. This module owns
|
|
* the common machinery and parameterises the divergent parts through the
|
|
* `runConsolidation` driver below; each drain stays a thin caller that
|
|
* supplies its reducer/types and keeps its own public summary shape.
|
|
*
|
|
* The timezone day-math helpers (`dayKeyForUserTz`,
|
|
* `canonicalDailyTimestamp`, `localStartOfDay`, `localDayWindow`), the
|
|
* `PerSampleRow` shape, and `CONSOLIDATION_GRACE_CUTOFF_HOURS` all live
|
|
* in the dependency-free leaf `consolidation-tz.ts`. Both this module
|
|
* and `drain-per-sample-cumulative.ts` import them from there, so there
|
|
* is no value-level cycle between the two — a cycle would otherwise trip
|
|
* a module-init TDZ in the production bundle.
|
|
*/
|
|
import type {
|
|
MeasurementType,
|
|
PrismaClient,
|
|
Prisma,
|
|
} from "@/generated/prisma/client";
|
|
|
|
import {
|
|
CONSOLIDATION_GRACE_CUTOFF_HOURS,
|
|
canonicalDailyTimestamp,
|
|
dayKeyForUserTz,
|
|
type PerSampleRow,
|
|
} from "./consolidation-tz";
|
|
|
|
// Re-export the shared grace constant so the established
|
|
// `consolidation-base` import surface stays unchanged for callers.
|
|
export { CONSOLIDATION_GRACE_CUTOFF_HOURS };
|
|
|
|
/** Fallback timezone for users with no `User.timezone` set. */
|
|
const DEFAULT_TIMEZONE = "Europe/Berlin";
|
|
|
|
/**
|
|
* Options shared by every drain. Each drain re-exports its own alias of
|
|
* this shape so the call sites and tests keep their established names.
|
|
*/
|
|
export interface ConsolidationOptions {
|
|
/** Limit the pass to a single user. Default = every user. */
|
|
userId?: string;
|
|
/** Preview-only mode — no DB writes. */
|
|
dryRun?: boolean;
|
|
/** Logger sink — defaults to `console.log`. */
|
|
log?: (line: string) => void;
|
|
}
|
|
|
|
/**
|
|
* Resolve a user's effective timezone, falling back to `Europe/Berlin`
|
|
* for an empty / null value.
|
|
*/
|
|
export function resolveUserTimezone(timezone: string | null): string {
|
|
return timezone && timezone.length > 0 ? timezone : DEFAULT_TIMEZONE;
|
|
}
|
|
|
|
/**
|
|
* Compute the cutoff instant for a grace window. Returns `null` when no
|
|
* positive `cutoffHours` is supplied (drain everything the caller points
|
|
* at), matching the one-shot / CLI default the drains carry.
|
|
*/
|
|
export function resolveCutoffInstant(
|
|
cutoffHours: number | undefined,
|
|
): Date | null {
|
|
return typeof cutoffHours === "number" && cutoffHours > 0
|
|
? new Date(Date.now() - cutoffHours * 60 * 60 * 1000)
|
|
: null;
|
|
}
|
|
|
|
/**
|
|
* Load the `{ id, timezone }` set the pass walks — a single user when
|
|
* `userId` is set, every user otherwise. Identical query across all
|
|
* three drains.
|
|
*/
|
|
export async function loadConsolidationUsers(
|
|
prismaClient: PrismaClient,
|
|
userId: string | undefined,
|
|
): Promise<Array<{ id: string; timezone: string | null }>> {
|
|
return userId
|
|
? prismaClient.user.findMany({
|
|
where: { id: userId },
|
|
select: { id: true, timezone: true },
|
|
})
|
|
: prismaClient.user.findMany({
|
|
select: { id: true, timezone: true },
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Group an array of per-sample rows into per-day buckets keyed by the
|
|
* user's timezone. Rows already in the daily-stats shape (externalId
|
|
* starting with `statsPrefix`) are skipped — re-running on a
|
|
* previously-collapsed bucket is a no-op. This is the single bucketing
|
|
* implementation behind `bucketRowsByUserDay`, `bucketMeanRows`, and
|
|
* `bucketLegacyStepRows`; the only thing that varied between them was the
|
|
* prefix they skip on.
|
|
*/
|
|
export function bucketRowsByDay(
|
|
rows: readonly PerSampleRow[],
|
|
tz: string,
|
|
statsPrefix: string,
|
|
): Map<string, PerSampleRow[]> {
|
|
const byDay = new Map<string, PerSampleRow[]>();
|
|
for (const row of rows) {
|
|
if (row.externalId !== null && row.externalId.startsWith(statsPrefix)) {
|
|
continue;
|
|
}
|
|
const key = dayKeyForUserTz(row.measuredAt, tz);
|
|
const slot = byDay.get(key) ?? [];
|
|
slot.push(row);
|
|
byDay.set(key, slot);
|
|
}
|
|
return byDay;
|
|
}
|
|
|
|
/** Outcome of writing a single per-day bucket. */
|
|
export type DayWriteOutcome =
|
|
{ kind: "written"; sourceRowsRemoved: number } | { kind: "skipped-conflict" };
|
|
|
|
/**
|
|
* Context a per-day write strategy receives. The strategy owns its own
|
|
* `$transaction` so the mint + delete commit atomically — the original
|
|
* drains each wrapped the pair in one transaction and that is preserved.
|
|
*/
|
|
export interface DayWriteContext {
|
|
prismaClient: PrismaClient;
|
|
userId: string;
|
|
type: MeasurementType;
|
|
externalId: string;
|
|
canonicalTimestamp: Date;
|
|
reducedValue: number;
|
|
dayRows: PerSampleRow[];
|
|
sourceRowIds: string[];
|
|
/**
|
|
* The user's resolved IANA timezone and the bucket's calendar-day key.
|
|
* A drain whose write grain is finer than the day (the dense-tier hourly
|
|
* fold) needs both to derive per-hour sub-buckets; the day-grain drains
|
|
* ignore them.
|
|
*/
|
|
tz: string;
|
|
dateKey: string;
|
|
}
|
|
|
|
/** Inputs to one per-user-type-day consolidation pass. */
|
|
export interface ConsolidationParams<TType extends MeasurementType> {
|
|
prismaClient: PrismaClient;
|
|
options: ConsolidationOptions & { cutoffHours?: number };
|
|
/** Types the pass scans. Accepts an array or a `ReadonlySet`. */
|
|
types: Iterable<TType>;
|
|
/** Maps a type to its canonical HealthKit identifier, or `null` to skip. */
|
|
hkIdentifierForType: (type: TType) => string | null;
|
|
/** Builds the daily `stats:<HKIdentifier>:<dateKey>` externalId. */
|
|
dailyStatsExternalId: (hkIdentifier: string, dateKey: string) => string;
|
|
/** Prefix marking an already-collapsed daily-stats row. */
|
|
statsPrefix: string;
|
|
/** Reduce a non-empty per-day bucket to its canonical value. */
|
|
reduce: (rows: readonly PerSampleRow[]) => number;
|
|
/**
|
|
* Build the per-(user, type) Prisma `where` for the scan, given the
|
|
* grace cutoff (or `null`). Owns the source-scope + soft-delete-filter
|
|
* differences between drains.
|
|
*/
|
|
buildScanWhere: (input: {
|
|
userId: string;
|
|
type: TType;
|
|
cutoffAt: Date | null;
|
|
statsPrefix: string;
|
|
}) => Prisma.MeasurementWhereInput;
|
|
/**
|
|
* Columns the scan selects. Mean needs `unit`; the others don't.
|
|
* Defaults to the shared set when omitted.
|
|
*/
|
|
scanSelect?: Prisma.MeasurementSelect;
|
|
/**
|
|
* Per-day callback invoked BEFORE the write transaction (dry-run
|
|
* included). Returns `false` to record the bucket but skip the mint
|
|
* (legacy-step existing-total case). Optional; defaults to always-write.
|
|
*/
|
|
onBucket?: (input: {
|
|
prismaClient: PrismaClient;
|
|
userId: string;
|
|
type: TType;
|
|
dateKey: string;
|
|
dayRows: PerSampleRow[];
|
|
reducedValue: number;
|
|
canonicalTimestamp: Date;
|
|
externalId: string;
|
|
}) => Promise<boolean> | boolean;
|
|
/**
|
|
* Write one per-day bucket inside the supplied transaction (mint +
|
|
* delete/soft-delete). Owns the hard-vs-soft delete + mint-source
|
|
* differences. `shouldMint` carries the `onBucket` decision.
|
|
*/
|
|
writeDay: (
|
|
ctx: DayWriteContext & { shouldMint: boolean },
|
|
) => Promise<DayWriteOutcome>;
|
|
/**
|
|
* Accumulate a written / previewed bucket into the drain's own summary.
|
|
* Called once per day the pass acts on. `outcome` is `null` on dry-run.
|
|
*/
|
|
recordBucket: (input: {
|
|
userId: string;
|
|
type: TType;
|
|
dateKey: string;
|
|
dayRows: PerSampleRow[];
|
|
reducedValue: number;
|
|
canonicalTimestamp: Date;
|
|
externalId: string;
|
|
shouldMint: boolean;
|
|
outcome: DayWriteOutcome | null;
|
|
}) => void;
|
|
/** Per-user log line, START — optional. */
|
|
onUserStart?: (input: {
|
|
userId: string;
|
|
tz: string;
|
|
dryRun: boolean;
|
|
}) => void;
|
|
/**
|
|
* Fired once per (user, type) immediately after the rows are bucketed,
|
|
* with the raw scanned-row count and per-day bucket count. Only invoked
|
|
* for types that yielded at least one source row. Lets a single-type
|
|
* drain (legacy steps) reproduce its scan-time log line. Optional.
|
|
*/
|
|
onScan?: (input: {
|
|
userId: string;
|
|
type: TType;
|
|
tz: string;
|
|
rowCount: number;
|
|
dayCount: number;
|
|
dryRun: boolean;
|
|
}) => void;
|
|
/** Per-user log line, COMPLETE — optional. */
|
|
onUserComplete?: (input: {
|
|
userId: string;
|
|
tz: string;
|
|
dryRun: boolean;
|
|
}) => void;
|
|
/**
|
|
* Per-day failure boundary — optional. When supplied, an error thrown
|
|
* while reducing / writing / recording ONE day bucket is reported here
|
|
* and the walk continues with the next bucket, so a single poisoned day
|
|
* (e.g. a unique-index collision on the mint) can no longer abort the
|
|
* whole global pass and strand every later user / type / day. When
|
|
* omitted, errors propagate exactly as before — drains whose `writeDay`
|
|
* classifies its own errors (and relies on the rethrow reaching pg-boss
|
|
* for a retry) keep their established semantics.
|
|
*/
|
|
onBucketError?: (input: {
|
|
userId: string;
|
|
type: TType;
|
|
dateKey: string;
|
|
error: unknown;
|
|
}) => void;
|
|
}
|
|
|
|
const DEFAULT_SCAN_SELECT: Prisma.MeasurementSelect = {
|
|
id: true,
|
|
type: true,
|
|
value: true,
|
|
measuredAt: true,
|
|
externalId: true,
|
|
};
|
|
|
|
/**
|
|
* Page size for the keyset-paginated source-row scan. Heavy multi-year
|
|
* tenants previously materialised the entire per-(user, type) history in a
|
|
* single unbounded `findMany` result set — the dominant Prisma intermediate
|
|
* cost and the root of the consolidation boot-storm on large accounts.
|
|
* Scanning in bounded keyset pages caps the per-query result set while still
|
|
* accumulating the same full row set.
|
|
*/
|
|
const CONSOLIDATION_SCAN_PAGE_SIZE = 5000;
|
|
|
|
/**
|
|
* Keyset-paginate the per-(user, type) source-row scan on `(measuredAt, id)`
|
|
* ascending, accumulating every page into one array. Output is identical to a
|
|
* single unbounded `findMany` — same `where`, same `select`, same ascending
|
|
* order — but the Prisma client never holds more than one page of rows at a
|
|
* time, so a multi-year tenant no longer materialises its whole history in one
|
|
* result set at worker boot.
|
|
*
|
|
* `(measuredAt, id)` is a stable keyset: `measuredAt` is the primary order and
|
|
* `id` breaks ties, so each page resumes exactly after the last row with no
|
|
* gaps or duplicates. The cursor clause is AND-combined with the caller's
|
|
* `baseWhere` so the scope / soft-delete / grace filters always still apply.
|
|
*/
|
|
export async function scanSourceRowsPaged(
|
|
prismaClient: PrismaClient,
|
|
baseWhere: Prisma.MeasurementWhereInput,
|
|
scanSelect: Prisma.MeasurementSelect,
|
|
pageSize: number = CONSOLIDATION_SCAN_PAGE_SIZE,
|
|
): Promise<PerSampleRow[]> {
|
|
const accumulated: PerSampleRow[] = [];
|
|
let cursor: { measuredAt: Date; id: string } | null = null;
|
|
|
|
for (;;) {
|
|
const where: Prisma.MeasurementWhereInput = cursor
|
|
? {
|
|
AND: [
|
|
baseWhere,
|
|
{
|
|
OR: [
|
|
{ measuredAt: { gt: cursor.measuredAt } },
|
|
{ measuredAt: cursor.measuredAt, id: { gt: cursor.id } },
|
|
],
|
|
},
|
|
],
|
|
}
|
|
: baseWhere;
|
|
|
|
const page = (await prismaClient.measurement.findMany({
|
|
where,
|
|
select: scanSelect,
|
|
orderBy: [{ measuredAt: "asc" }, { id: "asc" }],
|
|
take: pageSize,
|
|
})) as PerSampleRow[];
|
|
|
|
if (page.length === 0) break;
|
|
|
|
for (const row of page) accumulated.push(row);
|
|
|
|
// A short page is the last page — no further rows can satisfy the keyset.
|
|
if (page.length < pageSize) break;
|
|
|
|
const last = page[page.length - 1]!;
|
|
cursor = { measuredAt: last.measuredAt, id: last.id };
|
|
}
|
|
|
|
return accumulated;
|
|
}
|
|
|
|
/**
|
|
* Drive one consolidation pass. Walks `users → types → days`, scans live
|
|
* source rows inside the grace window, buckets them, reduces each day,
|
|
* and delegates the per-day mint + delete to `writeDay`. Returns the
|
|
* number of users scanned (so the caller can seed its summary totals)
|
|
* plus the number of day buckets absorbed by `onBucketError` (0 when the
|
|
* boundary is not supplied).
|
|
*
|
|
* Idempotency, the grace-window cutoff, and the "skip already-collapsed
|
|
* rows" predicate are all owned here; the divergent reducer / delete /
|
|
* scope / summary concerns are supplied by the params.
|
|
*/
|
|
export async function runConsolidation<TType extends MeasurementType>(
|
|
params: ConsolidationParams<TType>,
|
|
): Promise<{ usersScanned: number; dryRun: boolean; daysFailed: number }> {
|
|
const { prismaClient, options } = params;
|
|
const dryRun = options.dryRun ?? false;
|
|
const cutoffAt = resolveCutoffInstant(options.cutoffHours);
|
|
const scanSelect = params.scanSelect ?? DEFAULT_SCAN_SELECT;
|
|
let daysFailed = 0;
|
|
|
|
const users = await loadConsolidationUsers(prismaClient, options.userId);
|
|
|
|
for (const user of users) {
|
|
const tz = resolveUserTimezone(user.timezone);
|
|
params.onUserStart?.({ userId: user.id, tz, dryRun });
|
|
|
|
for (const type of params.types) {
|
|
const hkIdentifier = params.hkIdentifierForType(type);
|
|
if (!hkIdentifier) continue;
|
|
|
|
// Keyset-paginate the scan instead of loading the whole per-(user, type)
|
|
// history in one result set; the accumulated array is byte-identical to a
|
|
// single unbounded `findMany`, but a multi-year tenant no longer spikes
|
|
// worker memory at boot. See `scanSourceRowsPaged`.
|
|
const sourceRows = await scanSourceRowsPaged(
|
|
prismaClient,
|
|
params.buildScanWhere({
|
|
userId: user.id,
|
|
type,
|
|
cutoffAt,
|
|
statsPrefix: params.statsPrefix,
|
|
}),
|
|
scanSelect,
|
|
);
|
|
|
|
if (sourceRows.length === 0) continue;
|
|
|
|
const byDay = bucketRowsByDay(sourceRows, tz, params.statsPrefix);
|
|
|
|
params.onScan?.({
|
|
userId: user.id,
|
|
type,
|
|
tz,
|
|
rowCount: sourceRows.length,
|
|
dayCount: byDay.size,
|
|
dryRun,
|
|
});
|
|
|
|
for (const [dateKey, dayRows] of byDay) {
|
|
if (dayRows.length === 0) continue;
|
|
|
|
try {
|
|
const reducedValue = params.reduce(dayRows);
|
|
const canonicalTimestamp = canonicalDailyTimestamp(dateKey, tz);
|
|
const externalId = params.dailyStatsExternalId(hkIdentifier, dateKey);
|
|
|
|
const shouldMint = params.onBucket
|
|
? await params.onBucket({
|
|
prismaClient,
|
|
userId: user.id,
|
|
type,
|
|
dateKey,
|
|
dayRows,
|
|
reducedValue,
|
|
canonicalTimestamp,
|
|
externalId,
|
|
})
|
|
: true;
|
|
|
|
let outcome: DayWriteOutcome | null = null;
|
|
if (!dryRun) {
|
|
const sourceRowIds = dayRows.map((r) => r.id);
|
|
outcome = await params.writeDay({
|
|
prismaClient,
|
|
userId: user.id,
|
|
type,
|
|
externalId,
|
|
canonicalTimestamp,
|
|
reducedValue,
|
|
dayRows,
|
|
sourceRowIds,
|
|
tz,
|
|
dateKey,
|
|
shouldMint,
|
|
});
|
|
}
|
|
|
|
params.recordBucket({
|
|
userId: user.id,
|
|
type,
|
|
dateKey,
|
|
dayRows,
|
|
reducedValue,
|
|
canonicalTimestamp,
|
|
externalId,
|
|
shouldMint,
|
|
outcome,
|
|
});
|
|
} catch (err) {
|
|
// No boundary supplied → preserve the historical abort-the-run
|
|
// behaviour for the drains that classify errors in `writeDay`.
|
|
if (!params.onBucketError) throw err;
|
|
daysFailed += 1;
|
|
params.onBucketError({
|
|
userId: user.id,
|
|
type,
|
|
dateKey,
|
|
error: err,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
params.onUserComplete?.({ userId: user.id, tz, dryRun });
|
|
}
|
|
|
|
return { usersScanned: users.length, dryRun, daysFailed };
|
|
}
|