mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-13 22:32:04 +00:00
feat(measurements): consolidate gait/mobility HealthKit metrics nightly
Extend the daily-mean consolidation to cover the high-frequency gait/mobility metrics that previously mapped to no consolidation set and piled up at sampling granularity: walking asymmetry, walking double-support, walking steadiness, and walking heart-rate average. The correct daily reduction for these is the mean, matching the Health-app display; all four stay disjoint from the cumulative (sum) set. Fold the daily-mean drain onto the existing nightly cumulative-drain tick so new high-frequency spot samples no longer accumulate raw between worker reboots. The two passes share one concurrency-1 cron, keeping the maintenance window a single sequential walk. Boot-time discovery stays as the back-fill for accounts that accumulated raw rows before this shipped. Recompute the affected day's rollup bucket after the consolidation commits, the same way the ingest path does, so a rollup-covered read reflects the single consolidated row rather than a stale mean derived from the now soft-deleted per-sample rows. Regular heart-rate / pulse stays raw: it is a discrete clinical reading and feeds correlation and scatter analytics. No schema change.
This commit is contained in:
1 parent
5771b36899
commit
93790aa043
7 files changed
+284
-4
No files matched your search
@@ -118,6 +118,10 @@ import {
|
||||
drainPerSampleCumulative,
|
||||
DRAIN_CUMULATIVE_CUTOFF_HOURS,
|
||||
} from "@/lib/measurements/drain-per-sample-cumulative";
|
||||
import {
|
||||
consolidateDailyMean,
|
||||
MEAN_CONSOLIDATION_CUTOFF_HOURS,
|
||||
} from "@/lib/measurements/consolidate-daily-mean";
|
||||
import {
|
||||
STEP_CONSOLIDATION_QUEUE,
|
||||
STEP_CONSOLIDATION_CONCURRENCY,
|
||||
@@ -2601,6 +2605,32 @@ export async function startReminderWorker() {
|
||||
recordError();
|
||||
workerLog("error", "[drain-cumulative] run failed", err);
|
||||
}
|
||||
|
||||
// v1.8.5 — fold the daily-MEAN drain onto the same nightly tick as
|
||||
// the cumulative (SUM) drain. The mean drain was previously
|
||||
// boot-discovery only, so between worker reboots new high-frequency
|
||||
// spot samples (walking speed, respiratory rate, gait/mobility,
|
||||
// audio exposure) accumulated raw. Running both passes on one
|
||||
// concurrency-1 cron keeps the maintenance window a single
|
||||
// sequential walk and never crowds the request pool. The global
|
||||
// (no `userId`) signature drains every user; the 36-hour grace
|
||||
// cutoff keeps today's in-flight watch syncs raw. Boot discovery
|
||||
// stays as the back-fill for accounts that accumulated raw rows
|
||||
// before this cron shipped.
|
||||
try {
|
||||
const meanSummary = await consolidateDailyMean(getWorkerPrisma(), {
|
||||
dryRun: false,
|
||||
cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS,
|
||||
log: (line) => workerLog("info", line),
|
||||
});
|
||||
workerLog(
|
||||
"info",
|
||||
`[mean-consolidation] triggeredAt=${job.data.triggeredAt} usersScanned=${meanSummary.totals.usersScanned} daysConsolidated=${meanSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${meanSummary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${meanSummary.totals.dailyRowsUpserted}`,
|
||||
);
|
||||
} catch (err) {
|
||||
recordError();
|
||||
workerLog("error", "[mean-consolidation] nightly run failed", err);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@@ -648,10 +648,27 @@ describe("HIGH_FREQUENCY_MEAN_TYPES (v1.7.0)", () => {
|
||||
"AUDIO_EXPOSURE_HEADPHONE",
|
||||
"WALKING_SPEED",
|
||||
"WALKING_STEP_LENGTH",
|
||||
// v1.8.5 — the gait/mobility metrics that previously fell to no
|
||||
// consolidation set and piled up raw at sampling granularity.
|
||||
"WALKING_ASYMMETRY",
|
||||
"WALKING_DOUBLE_SUPPORT",
|
||||
"WALKING_STEADINESS",
|
||||
"WALKING_HEART_RATE_AVERAGE",
|
||||
].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("consolidates the v1.8.5 gait/mobility metrics", () => {
|
||||
for (const type of [
|
||||
"WALKING_ASYMMETRY",
|
||||
"WALKING_DOUBLE_SUPPORT",
|
||||
"WALKING_STEADINESS",
|
||||
"WALKING_HEART_RATE_AVERAGE",
|
||||
] as const) {
|
||||
expect(HIGH_FREQUENCY_MEAN_TYPES.has(type)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("every mean type resolves to a HealthKit identifier", () => {
|
||||
for (const type of HIGH_FREQUENCY_MEAN_TYPES) {
|
||||
expect(hkIdentifierForType(type)).not.toBeNull();
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
|
||||
const { recomputeBucketsForMeasurement } = vi.hoisted(() => ({
|
||||
recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
recomputeBucketsForMeasurement,
|
||||
}));
|
||||
|
||||
import {
|
||||
bucketMeanRows,
|
||||
@@ -6,7 +13,7 @@ import {
|
||||
meanBucketValue,
|
||||
} from "../consolidate-daily-mean";
|
||||
import type { PerSampleRow } from "../drain-per-sample-cumulative";
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
|
||||
|
||||
function row(
|
||||
id: string,
|
||||
@@ -14,10 +21,11 @@ function row(
|
||||
iso: string,
|
||||
externalId: string | null = null,
|
||||
unit = "m/s",
|
||||
type: MeasurementType = "WALKING_SPEED",
|
||||
): PerSampleRow {
|
||||
return {
|
||||
id,
|
||||
type: "WALKING_SPEED",
|
||||
type,
|
||||
value,
|
||||
measuredAt: new Date(iso),
|
||||
externalId,
|
||||
@@ -25,6 +33,10 @@ function row(
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
recomputeBucketsForMeasurement.mockClear();
|
||||
});
|
||||
|
||||
describe("consolidate-daily-mean — reducer + bucketing", () => {
|
||||
it("computes the arithmetic mean of a per-day bucket", () => {
|
||||
const rows = [
|
||||
@@ -141,6 +153,80 @@ describe("consolidateDailyMean — drain flow (mocked Prisma)", () => {
|
||||
};
|
||||
expect(updArg.data.deletedAt).toBeInstanceOf(Date);
|
||||
expect(summary.totals.daysConsolidated).toBe(1);
|
||||
|
||||
// T3 — the rollup DAY bucket must be recomputed for the touched
|
||||
// (user, type, day) so a rollup-covered read does not serve a
|
||||
// pre-drain mean derived from the now soft-deleted samples.
|
||||
expect(recomputeBucketsForMeasurement).toHaveBeenCalledTimes(1);
|
||||
const [uid, type, measuredAt] =
|
||||
recomputeBucketsForMeasurement.mock.calls[0];
|
||||
expect(uid).toBe("user-1");
|
||||
expect(type).toBe("WALKING_SPEED");
|
||||
expect(measuredAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it("routes the v1.8.5 gait/mobility types through the drain", async () => {
|
||||
const { mock, upsert } = buildPrismaMock({
|
||||
WALKING_ASYMMETRY: [
|
||||
row("a", 2.0, "2026-05-01T08:00:00.000Z", null, "%", "WALKING_ASYMMETRY"),
|
||||
row("b", 4.0, "2026-05-01T09:00:00.000Z", null, "%", "WALKING_ASYMMETRY"),
|
||||
],
|
||||
WALKING_HEART_RATE_AVERAGE: [
|
||||
row(
|
||||
"c",
|
||||
90,
|
||||
"2026-05-01T08:00:00.000Z",
|
||||
null,
|
||||
"count/min",
|
||||
"WALKING_HEART_RATE_AVERAGE",
|
||||
),
|
||||
row(
|
||||
"d",
|
||||
110,
|
||||
"2026-05-01T09:00:00.000Z",
|
||||
null,
|
||||
"count/min",
|
||||
"WALKING_HEART_RATE_AVERAGE",
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const summary = await consolidateDailyMean(mock, { log: () => {} });
|
||||
|
||||
// One consolidated daily row per type — the day's MEAN, not the sum.
|
||||
const byType = new Map(
|
||||
upsert.mock.calls.map((c) => {
|
||||
const arg = c[0] as {
|
||||
where: { userId_type_source_externalId: { type: string } };
|
||||
create: { value: number };
|
||||
};
|
||||
return [arg.where.userId_type_source_externalId.type, arg.create.value];
|
||||
}),
|
||||
);
|
||||
expect(byType.get("WALKING_ASYMMETRY")).toBeCloseTo(3.0, 6);
|
||||
expect(byType.get("WALKING_HEART_RATE_AVERAGE")).toBeCloseTo(100, 6);
|
||||
expect(summary.totals.daysConsolidated).toBe(2);
|
||||
// Recompute fires once per consolidated (type, day).
|
||||
expect(recomputeBucketsForMeasurement).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("never scans PULSE even alongside the gait types", async () => {
|
||||
const { mock, findManyMeasurement } = buildPrismaMock({});
|
||||
await consolidateDailyMean(mock, { log: () => {} });
|
||||
const scannedTypes = findManyMeasurement.mock.calls.map(
|
||||
(c) => (c[0] as { where: { type: string } }).where.type,
|
||||
);
|
||||
expect(scannedTypes).toContain("WALKING_ASYMMETRY");
|
||||
expect(scannedTypes).toContain("WALKING_STEADINESS");
|
||||
expect(scannedTypes).not.toContain("PULSE");
|
||||
});
|
||||
|
||||
it("does not recompute on a dry-run", async () => {
|
||||
const { mock } = buildPrismaMock({
|
||||
WALKING_SPEED: [row("a", 1.0, "2026-05-01T08:00:00.000Z")],
|
||||
});
|
||||
await consolidateDailyMean(mock, { dryRun: true, log: () => {} });
|
||||
expect(recomputeBucketsForMeasurement).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes the grace cutoff into the scan when cutoffHours is set", async () => {
|
||||
|
||||
@@ -527,6 +527,17 @@ export const HIGH_FREQUENCY_MEAN_TYPES: ReadonlySet<MeasurementType> = new Set<M
|
||||
"AUDIO_EXPOSURE_HEADPHONE",
|
||||
"WALKING_SPEED",
|
||||
"WALKING_STEP_LENGTH",
|
||||
// v1.8.5 — the gait/mobility metrics Apple Health emits at sampling
|
||||
// granularity (asymmetry/double-support per walk, walking heart-rate
|
||||
// average per walk, steadiness as a near-daily rollup). They previously
|
||||
// mapped to `latest` and belonged to no consolidation set, so every
|
||||
// sample piled up raw. MEAN is the correct daily reduction and matches
|
||||
// the Health-app display; all four stay disjoint from
|
||||
// `CUMULATIVE_HK_TYPES`.
|
||||
"WALKING_ASYMMETRY",
|
||||
"WALKING_DOUBLE_SUPPORT",
|
||||
"WALKING_STEADINESS",
|
||||
"WALKING_HEART_RATE_AVERAGE",
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -12,7 +12,8 @@
|
||||
*
|
||||
* Scope per `HIGH_FREQUENCY_MEAN_TYPES`:
|
||||
* RESPIRATORY_RATE, AUDIO_EXPOSURE_ENV, AUDIO_EXPOSURE_HEADPHONE,
|
||||
* WALKING_SPEED, WALKING_STEP_LENGTH
|
||||
* WALKING_SPEED, WALKING_STEP_LENGTH, WALKING_ASYMMETRY,
|
||||
* WALKING_DOUBLE_SUPPORT, WALKING_STEADINESS, WALKING_HEART_RATE_AVERAGE
|
||||
*
|
||||
* PULSE is NOT in the set — correlation/scatter analytics read raw PULSE
|
||||
* rows. PULSE keeps raw storage; its display stays daily-averaged via
|
||||
@@ -58,6 +59,7 @@ import {
|
||||
type DayWriteOutcome,
|
||||
} from "./consolidation-base";
|
||||
import { type PerSampleRow } from "./drain-per-sample-cumulative";
|
||||
import { recomputeBucketsForMeasurement } from "@/lib/rollups/measurement-rollups";
|
||||
|
||||
/**
|
||||
* Grace window — rows newer than `now() - cutoffHours` stay raw so
|
||||
@@ -241,6 +243,17 @@ export async function consolidateDailyMean(
|
||||
});
|
||||
removed = del.count;
|
||||
});
|
||||
|
||||
// T3 — recompute the affected (user, type, day) rollup buckets after
|
||||
// the consolidation commits, the same way the batch route does on
|
||||
// ingest. Without it, a rollup-covered read could serve a pre-drain
|
||||
// mean derived from the now soft-deleted per-sample rows: the rollup
|
||||
// tier reads only `deleted_at IS NULL`, so the stale DAY bucket must
|
||||
// be re-aggregated against the single consolidated row. The
|
||||
// canonical timestamp is local-noon of the consolidated day, which
|
||||
// lands the recompute on the correct UTC day bucket.
|
||||
await recomputeBucketsForMeasurement(userId, type, canonicalTimestamp);
|
||||
|
||||
return { kind: "written", sourceRowsRemoved: removed };
|
||||
},
|
||||
recordBucket: ({
|
||||
|
||||
Reference in new issue
Block a user