mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-12 22:26:13 +00:00
feat(arrivals): add the data-arrival event spine
Every ingest path in the product can now emit a typed, salience-classified event into pg-boss, consumed by one worker that claims a durable per-day reaction marker and fans out to the reaction surfaces. The recency classifier runs at the seam, before anything is enqueued, so a mass backfill — a multi-year Apple Health export, a provider re-sync, a 30-day catch-up — emits nothing at all. Non-salient writes are annotated and dropped. The worker makes no provider call by construction: it imports no provider module, writes only markers, and enqueues nothing that is not budget-gated downstream. Business refusals return a status rather than throwing, so a refusal is never retried against a ceiling that will not move until the local day rolls over. Seams: sleep (one hook, inherited by all seven transports), workouts (the batch route, the four provider syncs, the export importer), weight and blood pressure (both measurement routes), labs (manual, OCR commit, document fact commit). The provider workout syncs could not previously tell a created row from an updated one, so a re-sync of a recent session would have re-fired on every poll; they now recover the distinction from the row the upsert already returns. Reaction lines are AES-256-GCM at rest, following the InsightNarrative precedent, and the column is registered for key rotation. Markers are pruned after 14 days by a daily maintenance job.
This commit is contained in:
1 parent
571e415eaa
commit
5d62236412
25 files changed
+1302
-28
No files matched your search
@@ -66,6 +66,8 @@ import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import { enqueuePrDetection } from "@/lib/jobs/pr-detection";
|
||||
import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
|
||||
import { maybeEnqueueMorningRefresh } from "@/lib/daily/morning-refresh-trigger";
|
||||
import { emitDataArrival } from "@/lib/arrivals/emit-shared";
|
||||
import { groupRowsByArrivalKind } from "@/lib/arrivals/measurement-kind";
|
||||
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
|
||||
import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate";
|
||||
import {
|
||||
@@ -177,6 +179,39 @@ type BatchEntry = z.infer<typeof batchEntrySchema>;
|
||||
*/
|
||||
type EntryStatus = "inserted" | "updated" | "duplicate" | "skipped";
|
||||
|
||||
/**
|
||||
* v1.31.0 — the rows this write actually INSERTED, in the shape the arrival
|
||||
* kind grouper takes.
|
||||
*
|
||||
* Restricting to `status === "inserted"` is what keeps a re-sync silent: an
|
||||
* `updated` row is the `stats:` overwrite contract re-stating a value the
|
||||
* record already had, and a `duplicate` row landed nothing. The per-entry
|
||||
* statuses are reconciled against the insert race upstream, so the count is
|
||||
* accurate; in a rare concurrent-write tie it can be off on WHICH row, which
|
||||
* cannot change the outcome because the day-scoped singleton key collapses the
|
||||
* result either way.
|
||||
*/
|
||||
function insertedArrivalRows(
|
||||
prepared: ReadonlyArray<{
|
||||
index: number;
|
||||
row: Prisma.MeasurementCreateManyInput;
|
||||
}>,
|
||||
results: ReadonlyArray<{ status: EntryStatus } | undefined>,
|
||||
): Array<{ type: MeasurementType; measuredAt: Date }> {
|
||||
const rows: Array<{ type: MeasurementType; measuredAt: Date }> = [];
|
||||
for (const p of prepared) {
|
||||
if (results[p.index]?.status !== "inserted") continue;
|
||||
rows.push({
|
||||
type: p.row.type as MeasurementType,
|
||||
measuredAt:
|
||||
p.row.measuredAt instanceof Date
|
||||
? p.row.measuredAt
|
||||
: new Date(p.row.measuredAt as string),
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// v1.5.0 issue #213 — `stats:*` externalIds carry per-day cumulative
|
||||
// totals that the iOS HealthKit observer re-posts as the day progresses.
|
||||
// Treating those as immutable duplicates freezes the day's tile at the
|
||||
@@ -828,6 +863,35 @@ async function postBatch(request: NextRequest): Promise<Response> {
|
||||
: new Date(p.row.measuredAt as string),
|
||||
),
|
||||
).catch(() => {});
|
||||
|
||||
// v1.31.0 — the measurement arm of the data-arrival spine.
|
||||
//
|
||||
// ONE emit per kind per request, never one per row. The batch route is the
|
||||
// Apple-Health / iOS ingest path, so it is also the single biggest backfill
|
||||
// path in the product: a ten-year export arrives here in 500-row batches.
|
||||
// Emitting per row would put thousands of classifier calls (and thousands
|
||||
// of dropped-event annotations) on the hot path for no gain, since the
|
||||
// day-scoped singleton key collapses them to one job anyway.
|
||||
//
|
||||
// Blood pressure is deliberately one arrival, not two: it is stored as two
|
||||
// rows (SYS + DIA) sharing a `measuredAt`, and a reader cares about the
|
||||
// reading, not the arms.
|
||||
//
|
||||
// Only rows the write actually INSERTED count — `updatedCount` rows are the
|
||||
// `stats:` overwrite contract re-stating a value the record already had,
|
||||
// which is not news. If nothing was inserted the classifier returns `noop`
|
||||
// and nothing is enqueued.
|
||||
for (const [kind, group] of groupRowsByArrivalKind(
|
||||
insertedArrivalRows(prepared, results),
|
||||
)) {
|
||||
void emitDataArrival({
|
||||
userId: user.id,
|
||||
kind,
|
||||
newestSampleAt: group.newestAt,
|
||||
insertedCount: group.count,
|
||||
source: "batch",
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
return apiSuccess({
|
||||
|
||||
@@ -32,6 +32,11 @@ import {
|
||||
import { withIdempotency } from "@/lib/idempotency";
|
||||
import { encryptNote, shapeMeasurementNotes } from "@/lib/crypto/note-cipher";
|
||||
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
|
||||
import { emitDataArrival } from "@/lib/arrivals/emit-shared";
|
||||
import {
|
||||
ARRIVAL_MEASUREMENT_KIND,
|
||||
groupRowsByArrivalKind,
|
||||
} from "@/lib/arrivals/measurement-kind";
|
||||
import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate";
|
||||
import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
|
||||
import { runSafetyFloorCheck } from "@/lib/illness/safety-floor-check";
|
||||
@@ -846,6 +851,21 @@ async function postMeasurement(request: NextRequest) {
|
||||
action: "reminder.satisfy.enqueue",
|
||||
});
|
||||
|
||||
// v1.31.0 — the interactive arm of the data-arrival spine. This route is
|
||||
// the combined BP + pulse form, so it lands both BP arms in one
|
||||
// transaction; they are one reading and produce one arrival, which the
|
||||
// kind map already collapses. Every row here is a real insert (the
|
||||
// transaction throws rather than de-duplicating), so the count is exact.
|
||||
for (const [kind, group] of groupRowsByArrivalKind(results)) {
|
||||
void emitDataArrival({
|
||||
userId: user.id,
|
||||
kind,
|
||||
newestSampleAt: group.newestAt,
|
||||
insertedCount: group.count,
|
||||
source: "manual",
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
// v1.18.6 — absolute clinical safety-floor check on the just-written
|
||||
// readings (confirm-gated, module-gated, never diagnoses). The combined
|
||||
// BP form lands both arms here, so the BP floor sees a whole reading.
|
||||
@@ -998,6 +1018,22 @@ async function postMeasurement(request: NextRequest) {
|
||||
action: "reminder.satisfy.enqueue",
|
||||
});
|
||||
|
||||
// v1.31.0 — the single-entry arm of the data-arrival spine. One row, so at
|
||||
// most one kind; a type the spine does not track maps to null and emits
|
||||
// nothing.
|
||||
{
|
||||
const kind = ARRIVAL_MEASUREMENT_KIND[measurement.type];
|
||||
if (kind) {
|
||||
void emitDataArrival({
|
||||
userId: user.id,
|
||||
kind,
|
||||
newestSampleAt: measurement.measuredAt,
|
||||
insertedCount: 1,
|
||||
source: "manual",
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
// v1.18.6 — absolute clinical safety-floor check (confirm-gated,
|
||||
// module-gated, never diagnoses). A single-entry POST carries only one arm
|
||||
// of a BP reading, so the BP floor only fires through the combined-form /
|
||||
|
||||
Reference in new issue
Block a user