feat(arrivals): add the data-arrival event spine

Every ingest path in the product can now emit a typed, salience-classified
event into pg-boss, consumed by one worker that claims a durable per-day
reaction marker and fans out to the reaction surfaces.

The recency classifier runs at the seam, before anything is enqueued, so a
mass backfill — a multi-year Apple Health export, a provider re-sync, a
30-day catch-up — emits nothing at all. Non-salient writes are annotated and
dropped.

The worker makes no provider call by construction: it imports no provider
module, writes only markers, and enqueues nothing that is not budget-gated
downstream. Business refusals return a status rather than throwing, so a
refusal is never retried against a ceiling that will not move until the local
day rolls over.

Seams: sleep (one hook, inherited by all seven transports), workouts (the
batch route, the four provider syncs, the export importer), weight and blood
pressure (both measurement routes), labs (manual, OCR commit, document fact
commit).

The provider workout syncs could not previously tell a created row from an
updated one, so a re-sync of a recent session would have re-fired on every
poll; they now recover the distinction from the row the upsert already
returns.

Reaction lines are AES-256-GCM at rest, following the InsightNarrative
precedent, and the column is registered for key rotation. Markers are pruned
after 14 days by a daily maintenance job.
This commit is contained in:
Marc-André Bombeck committed 2026-07-19 13:17:34 +02:00
1 parent 571e415eaa
commit 5d62236412
25 files changed
+1302 -28

No files matched your search

+74
View File
@@ -68,6 +68,7 @@ import { withIdempotency } from "@/lib/idempotency";
import { checkRateLimit } from "@/lib/rate-limit";
import { enqueuePrDetection } from "@/lib/jobs/pr-detection";
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
import { emitDataArrival } from "@/lib/arrivals/emit-shared";
import {
createBatchWorkoutSchema,
MAX_WORKOUTS_PER_BATCH,
@@ -118,6 +119,65 @@ interface EntryResult {
reason?: string;
}
/**
* v1.31.0 — emit one data arrival per workout this batch actually inserted.
*
* `createMany` does not return ids, so they are recovered with a single indexed
* lookup on the `(userId, source, externalId)` dedup key. Entries without an
* external id cannot be looked up and are emitted without a `refId`; the spine
* then falls back to a day-scoped key for them, which is an honest degradation
* (no external id means no stable per-workout referent to react to anyway).
*
* Fully best-effort — every failure path returns quietly. A reaction is never
* worth failing an ingest that already succeeded.
*/
async function emitWorkoutArrivals(
userId: string,
prepared: ReadonlyArray<{
index: number;
row: Prisma.WorkoutCreateManyInput;
dedupKey: { source: string; externalId: string } | null;
}>,
results: ReadonlyArray<EntryResult | undefined>,
): Promise<void> {
const inserted = prepared.filter(
(p) => results[p.index]?.status === "inserted",
);
if (inserted.length === 0) return;
const externalIds = inserted
.map((p) => p.dedupKey?.externalId)
.filter((id): id is string => typeof id === "string");
const idByExternalId = new Map<string, string>();
if (externalIds.length > 0) {
const rows = await prisma.workout.findMany({
where: { userId, externalId: { in: externalIds } },
select: { id: true, externalId: true },
});
for (const row of rows) {
if (row.externalId) idByExternalId.set(row.externalId, row.id);
}
}
for (const p of inserted) {
const startedAt =
p.row.startedAt instanceof Date
? p.row.startedAt
: new Date(p.row.startedAt as string);
if (Number.isNaN(startedAt.getTime())) continue;
const externalId = p.dedupKey?.externalId;
await emitDataArrival({
userId,
kind: "workout",
newestSampleAt: startedAt,
insertedCount: 1,
refId: externalId ? idByExternalId.get(externalId) : undefined,
source: "batch",
});
}
}
export const POST = apiHandler(withIdempotency<[NextRequest]>(postBatch));
async function postBatch(request: NextRequest): Promise<Response> {
@@ -673,6 +733,20 @@ async function postBatch(request: NextRequest): Promise<Response> {
// workout-derived metrics.
if (insertedCount > 0) {
invalidateUserMeasurements(user.id);
// v1.31.0 — the workout arm of the data-arrival spine.
//
// Unlike every other kind, workouts emit ONE arrival PER workout, keyed by
// the workout's own id. Two sessions in a day are two events and each
// deserves its own reaction; a day-scoped key would silently collapse the
// second one.
//
// That needs the ids, and `createMany` does not return them, so this costs
// one extra indexed lookup — but only on a batch that actually inserted
// something, and only for entries carrying an external id. The classifier
// still runs per emit, so a historical import walks straight into
// `backfill` and enqueues nothing regardless of how many rows it wrote.
void emitWorkoutArrivals(user.id, prepared, results).catch(() => {});
}
return apiSuccess({