mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 15:42:24 +00:00
feat(arrivals): add the data-arrival event spine
Every ingest path in the product can now emit a typed, salience-classified event into pg-boss, consumed by one worker that claims a durable per-day reaction marker and fans out to the reaction surfaces. The recency classifier runs at the seam, before anything is enqueued, so a mass backfill — a multi-year Apple Health export, a provider re-sync, a 30-day catch-up — emits nothing at all. Non-salient writes are annotated and dropped. The worker makes no provider call by construction: it imports no provider module, writes only markers, and enqueues nothing that is not budget-gated downstream. Business refusals return a status rather than throwing, so a refusal is never retried against a ceiling that will not move until the local day rolls over. Seams: sleep (one hook, inherited by all seven transports), workouts (the batch route, the four provider syncs, the export importer), weight and blood pressure (both measurement routes), labs (manual, OCR commit, document fact commit). The provider workout syncs could not previously tell a created row from an updated one, so a re-sync of a recent session would have re-fired on every poll; they now recover the distinction from the row the upsert already returns. Reaction lines are AES-256-GCM at rest, following the InsightNarrative precedent, and the column is registered for key rotation. Markers are pruned after 14 days by a daily maintenance job.
This commit is contained in:
1 parent
571e415eaa
commit
5d62236412
25 files changed
+1302
-28
No files matched your search
@@ -131,6 +131,40 @@ export async function handlePushAttemptCleanup(
|
||||
});
|
||||
}
|
||||
|
||||
// v1.31.0 — daily prune for the data-arrival spine's reaction markers.
|
||||
//
|
||||
// Fourteen days rather than the ledgers' 90: a reaction marker is a
|
||||
// same-day surface ("just in", today's one generated line), so a row stops
|
||||
// being read the moment its local day rolls over. Two weeks is purely a
|
||||
// debugging margin — long enough to reconstruct what the spine reacted to
|
||||
// across a reported incident, short enough that a chatty account's markers
|
||||
// never accumulate. The `created_at` index makes the trailing-edge scan
|
||||
// cheap, exactly as it does for the push-attempt ledger above.
|
||||
const ARRIVAL_REACTION_RETENTION_DAYS = 14;
|
||||
|
||||
export interface ArrivalReactionCleanupPayload {
|
||||
triggeredAt: string;
|
||||
}
|
||||
|
||||
export async function handleArrivalReactionCleanup(
|
||||
jobs: Job<ArrivalReactionCleanupPayload>[],
|
||||
) {
|
||||
void jobs;
|
||||
await withBackgroundEvent("job.arrival_reaction_cleanup", async (evt) => {
|
||||
const p = getWorkerPrisma();
|
||||
try {
|
||||
const cutoff = new Date();
|
||||
cutoff.setUTCDate(cutoff.getUTCDate() - ARRIVAL_REACTION_RETENTION_DAYS);
|
||||
const deleted = await p.arrivalReaction.deleteMany({
|
||||
where: { createdAt: { lt: cutoff } },
|
||||
});
|
||||
evt.addMeta("arrival_reaction_cleanup_deleted", deleted.count);
|
||||
} catch (err) {
|
||||
evt.addWarning(`arrival-reaction-cleanup failed: ${err}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export interface MeasurementTombstoneCleanupPayload {
|
||||
triggeredAt: string;
|
||||
}
|
||||
|
||||
@@ -140,6 +140,8 @@ import {
|
||||
handleMoodReminderCleanup,
|
||||
PushAttemptCleanupPayload,
|
||||
handlePushAttemptCleanup,
|
||||
ArrivalReactionCleanupPayload,
|
||||
handleArrivalReactionCleanup,
|
||||
MeasurementTombstoneCleanupPayload,
|
||||
handleMeasurementTombstoneCleanup,
|
||||
handleRateLimitCleanup,
|
||||
@@ -231,6 +233,13 @@ const PUSH_ATTEMPT_CLEANUP_QUEUE = "push-attempt-cleanup";
|
||||
|
||||
const PUSH_ATTEMPT_CLEANUP_CRON = "35 3 * * *";
|
||||
|
||||
// v1.31.0 — daily prune for the data-arrival spine's reaction markers.
|
||||
// Slots at 04:10, after the 03:xx retention block and the 04:00 MCP-token
|
||||
// prune, so the maintenance window stays in a readable order.
|
||||
const ARRIVAL_REACTION_CLEANUP_QUEUE = "arrival-reaction-cleanup";
|
||||
|
||||
const ARRIVAL_REACTION_CLEANUP_CRON = "10 4 * * *";
|
||||
|
||||
const MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE = "measurement-tombstone-cleanup";
|
||||
|
||||
const MEASUREMENT_TOMBSTONE_CLEANUP_CRON = "40 3 * * *";
|
||||
@@ -307,6 +316,9 @@ const allQueues = [
|
||||
// v1.4.49 — push-attempt ledger cleanup. The daily schedule below would
|
||||
// silently no-op without this entry.
|
||||
PUSH_ATTEMPT_CLEANUP_QUEUE,
|
||||
// v1.31.0 — data-arrival reaction-marker prune. Without this entry the
|
||||
// daily schedule silently no-ops and the markers accumulate forever.
|
||||
ARRIVAL_REACTION_CLEANUP_QUEUE,
|
||||
// v1.7.0 — soft-deleted measurement tombstone prune. Without this entry
|
||||
// the daily schedule silently no-ops and pruned-past-retention tombstones
|
||||
// pile up forever.
|
||||
@@ -419,6 +431,8 @@ const schedules: ScheduleEntry[] = [
|
||||
[INTAKE_SLOT_DEDUP_QUEUE, INTAKE_SLOT_DEDUP_CRON],
|
||||
// v1.4.49 — daily 03:35 Europe/Berlin prune for push_attempts.
|
||||
[PUSH_ATTEMPT_CLEANUP_QUEUE, PUSH_ATTEMPT_CLEANUP_CRON],
|
||||
// v1.31.0 — daily 04:10 Europe/Berlin prune for arrival_reactions.
|
||||
[ARRIVAL_REACTION_CLEANUP_QUEUE, ARRIVAL_REACTION_CLEANUP_CRON],
|
||||
// v1.7.0 — daily 03:40 Europe/Berlin prune for expired measurement
|
||||
// tombstones.
|
||||
[MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, MEASUREMENT_TOMBSTONE_CLEANUP_CRON],
|
||||
@@ -597,6 +611,14 @@ export async function registerMaintenanceQueues(
|
||||
{ localConcurrency: 1 },
|
||||
handlePushAttemptCleanup,
|
||||
);
|
||||
// v1.31.0 — daily prune of the data-arrival reaction markers. Single-flight
|
||||
// like every other cleanup queue: two ticks racing the same DELETE is
|
||||
// wasted work.
|
||||
await boss.work<ArrivalReactionCleanupPayload>(
|
||||
ARRIVAL_REACTION_CLEANUP_QUEUE,
|
||||
{ localConcurrency: 1 },
|
||||
handleArrivalReactionCleanup,
|
||||
);
|
||||
// v1.7.0 — daily prune of expired measurement tombstones. Single-flight
|
||||
// like every other cleanup queue.
|
||||
await boss.work<MeasurementTombstoneCleanupPayload>(
|
||||
|
||||
@@ -29,6 +29,8 @@ import {
|
||||
runMorningDigestRefresh,
|
||||
type MorningDigestRefreshPayload,
|
||||
} from "@/lib/jobs/morning-digest-refresh";
|
||||
import { DATA_ARRIVAL_QUEUE, handleDataArrival } from "@/lib/jobs/data-arrival";
|
||||
import type { DataArrival } from "@/lib/arrivals/types";
|
||||
import {
|
||||
DAILY_BRIEFING_QUEUE,
|
||||
DAILY_BRIEFING_CRON,
|
||||
@@ -171,6 +173,12 @@ const allQueues = [
|
||||
// send-only queue driven by sleep ingest; without this entry pg-boss never
|
||||
// provisions the queue and every enqueue silently drops.
|
||||
MORNING_DIGEST_REFRESH_QUEUE,
|
||||
// v1.31.0 — the data-arrival spine. Every salient ingest emits here; the
|
||||
// worker claims the day's reaction marker and fans out to the reaction
|
||||
// surfaces. No cron schedule — a send-only queue driven by ingest; without
|
||||
// this entry pg-boss never provisions the queue and every emit silently
|
||||
// drops (the v1.4.37 dead-queue class).
|
||||
DATA_ARRIVAL_QUEUE,
|
||||
// v1.10.0 — computed scores (WX-C / WX-E). Nightly Recovery / Stress /
|
||||
// Strain compute + store. The queue MUST be registered here or pg-boss
|
||||
// never provisions it and the nightly schedule silently never fires.
|
||||
@@ -275,9 +283,30 @@ const schedules: ScheduleEntry[] = [
|
||||
* differently-shaped constraint over a working one for no gain, so they are
|
||||
* left alone.
|
||||
*
|
||||
* That leaves exactly one queue here whose key is inert today.
|
||||
* That leaves exactly two queues here whose keys would otherwise be inert.
|
||||
*/
|
||||
const queuePolicies: QueuePolicyTable = {
|
||||
// v1.31.0 — the arrival spine's day-scoped keys
|
||||
// (`arrival:<user>:<kind>:<localDate>`) carry the SAME requirement as the
|
||||
// morning refresh below, for the same reason: a user's local date cannot be
|
||||
// expressed as a wall-clock `singletonSeconds` window, so the date lives in
|
||||
// the key and the policy has to honour it. Under `standard` the key
|
||||
// coalesces nothing at all, and a device posting a batch per minute would
|
||||
// queue a job per batch instead of one per day.
|
||||
//
|
||||
// `exclusive` rather than `short` because the handler is self-converging:
|
||||
// it claims a durable unique row, so a second run while the first is active
|
||||
// is pure duplicated work, and the next ingest re-emits anyway.
|
||||
//
|
||||
// The `workout` kind is the deliberate exception — it is keyed per workout
|
||||
// id and DOES pass `singletonSeconds`, so it is constrained by pg-boss's
|
||||
// `job_i4` index regardless of this policy. Both shapes coexist on one
|
||||
// queue safely: they populate different columns.
|
||||
[DATA_ARRIVAL_QUEUE]: {
|
||||
policy: "exclusive",
|
||||
reason:
|
||||
"Day-scoped arrival keys carry the user's local date; under standard they coalesce nothing and a chatty ingest queues a job per batch. The worker claims a durable unique row, so a concurrent second run is pure duplicated work.",
|
||||
},
|
||||
// The sleep-arrival trigger has seven write seams that can each enqueue for
|
||||
// the same user and the same local date. The key is
|
||||
// `morning-refresh:<user>:<localDate>`, and `exclusive` turns that into a
|
||||
@@ -494,6 +523,15 @@ export async function registerStatusQueues(
|
||||
});
|
||||
},
|
||||
);
|
||||
// v1.31.0 — the data-arrival spine. Single-flight: the worker's whole job is
|
||||
// to claim a durable unique row, so two slots would only race each other for
|
||||
// a claim exactly one of them can win. It makes no provider call — a
|
||||
// module-graph test pins that — so one slot costs nothing in throughput.
|
||||
await boss.work<DataArrival>(
|
||||
DATA_ARRIVAL_QUEUE,
|
||||
{ localConcurrency: 1 },
|
||||
handleDataArrival,
|
||||
);
|
||||
// S5 — daily-briefing fallback slot. Single-flight; the `push_attempts`
|
||||
// frequency cap makes an overlapping tick a no-op (a second same-day attempt
|
||||
// is suppressed), and the tick only loads a digest for opted-in users at
|
||||
|
||||
Reference in new issue
Block a user